diff --git a/examples/harbor-hermes-switchyard/.env.example b/examples/harbor-hermes-switchyard/.env.example new file mode 100644 index 000000000..723c15f41 --- /dev/null +++ b/examples/harbor-hermes-switchyard/.env.example @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Copy this file to .env, chmod 0600, and replace every /absolute +# placeholder. Never commit the populated file. +EXAMPLE_ROOT=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard +TERMINAL_BENCH_RUN_ID=harbor-hermes-switchyard-phase2-run-1 +TERMINAL_BENCH_RUN_ROOT=/absolute/path/to/phase2-runs/harbor-hermes-switchyard-phase2-run-1 +TERMINAL_BENCH_ADMISSION_ROOT=/absolute/path/to/phase2-admission +# Automatically populated with a content-addressed Hermes runtime. Users do +# not pre-build task images or this runtime. +TERMINAL_BENCH_BOOTSTRAP_ROOT=/absolute/path/to/phase2-admission/bootstrap + +HARBOR_BIN=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/.venv/bin/harbor +EVAL_PYTHON=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/.venv/bin/python +TBENCH_DATASET_PATH=/absolute/path/to/exported/terminal-bench +SWITCHYARD_BUNDLE=/absolute/path/to/pinned-switchyard-bundle +RELAY_WHEEL=/absolute/path/to/nemo_relay-0.7.1-platform-wheel.whl +RELAY_ARCHITECTURE=x86_64 +PLUGIN_CONFIG_TEMPLATE=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/config/plugins.toml.in + +TERMINAL_BENCH_SMOKE_EVIDENCE=/absolute/path/to/phase2-admission/all-89-smoke.json +TERMINAL_BENCH_OFFLINE_EVIDENCE=/absolute/path/to/phase2-admission/offline-admission.json +PHOENIX_BASE_URL=https://your-phoenix-endpoint +PHOENIX_PROJECT=harbor-hermes-switchyard-phase2-run-1 +EVAL_COHORT=harbor-hermes-switchyard-phase2-run-1 + +TBENCH_SAMPLE_COUNT=89 +# Default: run this task alone before opening the full cohort lanes. +# Set an explicitly blank value to begin the full cohort immediately. +TBENCH_CANARY_TASK=adaptive-rejection-sampler +TBENCH_CONCURRENCY=4 +# Environment preparation is intentionally quieter than provider execution. +TBENCH_SETUP_CONCURRENCY=2 +TBENCH_SETUP_BATCH_SIZE=89 +TBENCH_SETUP_MAX_INFRA_ATTEMPTS=4 +TBENCH_PARALLEL_MAX_MEMORY_GB=2 +TBENCH_DOCKER_MEMORY_RESERVE_GB=4 +TBENCH_MINIMUM_FREE_GB=100 + +# This is the real Authorization header consumed by the Switchyard plugin. +# Quote the value so shell metacharacters are not interpreted when sourced. +SWITCHYARD_PROVIDER_AUTHORIZATION='Bearer replace-with-provider-token' diff --git a/examples/harbor-hermes-switchyard/.gitignore b/examples/harbor-hermes-switchyard/.gitignore new file mode 100644 index 000000000..4c49bd78f --- /dev/null +++ b/examples/harbor-hermes-switchyard/.gitignore @@ -0,0 +1 @@ +.env diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md new file mode 100644 index 000000000..bd1c35bb7 --- /dev/null +++ b/examples/harbor-hermes-switchyard/README.md @@ -0,0 +1,316 @@ +# Harbor + Hermes + Switchyard evaluation + +This example runs one complete Terminal-Bench 2.0 cohort through Harbor and +Hermes. Hermes owns an in-process NeMo Relay runtime satisfying `nemo-relay>=0.7.0`; Relay loads the +Switchyard native plugin, and Switchyard selects and calls the configured +provider route. It runs one resumable 89-task cohort. Multi-cohort execution +and result aggregation are intentionally out of scope for this example. + +## 1. Pinned inputs + +| Dependency | Input used by this example | +|---|---| +| NeMo Relay | Latest released `nemo-relay>=0.7.0` platform wheel, installed by digest rather than from this source checkout. | +| Hermes | `bbednarski9/hermes-agent`, detached commit `a3d472f0e6bdc376df87b1436a461c4796db6747` from PR #77915. | +| Switchyard | `bbednarski9/Switchyard`, detached commit `8daac03edf8544144833af1fd009b3da737715bc` from PR #270. | +| Harbor | `harbor==0.18.0`, local export of dataset `terminal-bench@2.0`. | + +Every source checkout is detached and verified. The Hermes installer is +followed by `uv sync --frozen`, then the selected released Relay wheel is +force-installed without dependencies and verified by digest. + +## 2. Request and lifecycle ownership + +There is no Switchyard service in this topology: + +1. Harbor owns the task container, Hermes lifecycle, timeout, verifier, and + task artifact collection. +2. The temporary adapter in `agents/harbor_hermes_agent.py` installs the exact + Hermes commit and projects Relay's config and native bundle. +3. Hermes initializes Relay; Relay's public loader activates + `nvidia.switchyard` from `[[plugins.dynamic]]`. +4. Relay dispatches the managed operation into the native intercept. +5. Switchyard selects a route and its client owns the provider HTTP request. +6. Hermes waits for operations, plugins, subscribers, and exporters before + returning to Harbor. + +The adapter can be removed after +[hermes-agent#77915](https://github.com/NousResearch/hermes-agent/pull/77915) +is upstream and Harbor can install an immutable compatible Hermes revision +while projecting the Relay configuration and plugin bundle. + +## 3. Configuration ownership + +The two configuration files have deliberately different responsibilities: + +- `.env.example` is copied to an untracked, mode-`0600` + `.env`. It contains per-machine paths, the run identity, Phoenix + destination, manually selected capacity, and the real + `SWITCHYARD_PROVIDER_AUTHORIZATION` header. +- `config/plugins.toml.in` is checked in and non-secret. It is the only source + of provider URLs, protocols, strong, weak, and judge models, routing/classifier + policy, native plugin manifest, authorization variable **name**, Relay + components, and OpenInference export behavior. + +The template configures AWS-hosted Opus 5 as the strong route, AWS-hosted +Sonnet 5 as the efficient route, and AWS-hosted Sonnet 4.6 as the classifier +judge, with a `0.5` threshold and +session affinity. The coordinator derives its required route-diversity gates +from the strong and efficient targets, while preflight also verifies the judge. +Environment variables cannot override these settings. +Runtime rendering is limited to the Hermes revision, collector endpoint, +Phoenix project/cohort attributes, and task-owned artifact locations. + +For each task, the runner writes only the provider Authorization header to a +mode-`0600` file in the host's canonical private temporary directory, +explicitly outside the run root, and bind-mounts it read-only at +`/run/secrets/switchyard-provider-authorization`. +The Hermes bridge reads and exports it inside the task container immediately +before the agent command. The credential value is therefore absent from Harbor +configuration, Docker Compose arguments, plans, logs, and retained evidence; +the temporary file is removed when the task runner exits. + +Harbor still requires a caller model. The example uses the intentionally +unserved `openai/ollama-route-stub` identity and projects a dead local OpenAI +endpoint. If Switchyard is bypassed, the request fails closed instead of +reaching a provider. + +## 4. Host prerequisites + +- Linux or macOS, Bash, Python 3.11+, Docker, and `tmux`; +- a local, immutable Terminal-Bench 2.0 dataset export containing 89 tasks; +- a Switchyard plugin bundle and released Relay wheel satisfying `nemo-relay>=0.7.0`, matching Docker's + architecture (`x86_64` or `aarch64`); +- a Phoenix endpoint accepting OTLP/HTTP OpenInference traces; and +- provider and registry access for the full cohort. The all-89 admission uses + neither; the Docker admission makes no provider calls but may pull its image, + pinned sources, and packages when they are not cached. + +On macOS, keep the dataset, bundle, wheel, admission, and run roots under a +directory shared with Docker (normally `/Users/...`). + +Install the exact Harbor-side requirements: + +```bash +cd examples/harbor-hermes-switchyard +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt +``` + +Copy and protect the environment file at the example root. Replace every +placeholder, including the complete provider Authorization header. Do not +source this file into the interactive shell used to start `tmux`. + +```bash +cp .env.example .env +chmod 0600 .env +./scripts/validate_phase2_environment.sh .env +``` + +The validator reports names and paths only. It rejects legacy secret-file +variables and never renders or prints the authorization value. + +## 5. Prepare and validate a cohort run + +Run every stage with the same immutable inputs. If the dataset, concurrency, +architecture, Relay wheel, Switchyard library, plugin template, or Hermes +commit changes, regenerate the affected admission evidence before creating a +plan. + +For the commands below, enter a short-lived shell with tracing disabled: + +```bash +set +x +set -a +source .env +set +a +set +x +``` + +## 6. Verify the complete dataset without provider tokens + +This all-89 no-token admission loads and uniquely selects all tasks, hashes their instructions and +verifiers, expands the complete Harbor job graph, denies registry/provider +access, and renders the runtime. It starts neither Docker nor an agent. + +```bash +mkdir -p "$TERMINAL_BENCH_ADMISSION_ROOT" +chmod 0700 "$TERMINAL_BENCH_ADMISSION_ROOT" +"$EVAL_PYTHON" "$EXAMPLE_ROOT/scripts/smoke_phase2_dataset.py" \ + --dataset-root "$TBENCH_DATASET_PATH" \ + --expected-count 89 \ + --concurrency "$TBENCH_CONCURRENCY" \ + --harbor-bin "$HARBOR_BIN" \ + --switchyard-bundle "$SWITCHYARD_BUNDLE" \ + --relay-wheel "$RELAY_WHEEL" \ + --relay-architecture "$RELAY_ARCHITECTURE" \ + --plugin-config-template "$PLUGIN_CONFIG_TEMPLATE" \ + --output "$TERMINAL_BENCH_SMOKE_EVIDENCE" +``` + +The passed evidence binds task names, task/instruction/verifier hashes, +concurrency, architecture, Relay wheel, Switchyard library, and plugin config. + +## 7. Verify the offline container runtime + +The Docker offline runtime admission uses a fresh admission root with test-only +structured overrides. Production +model, URL, and routing values remain owned by `plugins.toml.in`; these flags +exist only to point this closed offline test at its fake endpoints. + +```bash +OFFLINE_ROOT="$TERMINAL_BENCH_ADMISSION_ROOT/offline-runtime" +"$EVAL_PYTHON" "$EXAMPLE_ROOT/scripts/prepare_runtime.py" \ + --run-root "$OFFLINE_ROOT" \ + --switchyard-bundle "$SWITCHYARD_BUNDLE" \ + --relay-wheel "$RELAY_WHEEL" \ + --relay-architecture "$RELAY_ARCHITECTURE" \ + --plugin-config-template "$PLUGIN_CONFIG_TEMPLATE" \ + --test-provider-base-url http://127.0.0.1:8000/v1 \ + --test-strong-model phase2/fake-strong \ + --test-weak-model phase2/fake-weak \ + --test-judge-model phase2/fake-judge \ + --openinference-endpoint http://127.0.0.1:4318/v1/traces \ + --phoenix-project phase2-offline \ + --eval-cohort phase2-offline + +case "$RELAY_ARCHITECTURE" in + x86_64) export OFFLINE_COMPAT_PLATFORM=linux/amd64 ;; + aarch64) export OFFLINE_COMPAT_PLATFORM=linux/arm64 ;; + *) echo "unsupported architecture" >&2; return 2 ;; +esac +"$EXAMPLE_ROOT/scripts/run_offline_compatibility_smoke.sh" \ + "$OFFLINE_ROOT" "$TERMINAL_BENCH_OFFLINE_EVIDENCE" +``` + +This performs real Hermes→Relay→Switchyard calls against local fake provider +and OTLP endpoints and proves route selection, authorization injection, +observability, pinned-library loading, and clean shutdown. Its evidence binds +the same Hermes commit, Relay wheel, Switchyard library, architecture, and +plugin template consumed by the cohort. + +## 8. Create the immutable run plan + +The first command writes `plan.json`; any later invocation with different +immutable inputs is refused. Choose concurrency before this point. + +```bash +"$EXAMPLE_ROOT/run_phase2_cohort.sh" "$TERMINAL_BENCH_RUN_ROOT" --plan-only +``` + +**Optional canary-first scheduling.** + +The default `TBENCH_CANARY_TASK=adaptive-rejection-sampler` runs that one real +task first. A completed benchmark result (`validation.benchmark.status=passed`) +opens the parallel lane even when its benchmark reward is a non-pass. Phoenix +upload and other integration evidence remain cohort-level acceptance gates. +This is a conservative production check, not a separate command: launching the +full cohort on a fresh run root automatically starts the canary and then +continues with the remaining tasks. + +To skip that one-task checkpoint, set an explicitly blank value in `.env`: + +```bash +TBENCH_CANARY_TASK= +``` + +With the canary disabled, the full cohort starts immediately. Task 1 remains +the first selected task, but it is scheduled in the normal parallel or serial +lane rather than running alone first. + +## 9. Check capacity and provider availability + +```bash +"$EXAMPLE_ROOT/run_phase2_cohort.sh" "$TERMINAL_BENCH_RUN_ROOT" --preflight-only +``` + +Preflight authenticates to each configured provider's model catalog and +requires both TOML-owned route models to be present without persisting the +authorization value. It writes `preflight.json` with the verified model IDs, +Docker CPU/memory/architecture, free disk, configured endpoints, selected +concurrency, reserve, and the calculated requirement. It rejects: + +```text +max(concurrency × parallel_task_memory_gb, largest_task_memory_gb) + + docker_reserve_gb > Docker memory +``` + +It also rejects less than the configured free-disk minimum (100G by default), +concurrency above Docker's CPU count, and an architecture mismatch. The +defaults are a 2G parallel lane and 4G Docker reserve. + +## 10. Launch and resume the cohort + +Exit the secret-bearing admission shell first. From a shell where the protected +file has **not** been sourced, start one detached supervisor. Only the file path +is placed in the tmux server environment; the child sources it with xtrace +disabled and persists output below the run root. +Before the supervisor starts, the launcher copies only the plan-bound harness +sources into `runtime-harness/` below the run root and verifies their aggregate +hash. Retries execute this snapshot, so later checkout changes cannot alter an +active cohort. No environment file or secret is copied into the snapshot. + +```bash +exit # only when returning from the short-lived admission shell above +./scripts/launch_phase2_tmux.sh harbor-hermes-switchyard-phase2-run-1 +``` + +Operational commands: + +```bash +# Detect a live duplicate (success means the session exists). +tmux has-session -t harbor-hermes-switchyard-phase2-run-1 + +# Attach; detach without stopping the run with Ctrl-b d. +tmux attach-session -t harbor-hermes-switchyard-phase2-run-1 + +# Inspect durable output and sanitized cohort progress. +tail -F /absolute/path/to/phase2-run-root/supervisor.log +jq '{status,completed_tasks,planned_tasks,benchmark_pass_count,benchmark_nonpass_count}' \ + /absolute/path/to/phase2-run-root/summary.json + +# Graceful interruption. +tmux send-keys -t harbor-hermes-switchyard-phase2-run-1 C-c + +# After the old session exits, resume from the run-bound snapshot. This also +# works after a checkout update or host reboot. +/absolute/path/to/phase2-run-root/runtime-harness/scripts/launch_phase2_tmux.sh \ + /absolute/path/to/examples/harbor-hermes-switchyard/.env \ + harbor-hermes-switchyard-phase2-run-1 +``` + +The supervisor returns `0` after complete acceptance, `20` for a preserved +integration/harness blocker, and retries other exits with bounded exponential +backoff. The OS advisory lock rejects a second live coordinator. Validated +attempts are immutable and preserved on restart. `tmux` survives terminal +logout, not host reboot; after reboot, launch it again against the same root. +Agent setup also retries transient `apt-get` failures three times locally; +exhausted package-manager failures are classified as infrastructure and remain +subject to the cohort's bounded retry limit. + +## 11. Completion gates + +A task is complete when `validation.json` records +`benchmark.status=passed`. The top-level validation status mirrors benchmark +completion for compatibility. +A benchmark `reward.task_passed=false` is a valid completed result and is never +retried. + +Relay/Switchyard artifact checks, including the Phoenix upload result, are +recorded separately in `validation.integration`. An integration finding is +preserved in the task and cohort report; it does not discard a completed +benchmark result or cause the agent to be run again. It remains a cohort-level +acceptance gate. + +The cohort passes only when: + +- all 89 tasks have independently completed benchmark results; +- `cohort_gates.integration_validation` passes; +- direct artifacts and logs pass secret scans; +- cache-read evidence is nonzero; +- both models derived from `plugins.toml.in` appear in committed routes; and +- `summary.json.status` is `passed`. + +`report.md` is regenerated after each completed attempt and is safe for +progress review. Running multiple cohorts and aggregating their reports are not +part of this runbook. diff --git a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py new file mode 100644 index 000000000..ff33f7a0b --- /dev/null +++ b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py @@ -0,0 +1,633 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Temporary Harbor Hermes agent for a fork-hosted, commit-pinned Hermes build. + +The class deliberately inherits Harbor's built-in Hermes lifecycle. It only +changes installation, copies an immutable Relay configuration/plugin bundle +into the task environment, and frames the additional Phase 1 artifacts around +``super().run``. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import shlex +import time +import tomllib +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +from harbor.agents.installed.hermes import Hermes +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext +from typing_extensions import override + +_FULL_SHA = re.compile(r"[0-9a-f]{40}") +_SHA256 = re.compile(r"[0-9a-f]{64}") +_DEFAULT_HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" +_DEFAULT_HERMES_REF = "feat/relay-native-plugin-init" +_DEFAULT_HERMES_COMMIT = "a3d472f0e6bdc376df87b1436a461c4796db6747" +_DEFAULT_SWITCHYARD_COMMIT = "8daac03edf8544144833af1fd009b3da737715bc" +_ENV_NAME = re.compile(r"[A-Z_][A-Z0-9_]*") +_PROVIDER_AUTHORIZATION_FILE = "/run/secrets/switchyard-provider-authorization" +_HERMETIC_RUNTIME_ROOT = "/opt/hermes-runtime" +_HERMETIC_RUNTIME_SCHEMA = "harbor-hermes-switchyard.hermetic-runtime.v1" +_HERMETIC_CA_BUNDLE_RELATIVE = Path("hermes-agent-src/venv/lib/python3.11/site-packages/certifi/cacert.pem") +_HERMETIC_CA_BUNDLE = f"{_HERMETIC_RUNTIME_ROOT}/{_HERMETIC_CA_BUNDLE_RELATIVE.as_posix()}" +_HERMETIC_RUNTIME_READY_ATTEMPTS = 6 +_HERMETIC_RUNTIME_READY_DELAY_SECONDS = 2 + + +def _require_full_sha(value: str, name: str) -> str: + normalized = value.strip().lower() + if not _FULL_SHA.fullmatch(normalized): + raise ValueError(f"{name} must be a full 40-character hexadecimal commit") + return normalized + + +def _require_sha256(value: str, name: str) -> str: + normalized = value.strip().lower() + if not _SHA256.fullmatch(normalized): + raise ValueError(f"{name} must be a 64-character hexadecimal SHA-256") + return normalized + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _load_hermetic_runtime( + path: Path, + *, + expected_digest: str, + hermes_commit: str, + relay_wheel_sha256: str, + relay_architecture: str, +) -> dict[str, Any]: + marker = path / "payload.json" + if not marker.is_file(): + raise FileNotFoundError(marker) + payload = json.loads(marker.read_text(encoding="utf-8")) + expected = { + "schema_version": _HERMETIC_RUNTIME_SCHEMA, + "status": "passed", + "content_sha256": expected_digest, + "hermes_commit": hermes_commit, + "relay_wheel_sha256": relay_wheel_sha256, + "relay_architecture": relay_architecture, + } + mismatches = { + key: {"expected": value, "actual": payload.get(key)} + for key, value in expected.items() + if payload.get(key) != value + } + if mismatches: + raise ValueError(f"hermetic runtime metadata mismatch: {mismatches}") + required = ( + path / "bin" / "hermes", + path / "bin" / "python", + path / "bin" / "uv", + path / "hermes-agent-src" / "venv", + path / _HERMETIC_CA_BUNDLE_RELATIVE, + ) + missing = [str(candidate) for candidate in required if not candidate.exists()] + if missing: + raise FileNotFoundError(f"hermetic runtime is incomplete: {missing}") + return payload + + +def _hermetic_runtime_readiness_command( + runtime_root: str = _HERMETIC_RUNTIME_ROOT, + *, + attempts: int = _HERMETIC_RUNTIME_READY_ATTEMPTS, + delay_seconds: int = _HERMETIC_RUNTIME_READY_DELAY_SECONDS, +) -> str: + """Return a bounded probe that executes both nested runtime entrypoints.""" + if attempts < 1: + raise ValueError("attempts must be positive") + if delay_seconds < 0: + raise ValueError("delay_seconds cannot be negative") + runtime = shlex.quote(runtime_root) + attempt_numbers = " ".join(str(attempt) for attempt in range(1, attempts + 1)) + return ( + "runtime_ready=1; " + f"for attempt in {attempt_numbers}; do " + f'if {runtime}/bin/python -c "import importlib.metadata as m; ' + "assert tuple(map(int, m.version('nemo-relay').split('.'))) >= (0, 7, 0)\" " + f"&& {runtime}/bin/hermes version; then " + "runtime_ready=0; break; " + "else runtime_ready=$?; fi; " + f'if [ "$attempt" -lt {attempts} ]; then sleep {delay_seconds}; fi; ' + "done; " + '[ "$runtime_ready" -eq 0 ] || exit "$runtime_ready"; ' + ) + + +def _verify_elf_architecture(path: Path, architecture: str) -> None: + with path.open("rb") as stream: + header = stream.read(20) + if header[:4] != b"\x7fELF" or len(header) < 20 or header[5] != 1: + raise ValueError("Switchyard native library must be a little-endian ELF artifact") + expected_machine = {"x86_64": 62, "aarch64": 183}[architecture] + machine = int.from_bytes(header[18:20], "little") + if machine != expected_machine: + raise ValueError(f"Switchyard native library does not target {architecture}: ELF e_machine={machine}") + + +def _require_public_https_git_url(value: str) -> str: + parsed = urlsplit(value) + if ( + parsed.scheme != "https" + or parsed.hostname != "github.com" + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + or not parsed.path.endswith(".git") + ): + raise ValueError("repository_url must be a credential-free https://github.com/...git URL") + return value + + +def _find_named_component(config: dict[str, Any], kind: str) -> dict[str, Any]: + matches = [ + component + for component in config.get("components", []) + if isinstance(component, dict) and component.get("kind") == kind and component.get("enabled", True) + ] + if len(matches) != 1: + raise ValueError(f"Relay config must contain exactly one enabled {kind!r} component") + return matches[0] + + +def _validate_relay_config(path: Path) -> None: + with path.open("rb") as stream: + config = tomllib.load(stream) + + if config.get("version") != 1: + raise ValueError("Relay config must use top-level version = 1") + if config.get("dynamic_plugins"): + raise ValueError("Hermes [[dynamic_plugins]] worker records are not allowed in this example") + + plugins = config.get("plugins") + dynamic = plugins.get("dynamic") if isinstance(plugins, dict) else None + if not isinstance(dynamic, list) or len(dynamic) != 1: + raise ValueError("Relay config must contain exactly one [[plugins.dynamic]] record") + plugin = dynamic[0] + manifest = plugin.get("manifest") if isinstance(plugin, dict) else None + if manifest != "/opt/relay-plugins/nvidia.switchyard/relay-plugin.toml": + raise ValueError("the dynamic plugin must reference the staged Switchyard manifest") + + plugin_config = plugin.get("config") + if not isinstance(plugin_config, dict) or plugin_config.get("version") != 2: + raise ValueError("the Switchyard plugin must use config version = 2") + algorithm = plugin_config.get("algorithm") + expected_algorithm = { + "kind": "llm_classifier", + "classifier_target": "judge", + "weak_target": "weak", + "strong_target": "strong", + "base_threshold": 0.5, + "recent_turn_window": 0, + "session_affinity": True, + "message_hash_fallback": True, + } + if algorithm != expected_algorithm: + raise ValueError("the Switchyard classifier contract does not match the Phase 1 design") + if plugin_config.get("default_targets") != {"openai_chat": "strong"}: + raise ValueError("the Switchyard OpenAI default target must be strong") + + targets = plugin_config.get("targets") + if not isinstance(targets, dict) or set(targets) != {"strong", "weak", "judge"}: + raise ValueError("Switchyard must define exactly the strong, weak, and judge targets") + provider_models: set[str] = set() + for name, target in targets.items(): + if not isinstance(target, dict): + raise ValueError(f"Switchyard target {name!r} must be a table") + if target.get("protocol") != "openai_chat" or target.get("endpoint") != "/v1/chat/completions": + raise ValueError(f"Switchyard target {name!r} must use the OpenAI chat protocol") + if target.get("drop_caller_extra_body") is not True: + raise ValueError(f"Switchyard target {name!r} must drop Hermes' caller-specific extra_body wrapper") + base_url = target.get("base_url") + parsed_base_url = urlsplit(base_url) if isinstance(base_url, str) else None + if ( + parsed_base_url is None + or parsed_base_url.scheme not in {"http", "https"} + or not parsed_base_url.hostname + or parsed_base_url.username is not None + or parsed_base_url.password is not None + ): + raise ValueError(f"Switchyard target {name!r} must use a credential-free HTTP(S) base URL") + model = target.get("model") + if not isinstance(model, str) or not model: + raise ValueError(f"Switchyard target {name!r} must define a model") + provider_models.add(model) + header_env = target.get("header_env") + authorization_env = header_env.get("authorization") if isinstance(header_env, dict) else None + if not isinstance(authorization_env, str) or not _ENV_NAME.fullmatch(authorization_env): + raise ValueError(f"Switchyard target {name!r} must source authorization from an environment variable") + if len(provider_models) != 3: + raise ValueError("Switchyard strong, weak, and judge targets must use distinct models") + + pricing = _find_named_component(config, "pricing") + pricing_config = pricing.get("config") + sources = pricing_config.get("sources") if isinstance(pricing_config, dict) else None + if not isinstance(sources, list) or len(sources) != 1 or sources[0].get("type") != "inline": + raise ValueError("pricing must use exactly one inline catalog") + catalog = sources[0].get("catalog") + entries = catalog.get("entries") if isinstance(catalog, dict) and catalog.get("version") == 1 else None + if not isinstance(entries, list) or {entry.get("model_id") for entry in entries} != provider_models: + raise ValueError("pricing entries must match the Switchyard provider models") + for entry in entries: + rates = entry.get("rates") if isinstance(entry, dict) else None + if not isinstance(rates, dict) or any( + not isinstance(rates.get(key), (int, float)) or rates[key] <= 0 + for key in ("input_per_million", "output_per_million", "cache_read_per_million") + ): + raise ValueError("pricing entries must contain positive input, output, and cache-read rates") + + observability = _find_named_component(config, "observability") + observability_config = observability.get("config") + if not isinstance(observability_config, dict) or observability_config.get("version") != 3: + raise ValueError("the observability component must use schema version = 3") + atif = observability_config.get("atif") + caller_model = atif.get("model_name") if isinstance(atif, dict) else None + if not isinstance(caller_model, str) or not caller_model or caller_model in provider_models: + raise ValueError("the fail-closed Hermes caller model must not be a Switchyard provider model") + opentelemetry = observability_config.get("opentelemetry") + endpoints = opentelemetry.get("endpoints") if isinstance(opentelemetry, dict) else None + if not isinstance(opentelemetry, dict) or opentelemetry.get("enabled") is not True: + raise ValueError("OpenTelemetry export must be enabled") + if not isinstance(endpoints, list) or len(endpoints) != 1: + raise ValueError("observability must define exactly one OpenInference endpoint") + endpoint = endpoints[0] + if endpoint.get("type") != "openinference" or endpoint.get("transport") != "http_binary": + raise ValueError("the only telemetry endpoint must be OpenInference over OTLP/HTTP protobuf") + resource_attributes = endpoint.get("resource_attributes") + if not isinstance(resource_attributes, dict) or not all( + isinstance(resource_attributes.get(key), str) and resource_attributes[key] + for key in ("openinference.project.name", "evaluation.cohort") + ): + raise ValueError("OpenInference must carry project and evaluation cohort resource attributes") + + def reject_literal_headers(value: Any, location: str = "config") -> None: + if isinstance(value, dict): + for key, nested in value.items(): + if key == "headers": + raise ValueError(f"literal headers are forbidden; use header_env ({location}.headers)") + reject_literal_headers(nested, f"{location}.{key}") + elif isinstance(value, list): + for index, nested in enumerate(value): + reject_literal_headers(nested, f"{location}[{index}]") + + reject_literal_headers(config) + + +class HarborHermesAgent(Hermes): + """Hermes #77915 bridge retaining Harbor's built-in Hermes behavior.""" + + def __init__( + self, + *args: Any, + repository_url: str = _DEFAULT_HERMES_REPOSITORY, + repository_ref: str = _DEFAULT_HERMES_REF, + commit: str = _DEFAULT_HERMES_COMMIT, + relay_config_path: str, + switchyard_bundle_dir: str, + relay_wheel_path: str, + relay_wheel_sha256: str, + relay_architecture: str = "x86_64", + switchyard_commit: str = _DEFAULT_SWITCHYARD_COMMIT, + artifact_root: str = "/logs/agent/direct-hermes", + inject_post_response_failure: bool = False, + hermetic_runtime_dir: str | None = None, + hermetic_runtime_sha256: str | None = None, + **kwargs: Any, + ) -> None: + self.repository_url = _require_public_https_git_url(repository_url) + self.repository_ref = repository_ref.strip() + if not self.repository_ref or self.repository_ref.startswith("-"): + raise ValueError("repository_ref must be a non-option branch or tag name") + self.commit = _require_full_sha(commit, "commit") + self.switchyard_commit = _require_full_sha(switchyard_commit, "switchyard_commit") + self.relay_wheel_sha256 = _require_sha256(relay_wheel_sha256, "relay_wheel_sha256") + if relay_architecture not in {"x86_64", "aarch64"}: + raise ValueError("relay_architecture must be x86_64 or aarch64") + self.relay_architecture = relay_architecture + + self.relay_config_path = Path(relay_config_path).expanduser().resolve() + self.switchyard_bundle_dir = Path(switchyard_bundle_dir).expanduser().resolve() + self.relay_wheel_path = Path(relay_wheel_path).expanduser().resolve() + self.artifact_root = artifact_root.rstrip("/") + self.inject_post_response_failure = inject_post_response_failure + self.hermetic_runtime_dir: Path | None = None + self.hermetic_runtime_sha256: str | None = None + self._load_provider_authorization = False + if not self.artifact_root.startswith("/logs/agent/"): + raise ValueError("artifact_root must be an absolute child of /logs/agent") + if not self.relay_config_path.is_file(): + raise FileNotFoundError(self.relay_config_path) + if not self.relay_wheel_path.is_file(): + raise FileNotFoundError(self.relay_wheel_path) + if _sha256(self.relay_wheel_path) != self.relay_wheel_sha256: + raise ValueError("Relay wheel digest does not match relay_wheel_sha256") + if "manylinux" not in self.relay_wheel_path.name or relay_architecture not in self.relay_wheel_path.name: + raise ValueError(f"Relay wheel must target Linux {relay_architecture}") + _validate_relay_config(self.relay_config_path) + + self.switchyard_manifest = self.switchyard_bundle_dir / "relay-plugin.toml" + if not self.switchyard_manifest.is_file(): + raise FileNotFoundError(self.switchyard_manifest) + libraries = sorted( + path + for path in self.switchyard_bundle_dir.iterdir() + if path.is_file() and path.suffix in {".so", ".dylib", ".dll"} + ) + if len(libraries) != 1: + raise ValueError("Switchyard bundle must contain exactly one native library") + self.switchyard_library = libraries[0] + _verify_elf_architecture(self.switchyard_library, relay_architecture) + + if (hermetic_runtime_dir is None) != (hermetic_runtime_sha256 is None): + raise ValueError("hermetic_runtime_dir and hermetic_runtime_sha256 must be supplied together") + if hermetic_runtime_dir is not None and hermetic_runtime_sha256 is not None: + runtime_dir = Path(hermetic_runtime_dir).expanduser().resolve() + runtime_digest = _require_sha256(hermetic_runtime_sha256, "hermetic_runtime_sha256") + _load_hermetic_runtime( + runtime_dir, + expected_digest=runtime_digest, + hermes_commit=self.commit, + relay_wheel_sha256=self.relay_wheel_sha256, + relay_architecture=self.relay_architecture, + ) + self.hermetic_runtime_dir = runtime_dir + self.hermetic_runtime_sha256 = runtime_digest + + self._example_root = Path(__file__).resolve().parents[1] + self._finalizer_path = self._example_root / "scripts" / "finalize_artifacts.py" + if not self._finalizer_path.is_file(): + raise FileNotFoundError(self._finalizer_path) + self._relay_version_path = self._example_root / "scripts" / "relay_version.py" + if not self._relay_version_path.is_file(): + raise FileNotFoundError(self._relay_version_path) + + extra_env = dict(kwargs.pop("extra_env", None) or {}) + extra_env["HERMES_NEMO_RELAY_PLUGINS_TOML"] = "/tmp/hermes/relay/plugins.toml" + super().__init__(*args, version=self.commit, extra_env=extra_env, **kwargs) + + @override + async def exec_as_agent( + self, + environment: BaseEnvironment, + command: str, + env: dict[str, str] | None = None, + cwd: str | None = None, + timeout_sec: int | None = None, + ) -> Any: + if self.hermetic_runtime_dir is not None: + ca_bundle = shlex.quote(_HERMETIC_CA_BUNDLE) + command = ( + f"test -r {ca_bundle}; " + f"export SSL_CERT_FILE={ca_bundle}; " + f"export REQUESTS_CA_BUNDLE={ca_bundle}; " + f"export CURL_CA_BUNDLE={ca_bundle}; " + f"{command}" + ) + if self._load_provider_authorization: + secret_file = shlex.quote(_PROVIDER_AUTHORIZATION_FILE) + command = ( + f"test -r {secret_file}; " + f'export SWITCHYARD_PROVIDER_AUTHORIZATION="$(cat -- {secret_file})"; ' + 'test -n "$SWITCHYARD_PROVIDER_AUTHORIZATION"; ' + f"{command}" + ) + return await super().exec_as_agent( + environment, + command, + env=env, + cwd=cwd, + timeout_sec=timeout_sec, + ) + + @override + async def install(self, environment: BaseEnvironment) -> None: + if self.hermetic_runtime_dir is not None: + runtime = shlex.quote(_HERMETIC_RUNTIME_ROOT) + await self.exec_as_agent( + environment, + command=( + "set -euo pipefail; " + f"test -r {runtime}/payload.json; " + f"test -x {runtime}/bin/hermes; " + f"test -x {runtime}/bin/python; " + f"test -x {runtime}/bin/uv; " + f"test -r {shlex.quote(_HERMETIC_CA_BUNDLE)}; " + f"{_hermetic_runtime_readiness_command()}" + "rm -rf /tmp/hermes-agent-src; " + f"ln -s {runtime}/hermes-agent-src /tmp/hermes-agent-src; " + 'mkdir -p /tmp/hermes/bin "$HOME/.local/bin"; ' + f'ln -sf {runtime}/bin/hermes "$HOME/.local/bin/hermes"; ' + f"ln -sf {runtime}/bin/uv /tmp/hermes/bin/uv; " + f"if test -x {runtime}/bin/rg; then " + f'ln -sf {runtime}/bin/rg "$HOME/.local/bin/rg"; fi; ' + 'export PATH="$HOME/.local/bin:$PATH"' + ), + timeout_sec=90, + ) + return + + await self.exec_as_root( + environment, + command=( + "set -euo pipefail; last_status=1; " + "for attempt in 1 2 3; do " + "if apt-get update && apt-get install -y --no-install-recommends " + "ca-certificates build-essential curl git ripgrep xz-utils; then exit 0; " + "else last_status=$?; fi; " + 'if [ "$attempt" -eq 3 ]; then break; fi; ' + "rm -rf /var/lib/apt/lists/partial; sleep $((attempt * 5)); " + 'done; exit "$last_status"' + ), + env={"DEBIAN_FRONTEND": "noninteractive"}, + ) + + repository = shlex.quote(self.repository_url) + repository_ref = shlex.quote(self.repository_ref) + commit = shlex.quote(self.commit) + install_dir = "/tmp/hermes-agent-src" + await self.exec_as_agent( + environment, + command=( + "set -euo pipefail; " + f"git clone --no-tags --branch {repository_ref} {repository} {install_dir}; " + f"git -C {install_dir} fetch --depth 1 origin {commit}; " + f"git -C {install_dir} checkout --detach {commit}; " + f'test "$(git -C {install_dir} rev-parse HEAD)" = {commit}; ' + "mkdir -p /tmp/hermes-install-path; " + "ln -sf /bin/true /tmp/hermes-install-path/ffmpeg; " + f"HERMES_HOME=/tmp/hermes HERMES_INSTALL_DIR={install_dir} " + "PATH=/tmp/hermes-install-path:$PATH " + f"bash {install_dir}/scripts/install.sh --skip-setup --skip-browser " + f"--no-skills --dir {install_dir} --branch {repository_ref} " + f"--commit {commit} --force-commit; " + f'test "$(git -C {install_dir} rev-parse HEAD)" = {commit}; ' + f"cd {install_dir}; " + f"UV_PROJECT_ENVIRONMENT={install_dir}/venv " + "/tmp/hermes/bin/uv sync --frozen --extra all; " + 'export PATH="$HOME/.local/bin:$PATH"; ' + "hermes version; " + f'{install_dir}/venv/bin/python -c "import importlib.metadata as m; ' + "assert tuple(map(int, m.version('nemo-relay').split('.'))) >= (0, 7, 0)\"" + ), + ) + + @override + async def setup(self, environment: BaseEnvironment) -> None: + await super().setup(environment) + await self.exec_as_root( + environment, + command=( + "mkdir -p /tmp/hermes/relay /opt/relay-wheels /opt/relay-plugins/nvidia.switchyard /installed-agent" + ), + ) + await environment.upload_file(self.relay_config_path, "/tmp/hermes/relay/plugins.toml") + relay_wheel = f"/opt/relay-wheels/{self.relay_wheel_path.name}" + await environment.upload_file(self.relay_wheel_path, relay_wheel) + if self.hermetic_runtime_dir is None: + relay_install = ( + "/tmp/hermes/bin/uv pip install " + "--python /tmp/hermes-agent-src/venv/bin/python " + f"--force-reinstall --no-deps {shlex.quote(relay_wheel)}; " + "/tmp/hermes-agent-src/venv/bin/python" + ) + else: + relay_install = f"{_HERMETIC_RUNTIME_ROOT}/bin/python" + await self.exec_as_agent( + environment, + command=( + "set -euo pipefail; " + f"test \"$(sha256sum {shlex.quote(relay_wheel)} | cut -d' ' -f1)\" = " + f"{shlex.quote(self.relay_wheel_sha256)}; " + f'{relay_install} -c "import importlib.metadata as m; ' + "assert tuple(map(int, m.version('nemo-relay').split('.'))) >= (0, 7, 0)\"" + ), + timeout_sec=120, + ) + await environment.upload_dir(self.switchyard_bundle_dir, "/opt/relay-plugins/nvidia.switchyard") + await environment.upload_file(self._finalizer_path, "/installed-agent/finalize_artifacts.py") + await environment.upload_file(self._relay_version_path, "/installed-agent/relay_version.py") + probe_python = ( + f"{_HERMETIC_RUNTIME_ROOT}/bin/python" + if self.hermetic_runtime_dir is not None + else "/tmp/hermes-agent-src/venv/bin/python" + ) + switchyard_library = f"/opt/relay-plugins/nvidia.switchyard/{self.switchyard_library.name}" + await self.exec_as_agent( + environment, + command=( + f"{probe_python} -c " + + shlex.quote( + "import ctypes, importlib.metadata as m; " + "assert tuple(map(int, m.version('nemo-relay').split('.'))) >= (0, 7, 0); " + f"library = ctypes.CDLL({switchyard_library!r}); " + "assert getattr(library, 'nemo_relay_register_plugin')" + ) + ), + timeout_sec=30, + ) + await self.exec_as_agent( + environment, + command=self._finalizer_command("initialize"), + env={"HERMES_HOME": "/tmp/hermes"}, + timeout_sec=30, + ) + + def _finalizer_command( + self, + mode: str, + *, + started_at: float | None = None, + error_type: str = "", + ) -> str: + arguments = [ + ( + f"{_HERMETIC_RUNTIME_ROOT}/bin/python" + if self.hermetic_runtime_dir is not None + else "/tmp/hermes-agent-src/venv/bin/python" + ), + "/installed-agent/finalize_artifacts.py", + mode, + "--artifact-root", + self.artifact_root, + "--hermes-repository", + self.repository_url, + "--hermes-commit", + self.commit, + "--switchyard-commit", + self.switchyard_commit, + "--relay-wheel-sha256", + self.relay_wheel_sha256, + "--relay-config", + "/tmp/hermes/relay/plugins.toml", + "--switchyard-manifest", + "/opt/relay-plugins/nvidia.switchyard/relay-plugin.toml", + "--switchyard-library", + f"/opt/relay-plugins/nvidia.switchyard/{self.switchyard_library.name}", + "--session-handle", + self.session_id or "", + ] + if started_at is not None: + arguments.extend(["--started-at", str(started_at)]) + if error_type: + arguments.extend(["--error-type", error_type]) + return " ".join(shlex.quote(value) for value in arguments) + + @override + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + started_at = time.time() + error: BaseException | None = None + try: + self._load_provider_authorization = True + try: + await super().run(instruction, environment, context) + finally: + self._load_provider_authorization = False + except BaseException as exc: + error = exc + raise + finally: + try: + await self.exec_as_agent( + environment, + command=self._finalizer_command( + "complete", + started_at=started_at, + error_type=( + type(error).__name__ + if error is not None + else ("InjectedPostResponseFailure" if self.inject_post_response_failure else "") + ), + ), + env={"HERMES_HOME": "/tmp/hermes"}, + timeout_sec=30, + ) + except Exception: + if error is None: + raise + self.logger.exception("Could not frame direct Hermes artifacts after agent failure") + + +__all__ = ["HarborHermesAgent"] diff --git a/examples/harbor-hermes-switchyard/config/otel-collector.yaml b/examples/harbor-hermes-switchyard/config/otel-collector.yaml new file mode 100644 index 000000000..68f78738d --- /dev/null +++ b/examples/harbor-hermes-switchyard/config/otel-collector.yaml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +exporters: + file/openinference: + path: /artifacts/trajectory.openinference.json + +service: + pipelines: + traces/openinference: + receivers: [otlp] + exporters: [file/openinference] diff --git a/examples/harbor-hermes-switchyard/config/plugins.toml.in b/examples/harbor-hermes-switchyard/config/plugins.toml.in new file mode 100644 index 000000000..e391f86bc --- /dev/null +++ b/examples/harbor-hermes-switchyard/config/plugins.toml.in @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = 1 + +[[components]] +kind = "pricing" +enabled = true + +[[components.config.sources]] +type = "inline" + +[components.config.sources.catalog] +version = 1 + +[[components.config.sources.catalog.entries]] +provider = "openai" +model_id = "aws/anthropic/bedrock-claude-opus-5" +currency = "USD" +unit = "per_token" +pricing_as_of = "2026-08-07" +pricing_source = "Anthropic public list pricing" + +[components.config.sources.catalog.entries.rates] +input_per_million = 5.0 +output_per_million = 25.0 +cache_read_per_million = 0.5 +cache_write_per_million = 6.25 + +[components.config.sources.catalog.entries.prompt_cache] +read_accounting = "included_in_prompt_tokens" + +[[components.config.sources.catalog.entries]] +provider = "openai" +model_id = "aws/anthropic/bedrock-claude-sonnet-4-6" +currency = "USD" +unit = "per_token" +pricing_as_of = "2026-08-07" +pricing_source = "Anthropic public list pricing" + +[components.config.sources.catalog.entries.rates] +input_per_million = 3.0 +output_per_million = 15.0 +cache_read_per_million = 0.3 +cache_write_per_million = 3.75 + +[components.config.sources.catalog.entries.prompt_cache] +read_accounting = "included_in_prompt_tokens" + +[[components.config.sources.catalog.entries]] +provider = "openai" +model_id = "aws/anthropic/bedrock-claude-sonnet-5" +currency = "USD" +unit = "per_token" +pricing_as_of = "2026-08-07" +pricing_source = "Anthropic public list pricing" + +[components.config.sources.catalog.entries.rates] +input_per_million = 3.0 +output_per_million = 15.0 +cache_read_per_million = 0.3 +cache_write_per_million = 3.75 + +[components.config.sources.catalog.entries.prompt_cache] +read_accounting = "included_in_prompt_tokens" + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 3 + +[components.config.atof] +enabled = true + +[[components.config.atof.sinks]] +type = "file" +mode = "append" +output_directory = "/logs/agent/direct-hermes/relay" +filename = "trajectory.atof.jsonl" + +[components.config.atif] +enabled = true +agent_name = "Hermes" +agent_version = "@HERMES_COMMIT@" +model_name = "ollama-route-stub" +output_directory = "/logs/agent/direct-hermes/relay/atif" +filename_template = "trajectory-{session_id}.atif.json" + +[components.config.opentelemetry] +enabled = true + +[[components.config.opentelemetry.endpoints]] +type = "openinference" +transport = "http_binary" +endpoint = "@OPENINFERENCE_ENDPOINT@" +service_name = "harbor-hermes-switchyard" +service_namespace = "nemo-relay-examples" +instrumentation_scope = "harbor-hermes-switchyard" +timeout_millis = 5000 + +[components.config.opentelemetry.endpoints.resource_attributes] +"openinference.project.name" = "@PHOENIX_PROJECT@" +"evaluation.cohort" = "@EVAL_COHORT@" + +[[plugins.dynamic]] +manifest = "/opt/relay-plugins/nvidia.switchyard/relay-plugin.toml" + +[plugins.dynamic.config] +version = 2 +priority = 0 +max_retries = 1 + +[plugins.dynamic.config.algorithm] +kind = "llm_classifier" +classifier_target = "judge" +weak_target = "weak" +strong_target = "strong" +base_threshold = 0.5 +recent_turn_window = 0 +session_affinity = true +message_hash_fallback = true + +[plugins.dynamic.config.default_targets] +openai_chat = "strong" + +[plugins.dynamic.config.targets.strong] +model = "aws/anthropic/bedrock-claude-opus-5" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://inference-api.nvidia.com/v1" +weight = 1 +drop_caller_extra_body = true + +[plugins.dynamic.config.targets.strong.header_env] +authorization = "SWITCHYARD_PROVIDER_AUTHORIZATION" + +[plugins.dynamic.config.targets.weak] +model = "aws/anthropic/bedrock-claude-sonnet-5" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://inference-api.nvidia.com/v1" +weight = 1 +drop_caller_extra_body = true + +[plugins.dynamic.config.targets.weak.header_env] +authorization = "SWITCHYARD_PROVIDER_AUTHORIZATION" + +[plugins.dynamic.config.targets.judge] +model = "aws/anthropic/bedrock-claude-sonnet-4-6" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://inference-api.nvidia.com/v1" +weight = 1 +drop_caller_extra_body = true + +[plugins.dynamic.config.targets.judge.header_env] +authorization = "SWITCHYARD_PROVIDER_AUTHORIZATION" diff --git a/examples/harbor-hermes-switchyard/requirements.txt b/examples/harbor-hermes-switchyard/requirements.txt new file mode 100644 index 000000000..00cc8a995 --- /dev/null +++ b/examples/harbor-hermes-switchyard/requirements.txt @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +harbor==0.18.0 +opentelemetry-proto==1.38.0 +protobuf==6.33.5 +typing-extensions==4.15.0 +tomli-w==1.2.0 diff --git a/examples/harbor-hermes-switchyard/run_phase2_cohort.sh b/examples/harbor-hermes-switchyard/run_phase2_cohort.sh new file mode 100755 index 000000000..708ae888f --- /dev/null +++ b/examples/harbor-hermes-switchyard/run_phase2_cohort.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +run_root="${1:-${TERMINAL_BENCH_RUN_ROOT:-}}" +if [[ -z "$run_root" || "$run_root" != /* ]]; then + echo "usage: $0 /absolute/phase2-run-root [coordinator options]" >&2 + exit 2 +fi +shift + +dataset="${TBENCH_DATASET:-terminal-bench@2.0}" +dataset_name="${dataset%@*}" +dataset_export_root="${TBENCH_DATASET_EXPORT_ROOT:-$(dirname "$run_root")/harbor-datasets}" +dataset_root="${TBENCH_DATASET_PATH:-$dataset_export_root/$dataset_name}" +harbor_bin="${HARBOR_BIN:-$example_root/.venv/bin/harbor}" +python_bin="${EVAL_PYTHON:-$example_root/.venv/bin/python}" +smoke_evidence="${TERMINAL_BENCH_SMOKE_EVIDENCE:-}" +offline_evidence="${TERMINAL_BENCH_OFFLINE_EVIDENCE:-}" +phoenix_url="${PHOENIX_BASE_URL:-}" +phoenix_project="${PHOENIX_PROJECT:-}" +eval_cohort="${EVAL_COHORT:-}" +switchyard_bundle="${SWITCHYARD_BUNDLE:-}" +relay_wheel="${RELAY_WHEEL:-}" +relay_architecture="${RELAY_ARCHITECTURE:-x86_64}" +plugin_config_template="${PLUGIN_CONFIG_TEMPLATE:-$example_root/config/plugins.toml.in}" +sample_count="${TBENCH_SAMPLE_COUNT:-89}" +# An explicitly blank value disables canary-first scheduling. An unset value +# keeps the conservative default. +canary_task="${TBENCH_CANARY_TASK-adaptive-rejection-sampler}" +concurrency="${TBENCH_CONCURRENCY:-4}" +setup_concurrency="${TBENCH_SETUP_CONCURRENCY:-2}" +setup_batch_size="${TBENCH_SETUP_BATCH_SIZE:-89}" +setup_max_infra_attempts="${TBENCH_SETUP_MAX_INFRA_ATTEMPTS:-4}" +parallel_memory_gb="${TBENCH_PARALLEL_MAX_MEMORY_GB:-2}" +docker_memory_reserve_gb="${TBENCH_DOCKER_MEMORY_RESERVE_GB:-4}" +minimum_free_gb="${TBENCH_MINIMUM_FREE_GB:-100}" +bootstrap_root="${TERMINAL_BENCH_BOOTSTRAP_ROOT:-${TERMINAL_BENCH_ADMISSION_ROOT:-$(dirname "$run_root")}/bootstrap}" + +for required in \ + "$harbor_bin" \ + "$python_bin" \ + "$smoke_evidence" \ + "$offline_evidence" \ + "$dataset_root" \ + "$plugin_config_template" \ + "$switchyard_bundle/relay-plugin.toml" \ + "$relay_wheel"; do + [[ -e "$required" ]] || { + echo "required Phase 2 input is missing: $required" >&2 + exit 2 + } +done +if [[ -z "${SWITCHYARD_PROVIDER_AUTHORIZATION:-}" ]]; then + echo "SWITCHYARD_PROVIDER_AUTHORIZATION must be set by the protected Phase 2 environment file" >&2 + exit 2 +fi +for label in phoenix_url phoenix_project eval_cohort; do + [[ -n "${!label}" ]] || { + echo "${label^^} must be set" >&2 + exit 2 + } +done + +if [[ "$dataset_root" != /* || ! -d "$dataset_root" ]]; then + echo "TBENCH_DATASET_PATH must select an existing absolute local dataset directory" >&2 + echo "Phase 2 never downloads or resolves a dataset through the Harbor registry" >&2 + exit 2 +fi + +exec "$python_bin" "$example_root/scripts/run_phase2_cohort.py" \ + --run-root "$run_root" \ + --dataset "$dataset" \ + --dataset-root "$dataset_root" \ + --sample-count "$sample_count" \ + --canary-task "$canary_task" \ + --concurrency "$concurrency" \ + --setup-concurrency "$setup_concurrency" \ + --setup-batch-size "$setup_batch_size" \ + --setup-max-infra-attempts "$setup_max_infra_attempts" \ + --parallel-max-memory-gb "$parallel_memory_gb" \ + --docker-memory-reserve-gb "$docker_memory_reserve_gb" \ + --minimum-free-gb "$minimum_free_gb" \ + --smoke-evidence "$smoke_evidence" \ + --offline-evidence "$offline_evidence" \ + --plugin-config-template "$plugin_config_template" \ + --task-runner "$example_root/run_terminal_bench.sh" \ + --harbor-bin "$harbor_bin" \ + --python-bin "$python_bin" \ + --phoenix-url "$phoenix_url" \ + --phoenix-project "$phoenix_project" \ + --eval-cohort "$eval_cohort" \ + --switchyard-bundle "$switchyard_bundle" \ + --relay-wheel "$relay_wheel" \ + --relay-architecture "$relay_architecture" \ + --bootstrap-root "$bootstrap_root" \ + "$@" diff --git a/examples/harbor-hermes-switchyard/run_terminal_bench.sh b/examples/harbor-hermes-switchyard/run_terminal_bench.sh new file mode 100755 index 000000000..6cf56c1f5 --- /dev/null +++ b/examples/harbor-hermes-switchyard/run_terminal_bench.sh @@ -0,0 +1,382 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +run_root="${1:-}" +task_name="${TASK_NAME:-adaptive-rejection-sampler}" +upstream_auth_env="SWITCHYARD_PROVIDER_AUTHORIZATION" +fail_closed_openai_base_url="http://127.0.0.1:9/v1" +phoenix_base="${PHOENIX_BASE_URL:-}" +phoenix_project="${PHOENIX_PROJECT:-harbor-hermes-switchyard-phase1}" +eval_cohort="${EVAL_COHORT:-harbor-hermes-switchyard-phase1}" +default_harbor_bin="$example_root/.venv/bin/harbor" +default_python_bin="$example_root/.venv/bin/python" +harbor_bin="${HARBOR_BIN:-$default_harbor_bin}" +python_bin="${EVAL_PYTHON:-$default_python_bin}" +expected_harbor_version="0.18.0" +eval_phase="${EVAL_PHASE:-phase1}" +tbench_dataset_path="${TBENCH_DATASET_PATH:-}" +switchyard_bundle="${SWITCHYARD_BUNDLE:-}" +relay_wheel="${RELAY_WHEEL:-}" +relay_architecture="${RELAY_ARCHITECTURE:-x86_64}" +plugin_config_template="${PLUGIN_CONFIG_TEMPLATE:-$example_root/config/plugins.toml.in}" +agent_timeout_multiplier="${AGENT_TIMEOUT_MULTIPLIER:-3}" +agent_setup_timeout_multiplier="${AGENT_SETUP_TIMEOUT_MULTIPLIER:-6}" +environment_build_timeout_multiplier="${ENVIRONMENT_BUILD_TIMEOUT_MULTIPLIER:-6}" +collector_image="${OTEL_COLLECTOR_IMAGE:-otel/opentelemetry-collector-contrib:0.135.0}" +inject_post_response_failure="${INJECT_POST_RESPONSE_FAILURE:-false}" +hermetic_runtime_dir="${HERMETIC_RUNTIME_DIR:-}" +hermetic_runtime_sha256="${HERMETIC_RUNTIME_SHA256:-}" +harbor_force_build="${HARBOR_FORCE_BUILD:-true}" + +if [[ -z "$run_root" || "$run_root" != /* ]]; then + echo "usage: $0 /absolute/new-run-root" >&2 + exit 2 +fi +if [[ -e "$run_root" ]]; then + echo "run root already exists: $run_root" >&2 + exit 2 +fi +for required in "$plugin_config_template" "$phoenix_base"; do + [[ -n "$required" ]] || { + echo "PLUGIN_CONFIG_TEMPLATE and PHOENIX_BASE_URL are required" >&2 + exit 2 + } +done +if [[ ! -f "$plugin_config_template" ]]; then + echo "plugin configuration template is missing: $plugin_config_template" >&2 + exit 2 +fi +for dependency in curl docker "$harbor_bin" "$python_bin"; do + command -v "$dependency" >/dev/null || { + echo "missing required command: $dependency" >&2 + exit 1 + } +done +observed_harbor_version="$($python_bin -c 'import importlib.metadata; print(importlib.metadata.version("harbor"))')" +if [[ "$observed_harbor_version" != "$expected_harbor_version" ]]; then + echo "Harbor $expected_harbor_version is required; $python_bin provides $observed_harbor_version" >&2 + exit 2 +fi +observed_harbor_cli_version="$($harbor_bin --version)" +if [[ "$observed_harbor_cli_version" != "$expected_harbor_version" ]]; then + echo "Harbor CLI $expected_harbor_version is required; $harbor_bin reports $observed_harbor_cli_version" >&2 + exit 2 +fi +if [[ ! "$eval_phase" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + echo "EVAL_PHASE must contain only lowercase letters, digits, and hyphens" >&2 + exit 2 +fi +dataset_args=(--dataset terminal-bench@2.0) +if [[ -n "$tbench_dataset_path" ]]; then + if [[ "$tbench_dataset_path" != /* || ! -d "$tbench_dataset_path" ]]; then + echo "TBENCH_DATASET_PATH must be an absolute local dataset directory" >&2 + exit 2 + fi + dataset_args=(--path "$tbench_dataset_path") +elif [[ "$eval_phase" == "phase2" ]]; then + echo "Phase 2 requires TBENCH_DATASET_PATH and never resolves the remote registry" >&2 + exit 2 +fi +if [[ -z "${!upstream_auth_env:-}" ]]; then + echo "required provider authorization environment variable is unset: $upstream_auth_env" >&2 + exit 2 +fi +if [[ "$relay_architecture" != "x86_64" && "$relay_architecture" != "aarch64" ]]; then + echo "RELAY_ARCHITECTURE must be x86_64 or aarch64" >&2 + exit 2 +fi +if [[ ( -n "$hermetic_runtime_dir" && -z "$hermetic_runtime_sha256" ) || \ + ( -z "$hermetic_runtime_dir" && -n "$hermetic_runtime_sha256" ) ]]; then + echo "HERMETIC_RUNTIME_DIR and HERMETIC_RUNTIME_SHA256 must be supplied together" >&2 + exit 2 +fi +if [[ -n "$hermetic_runtime_dir" ]]; then + if [[ "$hermetic_runtime_dir" != /* || ! -f "$hermetic_runtime_dir/payload.json" ]]; then + echo "HERMETIC_RUNTIME_DIR must be an absolute prepared runtime directory" >&2 + exit 2 + fi + if [[ ! "$hermetic_runtime_sha256" =~ ^[0-9a-f]{64}$ ]]; then + echo "HERMETIC_RUNTIME_SHA256 must be a lowercase SHA-256 digest" >&2 + exit 2 + fi +fi +if [[ "$harbor_force_build" != "true" && "$harbor_force_build" != "false" ]]; then + echo "HARBOR_FORCE_BUILD must be true or false" >&2 + exit 2 +fi + +docker info >/dev/null +curl --fail --silent --show-error \ + --connect-timeout 5 --max-time 10 \ + --retry 2 --retry-all-errors --retry-delay 2 \ + "$phoenix_base" >/dev/null + +temporary_build="" +temporary_secret_dir="" +collector_name="" +collector_running=0 +cleanup() { + local status=$? + if [[ "$collector_running" == 1 ]]; then + docker stop --time 10 "$collector_name" >/dev/null 2>&1 || true + fi + if [[ -n "$temporary_build" && -d "$temporary_build" ]]; then + rm -rf "$temporary_build" + fi + if [[ -n "$temporary_secret_dir" && -d "$temporary_secret_dir" ]]; then + rm -rf "$temporary_secret_dir" + fi + return "$status" +} +trap cleanup EXIT + +host_temporary_root="$(cd "${TMPDIR:-/tmp}" && pwd -P)" +case "$host_temporary_root/" in + "$run_root/"*) + echo "Host temporary directory must be outside the task run root" >&2 + exit 2 + ;; +esac +temporary_secret_dir="$(mktemp -d "$host_temporary_root/harbor-phase2-secret.XXXXXX")" +chmod 0700 "$temporary_secret_dir" +provider_authorization_file="$temporary_secret_dir/switchyard-provider-authorization" +(umask 077; printf '%s' "${!upstream_auth_env}" >"$provider_authorization_file") +chmod 0600 "$provider_authorization_file" +provider_authorization_target="/run/secrets/switchyard-provider-authorization" +mounts_json="$($python_bin -c ' +import json, sys +mounts = [{ + "type": "bind", + "source": sys.argv[1], + "target": sys.argv[2], + "read_only": True, + "bind": {"create_host_path": False}, +}] +if sys.argv[3]: + mounts.append({ + "type": "bind", + "source": sys.argv[3], + "target": "/opt/hermes-runtime", + "read_only": True, + "bind": {"create_host_path": False}, + }) +print(json.dumps(mounts, separators=(",", ":"))) +' "$provider_authorization_file" "$provider_authorization_target" "$hermetic_runtime_dir")" + +if [[ -z "$switchyard_bundle" ]]; then + temporary_build="$(mktemp -d "$(dirname "$run_root")/.switchyard-build.XXXXXX")" + switchyard_bundle="$temporary_build/bundle" + SWITCHYARD_TARGET_ARCHITECTURE="$relay_architecture" \ + "$example_root/scripts/build_switchyard_plugin.sh" "$switchyard_bundle" +fi + +free_port="$($python_bin - <<'PY' +import socket +with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + print(sock.getsockname()[1]) +PY +)" +openinference_endpoint="http://host.docker.internal:$free_port/v1/traces" + +prepare_args=( + "$example_root/scripts/prepare_runtime.py" + --run-root "$run_root" + --switchyard-bundle "$switchyard_bundle" + --relay-architecture "$relay_architecture" + --plugin-config-template "$plugin_config_template" + --openinference-endpoint "$openinference_endpoint" + --phoenix-project "$phoenix_project" + --eval-cohort "$eval_cohort" +) +if [[ -n "$relay_wheel" ]]; then + prepare_args+=(--relay-wheel "$relay_wheel") +fi +"$python_bin" "${prepare_args[@]}" >"$run_root.prepare.log" + +hermes_caller_model="$($python_bin -c 'import json,sys; print(json.load(open(sys.argv[1]))["routing"]["hermes_caller_model"])' "$run_root/runtime/provenance.json")" + +relay_wheel_sha256="$($python_bin -c 'import json,sys; print(json.load(open(sys.argv[1]))["nemo_relay"]["wheel_sha256"])' "$run_root/runtime/provenance.json")" +relay_wheel_path="$($python_bin -c 'import json,pathlib,sys; p=json.load(open(sys.argv[1])); print(pathlib.Path(sys.argv[1]).parent / "wheels" / p["nemo_relay"]["wheel"])' "$run_root/runtime/provenance.json")" + +"$python_bin" "$example_root/scripts/verify_harbor_hermes_compat.py" \ + --bridge "$example_root/agents/harbor_hermes_agent.py" \ + --relay-config "$run_root/runtime/plugins.toml" \ + --output "$run_root/artifacts/harbor-hermes-compatibility.json" \ + >"$run_root/compatibility.log" + +mkdir -m 0700 "$run_root/telemetry" +collector_name="harbor-hermes-switchyard-$($python_bin -c 'import uuid; print(uuid.uuid4().hex[:12])')" +docker run --detach --rm \ + --name "$collector_name" \ + --publish "127.0.0.1:$free_port:4318" \ + --volume "$example_root/config/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro" \ + --volume "$run_root/telemetry:/artifacts" \ + "$collector_image" \ + --config=/etc/otelcol-contrib/config.yaml >"$run_root/collector.container-id" +collector_running=1 + +export PYTHONPATH="$example_root/agents${PYTHONPATH:+:$PYTHONPATH}" +export OPENAI_API_KEY="relay-managed-placeholder" +job_name="${eval_phase}-${task_name}-$(date -u +%Y%m%dT%H%M%SZ)" +agent_hosts=(--allow-agent-host host.docker.internal) +while IFS= read -r upstream_host; do + if [[ -n "$upstream_host" && "$upstream_host" != "host.docker.internal" ]]; then + agent_hosts+=(--allow-agent-host "$upstream_host") + fi +done < <("$python_bin" -c ' +import json, sys +from urllib.parse import urlsplit +values = json.load(open(sys.argv[1]))["routing"] +print("\n".join(sorted({urlsplit(value).hostname for key, value in values.items() if key.endswith("_base_url")}))) +' "$run_root/runtime/provenance.json") +agent_kwargs=() +if [[ "$inject_post_response_failure" == "true" ]]; then + agent_kwargs+=(--ak inject_post_response_failure=true) +elif [[ "$inject_post_response_failure" != "false" ]]; then + echo "INJECT_POST_RESPONSE_FAILURE must be true or false" >&2 + exit 2 +fi +if [[ -n "$hermetic_runtime_dir" ]]; then + agent_kwargs+=( + --ak "hermetic_runtime_dir=$hermetic_runtime_dir" + --ak "hermetic_runtime_sha256=$hermetic_runtime_sha256" + ) +fi +harbor_build_args=() +if [[ "$harbor_force_build" == "true" ]]; then + harbor_build_args+=(--force-build) +fi +( + "$harbor_bin" run \ + "${dataset_args[@]}" \ + --include-task-name "$task_name" \ + --n-tasks 1 \ + --agent harbor_hermes_agent:HarborHermesAgent \ + --model "openai/$hermes_caller_model" \ + --ak "repository_url=https://github.com/bbednarski9/hermes-agent.git" \ + --ak "repository_ref=feat/relay-native-plugin-init" \ + --ak "commit=a3d472f0e6bdc376df87b1436a461c4796db6747" \ + --ak "relay_config_path=$run_root/runtime/plugins.toml" \ + --ak "switchyard_bundle_dir=$run_root/runtime/switchyard-plugin" \ + --ak "relay_wheel_path=$relay_wheel_path" \ + --ak "relay_wheel_sha256=$relay_wheel_sha256" \ + --ak "relay_architecture=$relay_architecture" \ + "${agent_kwargs[@]}" \ + --ae 'OPENAI_API_KEY=${OPENAI_API_KEY}' \ + --ae "OPENAI_BASE_URL=$fail_closed_openai_base_url" \ + --ae 'OPENROUTER_API_KEY=relay-intercepted' \ + --ae "OPENROUTER_BASE_URL=$fail_closed_openai_base_url" \ + --mounts "$mounts_json" \ + "${agent_hosts[@]}" \ + --artifact /logs/agent/direct-hermes \ + --agent-include-logs hermes-session.jsonl \ + --agent-include-logs hermes.txt \ + --job-name "$job_name" \ + --jobs-dir "$run_root/jobs" \ + --n-concurrent 1 \ + --n-attempts 1 \ + --agent-timeout-multiplier "$agent_timeout_multiplier" \ + --agent-setup-timeout-multiplier "$agent_setup_timeout_multiplier" \ + --environment-build-timeout-multiplier "$environment_build_timeout_multiplier" \ + "${harbor_build_args[@]}" \ + --yes +) >"$run_root/harbor.log" 2>&1 + +docker stop --time 10 "$collector_name" >/dev/null +collector_running=0 + +direct_result="$($python_bin - "$run_root/jobs/$job_name" <<'PY' +import pathlib +import sys + +matches = sorted( + pathlib.Path(sys.argv[1]).glob( + "*/artifacts/logs/agent/direct-hermes/direct-hermes-result.json" + ) +) +if len(matches) != 1: + raise SystemExit(f"expected one direct Hermes result, found {len(matches)}") +print(matches[0]) +PY +)" +if [[ -z "$direct_result" ]]; then + echo "direct Hermes result discovery returned an empty path" >&2 + exit 1 +fi +artifact_root="$(dirname "$direct_result")" +openinference="$run_root/telemetry/trajectory.openinference.json" + +validation_args=( + "$example_root/scripts/validate_run.py" + --artifacts "$artifact_root" + --provenance "$run_root/runtime/provenance.json" + --openinference "$openinference" + --harbor-job-dir "$run_root/jobs/$job_name" + --scan-root "$run_root/jobs/$job_name" + --secret-env "$upstream_auth_env" + --output "$artifact_root/validation.json" +) +if [[ "$inject_post_response_failure" == "true" ]]; then + validation_args+=(--expect-late-failure) +fi +"$python_bin" "${validation_args[@]}" >"$run_root/validation.log" + +if ! "$python_bin" "$example_root/scripts/upload_openinference.py" \ + --openinference "$openinference" \ + --phoenix-url "$phoenix_base" \ + --project "$phoenix_project" \ + --output "$artifact_root/phoenix-upload.json" \ + >"$run_root/phoenix-upload.log"; then + "$python_bin" - "$artifact_root/phoenix-upload.json" <<'PY' +import json +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +path.write_text(json.dumps({"status": "failed", "error": "Phoenix upload command failed"}, indent=2) + "\n") +PY +fi + +"$python_bin" - "$artifact_root" "$run_root" "$job_name" "$task_name" <<'PY' +import json +import pathlib +import sys + +artifacts = pathlib.Path(sys.argv[1]) +run_root = pathlib.Path(sys.argv[2]) +summary = { + "schema_version": "harbor-hermes-switchyard.task-summary.v1", + "job_name": sys.argv[3], + "task_name": sys.argv[4], + "artifacts": str(artifacts), + "validation": json.loads((artifacts / "validation.json").read_text()), + "phoenix_upload": json.loads((artifacts / "phoenix-upload.json").read_text()), +} +integration = summary["validation"].setdefault("integration", {"status": "passed", "errors": [], "warnings": []}) +integration_errors = list(integration.get("errors", [])) +integration["phoenix_upload"] = summary["phoenix_upload"] +if summary["phoenix_upload"].get("status") != "passed": + integration_errors.append("Phoenix upload did not pass") +integration["errors"] = sorted(set(integration_errors)) +integration["status"] = "passed" if not integration["errors"] else "failed" +(artifacts / "validation.json").write_text(json.dumps(summary["validation"], indent=2, sort_keys=True) + "\n") +summary["benchmark_completion"] = summary["validation"].get("benchmark", {}) +summary["integration_validation"] = integration +benchmark_complete = summary["benchmark_completion"].get( + "status", summary["validation"].get("status") +) == "passed" +summary["status"] = "passed" if benchmark_complete else "failed" +if summary["status"] != "passed": + raise SystemExit("benchmark completion did not pass") +(run_root / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") +print(json.dumps(summary, indent=2)) +PY + +echo "Phase 1 task passed: $task_name" +echo "Run root: $run_root" +echo "Artifacts: $artifact_root" diff --git a/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py b/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py new file mode 100755 index 000000000..edf06ae3d --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build one reusable, provider-free Hermes runtime for Phase 2 setup. + +The coordinator invokes this content-addressed materialization automatically +when it is absent. Network access is confined to this one step; task containers +consume the resulting directory through a read-only bind mount and perform no +apt, Git, or Python package installation. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +_SCRIPT_ROOT = Path(__file__).resolve().parent +if str(_SCRIPT_ROOT) not in sys.path: + sys.path.insert(0, str(_SCRIPT_ROOT)) +from relay_version import wheel_version + +SCHEMA_VERSION = "harbor-hermes-switchyard.hermetic-runtime.v1" +DEFAULT_HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" +DEFAULT_HERMES_REF = "feat/relay-native-plugin-init" +DEFAULT_HERMES_COMMIT = "a3d472f0e6bdc376df87b1436a461c4796db6747" +UV_VERSION = "0.11.16" +PYTHON_VERSION = "3.11.13" +BUILDER_IMAGE = "python:3.11-bullseye" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def sha256_tree(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(candidate for candidate in root.rglob("*") if candidate.is_file()): + if path.name == "payload.json": + continue + relative = path.relative_to(root).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(4, "big")) + digest.update(relative) + digest.update(path.stat().st_mode.to_bytes(4, "big")) + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def run(command: list[str], *, attempts: int = 4, **kwargs: object) -> None: + for attempt in range(1, attempts + 1): + try: + subprocess.run(command, check=True, **kwargs) + return + except subprocess.CalledProcessError: + if attempt == attempts: + raise + time.sleep(5 * (2 ** (attempt - 1))) + + +def materialize_source( + destination: Path, + *, + repository: str, + repository_ref: str, + commit: str, +) -> None: + clone = destination.parent / "clone" + for attempt in range(1, 5): + shutil.rmtree(clone, ignore_errors=True) + try: + run( + [ + "git", + "clone", + "--no-tags", + "--filter=blob:none", + "--branch", + repository_ref, + repository, + str(clone), + ], + attempts=1, + ) + break + except subprocess.CalledProcessError: + if attempt == 4: + raise + time.sleep(5 * (2 ** (attempt - 1))) + run(["git", "-C", str(clone), "fetch", "--depth", "1", "origin", commit]) + run(["git", "-C", str(clone), "checkout", "--detach", commit]) + actual = subprocess.check_output( + ["git", "-C", str(clone), "rev-parse", "HEAD"], text=True + ).strip() + if actual != commit: + raise RuntimeError(f"Hermes checkout mismatch: expected {commit}, got {actual}") + shutil.copytree(clone, destination, ignore=shutil.ignore_patterns(".git")) + + +def build_payload( + output: Path, + *, + source: Path, + relay_wheel: Path, + platform: str, +) -> None: + script = r''' +set -euo pipefail +python -m pip install --no-cache-dir "uv==${UV_VERSION}" + +mkdir -p /opt/hermes-runtime/bin /opt/hermes-runtime/lib +cp -a /source /opt/hermes-runtime/hermes-agent-src +cp /usr/local/bin/uv /opt/hermes-runtime/bin/uv + +/usr/local/bin/uv python install "${PYTHON_VERSION}" \ + --install-dir /opt/hermes-runtime/python --no-bin --compile-bytecode +python_bin="$(find /opt/hermes-runtime/python -type f -path '*/bin/python3.11' -print -quit)" +test -n "$python_bin" + +UV_PROJECT_ENVIRONMENT=/opt/hermes-runtime/hermes-agent-src/venv \ +UV_PYTHON_DOWNLOADS=never \ + /usr/local/bin/uv sync --frozen --extra all \ + --project /opt/hermes-runtime/hermes-agent-src --python "$python_bin" +/usr/local/bin/uv pip install \ + --python /opt/hermes-runtime/hermes-agent-src/venv/bin/python \ + --force-reinstall --no-deps "/input/${RELAY_WHEEL_NAME}" + +cat > /opt/hermes-runtime/bin/python <<'EOF' +#!/bin/sh +set -eu +export LD_LIBRARY_PATH="/opt/hermes-runtime/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +exec /opt/hermes-runtime/hermes-agent-src/venv/bin/python "$@" +EOF +cat > /opt/hermes-runtime/bin/hermes <<'EOF' +#!/bin/sh +set -eu +export LD_LIBRARY_PATH="/opt/hermes-runtime/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +exec /opt/hermes-runtime/hermes-agent-src/venv/bin/hermes "$@" +EOF +chmod 0755 /opt/hermes-runtime/bin/python /opt/hermes-runtime/bin/hermes \ + /opt/hermes-runtime/bin/uv + +/opt/hermes-runtime/bin/hermes version +/opt/hermes-runtime/bin/python -c \ + 'import importlib.metadata as m; assert tuple(map(int, m.version("nemo-relay").split("."))) >= (0, 7, 0)' +''' + env = os.environ.copy() + env.update({"UV_VERSION": UV_VERSION, "PYTHON_VERSION": PYTHON_VERSION}) + run( + [ + "docker", + "run", + "--rm", + "--platform", + platform, + "--env", + f"UV_VERSION={UV_VERSION}", + "--env", + f"PYTHON_VERSION={PYTHON_VERSION}", + "--env", + f"RELAY_WHEEL_NAME={relay_wheel.name}", + "--volume", + f"{source}:/source:ro", + "--volume", + f"{relay_wheel}:/input/{relay_wheel.name}:ro", + "--volume", + f"{output}:/opt/hermes-runtime", + BUILDER_IMAGE, + "bash", + "-lc", + script, + ], + attempts=1, + env=env, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--relay-wheel", type=Path, required=True) + parser.add_argument("--relay-architecture", choices=("x86_64", "aarch64"), required=True) + parser.add_argument("--hermes-repository", default=DEFAULT_HERMES_REPOSITORY) + parser.add_argument("--hermes-ref", default=DEFAULT_HERMES_REF) + parser.add_argument("--hermes-commit", default=DEFAULT_HERMES_COMMIT) + args = parser.parse_args() + + output = args.output.expanduser().resolve() + relay_wheel = args.relay_wheel.expanduser().resolve(strict=True) + if output.exists(): + raise FileExistsError(f"output already exists: {output}") + expected_arch = args.relay_architecture + if "manylinux" not in relay_wheel.name or expected_arch not in relay_wheel.name: + raise ValueError(f"Relay wheel does not target Linux {expected_arch}: {relay_wheel.name}") + platform = {"x86_64": "linux/amd64", "aarch64": "linux/arm64"}[expected_arch] + + output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary_output = output.with_name(f".{output.name}.building") + if temporary_output.exists(): + raise FileExistsError(f"stale temporary output exists: {temporary_output}") + temporary_output.mkdir(mode=0o700) + try: + with tempfile.TemporaryDirectory( + prefix="hermes-source-", dir=output.parent + ) as temporary: + source = Path(temporary) / "source" + materialize_source( + source, + repository=args.hermes_repository, + repository_ref=args.hermes_ref, + commit=args.hermes_commit, + ) + for attempt in range(1, 5): + try: + build_payload( + temporary_output, + source=source, + relay_wheel=relay_wheel, + platform=platform, + ) + break + except subprocess.CalledProcessError: + if attempt == 4: + raise + shutil.rmtree(temporary_output) + temporary_output.mkdir(mode=0o700) + time.sleep(5 * (2 ** (attempt - 1))) + content_sha256 = sha256_tree(temporary_output) + marker = { + "schema_version": SCHEMA_VERSION, + "status": "passed", + "content_sha256": content_sha256, + "hermes_repository": args.hermes_repository, + "hermes_ref": args.hermes_ref, + "hermes_commit": args.hermes_commit, + "relay_version": wheel_version(relay_wheel), + "relay_wheel_sha256": sha256_file(relay_wheel), + "relay_architecture": expected_arch, + "builder_image": BUILDER_IMAGE, + "python_version": PYTHON_VERSION, + "uv_version": UV_VERSION, + } + (temporary_output / "payload.json").write_text( + json.dumps(marker, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + os.chmod(temporary_output / "payload.json", 0o444) + temporary_output.rename(output) + print(json.dumps({"output": str(output), **marker}, indent=2, sort_keys=True)) + except BaseException: + shutil.rmtree(temporary_output, ignore_errors=True) + raise + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh b/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh new file mode 100755 index 000000000..05d40ad9c --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +switchyard_repository="${SWITCHYARD_REPOSITORY:-https://github.com/bbednarski9/Switchyard.git}" +switchyard_commit="${SWITCHYARD_COMMIT:-8daac03edf8544144833af1fd009b3da737715bc}" +target_architecture="${SWITCHYARD_TARGET_ARCHITECTURE:-x86_64}" +output_dir="${1:-}" + +if [[ -z "$output_dir" ]]; then + echo "usage: $0 /absolute/output-directory" >&2 + exit 2 +fi +if [[ "$output_dir" != /* ]]; then + echo "output directory must be absolute" >&2 + exit 2 +fi +if [[ -e "$output_dir" ]]; then + echo "refusing to overwrite existing output directory: $output_dir" >&2 + exit 2 +fi +if [[ ! "$switchyard_commit" =~ ^[0-9a-f]{40}$ ]]; then + echo "SWITCHYARD_COMMIT must be a full commit SHA" >&2 + exit 2 +fi + +for dependency in docker git python3; do + command -v "$dependency" >/dev/null || { + echo "missing required command: $dependency" >&2 + exit 1 + } +done +docker info >/dev/null + +docker_architecture="$(docker info --format '{{.Architecture}}')" +if [[ "$target_architecture" != "x86_64" && "$target_architecture" != "aarch64" ]]; then + echo "SWITCHYARD_TARGET_ARCHITECTURE must be x86_64 or aarch64" >&2 + exit 2 +fi +if [[ "$docker_architecture" == "aarch64" || "$docker_architecture" == "arm64" ]]; then + builder_image="${SWITCHYARD_BUILDER_IMAGE:-rust:1.96.1-bullseye@sha256:69e444ec65a82386d041a4a3d15e47a797967b90ae24aa342bd8a3600dd9e244}" + builder_platform="linux/arm64" + if [[ "$target_architecture" == "x86_64" ]]; then + cargo_target="x86_64-unknown-linux-gnu" + library_path="/tmp/target/x86_64-unknown-linux-gnu/release/libswitchyard_nemo_relay_plugin.so" + else + cargo_target="" + library_path="/tmp/target/release/libswitchyard_nemo_relay_plugin.so" + fi +else + if [[ "$target_architecture" != "x86_64" ]]; then + echo "aarch64 cross-builds from an x86_64 Docker host are not supported" >&2 + exit 2 + fi + builder_image="${SWITCHYARD_BUILDER_IMAGE:-rust:1.96.1-bullseye@sha256:65136b30fc6b10112cbae63a868da085a878679a80d562272e485ecaaad3276a}" + builder_platform="linux/amd64" + cargo_target="" + library_path="/tmp/target/release/libswitchyard_nemo_relay_plugin.so" +fi + +# Stage beside the requested output so source and result use the same +# Docker-shared filesystem. Colima installations often do not share $TMPDIR or +# host /private/tmp even though those paths also exist inside the VM. +output_parent="$(dirname "$output_dir")" +if [[ ! -d "$output_parent" ]]; then + echo "output parent must already exist: $output_parent" >&2 + exit 2 +fi +build_root="$(mktemp -d "$output_parent/.switchyard-relay-plugin.XXXXXX")" +source_dir="$build_root/source" +staging_dir="${output_dir}.partial.$$" +if [[ -e "$staging_dir" ]]; then + echo "refusing to overwrite existing staging directory: $staging_dir" >&2 + exit 2 +fi +mkdir -m 0700 "$staging_dir" +cleanup() { + rm -rf "$build_root" "$staging_dir" +} +trap cleanup EXIT + +git clone --filter=blob:none --no-checkout "$switchyard_repository" "$source_dir" +git -C "$source_dir" fetch --depth 1 origin "$switchyard_commit" +git -C "$source_dir" checkout --detach "$switchyard_commit" +actual_commit="$(git -C "$source_dir" rev-parse HEAD)" +if [[ "$actual_commit" != "$switchyard_commit" ]]; then + echo "Switchyard checkout mismatch: expected $switchyard_commit, got $actual_commit" >&2 + exit 1 +fi + +docker run --rm \ + --platform "$builder_platform" \ + --env PHASE1_CARGO_TARGET="$cargo_target" \ + --env PHASE1_LIBRARY_PATH="$library_path" \ + --volume "$source_dir:/src:ro" \ + --volume "$staging_dir:/out" \ + "$builder_image" \ + bash -lc ' + set -euo pipefail + test -f /src/Cargo.toml + export DEBIAN_FRONTEND=noninteractive + export PATH="/usr/local/cargo/bin:$PATH" + apt-get update + apt-get install -y --no-install-recommends ca-certificates clang cmake pkg-config python3 + if [[ -n "$PHASE1_CARGO_TARGET" ]]; then + apt-get install -y --no-install-recommends crossbuild-essential-amd64 + rustup target add "$PHASE1_CARGO_TARGET" + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc + export CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc + export CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++ + export AR_x86_64_unknown_linux_gnu=x86_64-linux-gnu-ar + fi + mkdir -p /tmp/switchyard + cp -a /src/. /tmp/switchyard/ + cd /tmp/switchyard + export CARGO_TARGET_DIR=/tmp/target + cargo_args=(build --locked --release -p switchyard-nemo-relay-plugin) + if [[ -n "$PHASE1_CARGO_TARGET" ]]; then + cargo_args+=(--target "$PHASE1_CARGO_TARGET") + fi + cargo "${cargo_args[@]}" + python3 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py \ + --library "$PHASE1_LIBRARY_PATH" \ + --output /out + ' + +python3 - "$staging_dir" "$switchyard_repository" "$switchyard_commit" "$builder_image" "$builder_platform" "$cargo_target" "$target_architecture" <<'PY' +import hashlib +import json +import pathlib +import sys +import tomllib + +output = pathlib.Path(sys.argv[1]) +repository, commit, builder, builder_platform, cargo_target, target_architecture = sys.argv[2:] +manifest_path = output / "relay-plugin.toml" +if not manifest_path.is_file(): + raise SystemExit("bundle did not contain relay-plugin.toml") +with manifest_path.open("rb") as stream: + manifest = tomllib.load(stream) +if manifest.get("plugin", {}).get("id") != "nvidia.switchyard": + raise SystemExit("bundle manifest has the wrong plugin id") +libraries = [ + path for path in output.iterdir() + if path.is_file() and path.suffix in {".so", ".dylib", ".dll"} +] +if len(libraries) != 1: + raise SystemExit(f"expected one native library, found {len(libraries)}") +with libraries[0].open("rb") as stream: + elf_header = stream.read(20) +if elf_header[:4] != b"\x7fELF" or elf_header[5] != 1: + raise SystemExit("native library must be a little-endian ELF artifact") +machine = int.from_bytes(elf_header[18:20], "little") +expected_machine = {"x86_64": 62, "aarch64": 183}[target_architecture] +if machine != expected_machine: + raise SystemExit( + f"native library architecture mismatch: expected {target_architecture}, ELF e_machine={machine}" + ) + +def digest(path: pathlib.Path) -> str: + value = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + value.update(block) + return value.hexdigest() + +provenance = { + "schema_version": "harbor-hermes-switchyard.bundle.v1", + "repository": repository, + "commit": commit, + "builder_image": builder, + "builder_platform": builder_platform, + "cargo_target": cargo_target or f"native-{target_architecture}", + "target_architecture": target_architecture, + "plugin_id": "nvidia.switchyard", + "manifest_sha256": digest(manifest_path), + "library": libraries[0].name, + "library_sha256": digest(libraries[0]), +} +(output / "bundle-provenance.json").write_text( + json.dumps(provenance, indent=2, sort_keys=True) + "\n", + encoding="utf-8", +) +print(json.dumps(provenance, indent=2)) +PY + +mv "$staging_dir" "$output_dir" diff --git a/examples/harbor-hermes-switchyard/scripts/exec_process_group.py b/examples/harbor-hermes-switchyard/scripts/exec_process_group.py new file mode 100755 index 000000000..cc1ddcc38 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/exec_process_group.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exec a command as the leader of a new POSIX process group.""" + +from __future__ import annotations + +import os +import sys +from errno import EPERM + + +def main() -> int: + if len(sys.argv) < 2: + raise SystemExit("usage: exec_process_group.py COMMAND [ARG ...]") + try: + os.setsid() + except PermissionError as error: + if error.errno != EPERM: + raise + # Some launchers already make the child a process-group leader. In + # that case setsid(2) is forbidden, but the existing isolated group is + # exactly what the supervisor needs to signal as a unit. + if os.getpgrp() != os.getpid(): + raise RuntimeError("could not isolate the coordinator process group") from error + os.execvp(sys.argv[1], sys.argv[1:]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py b/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py new file mode 100755 index 000000000..a42ef3afd --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Small OpenAI Chat-compatible provider used by the offline Phase 1 smoke.""" + +from __future__ import annotations + +import argparse +import json +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + + +class Handler(BaseHTTPRequestHandler): + token: str + request_log: Path + + def log_message(self, _format: str, *_args: Any) -> None: + return + + def do_GET(self) -> None: # noqa: N802 + if self.path == "/healthz": + self.send_response(200) + self.send_header("content-type", "application/json") + self.end_headers() + self.wfile.write(b'{"status":"ok"}') + return + self.send_error(404) + + def do_POST(self) -> None: # noqa: N802 + if self.path != "/v1/chat/completions": + self.send_error(404) + return + if self.headers.get("authorization") != f"Bearer {self.token}": + self.send_error(401) + return + try: + length = int(self.headers.get("content-length", "0")) + request = json.loads(self.rfile.read(length)) + except (ValueError, json.JSONDecodeError): + self.send_error(400) + return + messages = request.get("messages", []) + serialized_messages = json.dumps(messages, separators=(",", ":")) + is_classifier = request.get("response_format") is not None or ( + "p_solve" in serialized_messages and "capability_boundary" in serialized_messages + ) + log_entry = { + "path": self.path, + "model": request.get("model"), + "message_count": len(messages), + "request_kind": "classifier" if is_classifier else "completion", + "authorization_present": True, + } + with self.request_log.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(log_entry, separators=(",", ":")) + "\n") + content = "OFFLINE_SWITCHYARD_OK" + if is_classifier: + force_strong = "force strong route" in serialized_messages + content = json.dumps( + { + "p_solve": 0.01 if force_strong else 0.99, + "capability_boundary": "supported", + "primary_rule": "SUP-1", + "crux": "deterministic offline smoke task", + }, + separators=(",", ":"), + ) + response = { + "id": "chatcmpl-phase1", + "object": "chat.completion", + "created": int(time.time()), + "model": request.get("model", "phase1-model"), + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": content, + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 4, + "completion_tokens": 3, + "total_tokens": 7, + }, + } + body = json.dumps(response).encode("utf-8") + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--bind", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--token", required=True) + parser.add_argument("--request-log", type=Path, required=True) + args = parser.parse_args() + args.request_log.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + Handler.token = args.token + Handler.request_log = args.request_log + ThreadingHTTPServer((args.bind, args.port), Handler).serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/fake_otlp_collector.py b/examples/harbor-hermes-switchyard/scripts/fake_otlp_collector.py new file mode 100755 index 000000000..ac38e2fb9 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/fake_otlp_collector.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Accept OTLP/HTTP protobuf requests for the offline compatibility smoke.""" + +from __future__ import annotations + +import argparse +import json +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + + +class Handler(BaseHTTPRequestHandler): + request_log: Path + + def log_message(self, _format: str, *_args: Any) -> None: + return + + def do_GET(self) -> None: # noqa: N802 + if self.path in {"/", "/healthz"}: + self.send_response(200) + self.end_headers() + return + self.send_error(404) + + def do_POST(self) -> None: # noqa: N802 + if self.path != "/v1/traces": + self.send_error(404) + return + length = int(self.headers.get("content-length", "0")) + body = self.rfile.read(length) + with self.request_log.open("a", encoding="utf-8") as stream: + stream.write( + json.dumps( + { + "path": self.path, + "content_type": self.headers.get("content-type"), + "bytes": len(body), + }, + separators=(",", ":"), + ) + + "\n" + ) + response = b"{}" + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(response))) + self.end_headers() + self.wfile.write(response) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, default=4318) + parser.add_argument("--request-log", type=Path, required=True) + args = parser.parse_args() + args.request_log.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + Handler.request_log = args.request_log + ThreadingHTTPServer(("127.0.0.1", args.port), Handler).serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py b/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py new file mode 100755 index 000000000..4f13efaa4 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py @@ -0,0 +1,305 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Create the direct Hermes result and lifecycle receipt inside a Harbor task.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import os +import re +import sys +import time +import tomllib +from pathlib import Path + +_SCRIPT_ROOT = Path(__file__).resolve().parent +if str(_SCRIPT_ROOT) not in sys.path: + sys.path.insert(0, str(_SCRIPT_ROOT)) +from relay_version import require_supported_version +from typing import Any + +SCHEMA_VERSION = "harbor-hermes-switchyard.phase1.v1" +MAX_DIAGNOSTIC_BYTES = 1024 * 1024 +HERMES_SESSION = Path("/logs/agent/hermes-session.jsonl") +HERMES_LOG = Path("/logs/agent/hermes.txt") + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def atomic_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.chmod(temporary, 0o600) + temporary.replace(path) + + +def checked_artifact_root(raw: str) -> Path: + path = Path(raw) + if not path.is_absolute(): + raise ValueError("artifact root must be absolute") + resolved = path.resolve(strict=False) + allowed = Path("/logs/agent").resolve() + if resolved == allowed or allowed not in resolved.parents: + raise ValueError("artifact root must be a child of /logs/agent") + resolved.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(resolved, 0o700) + return resolved + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=("initialize", "complete")) + parser.add_argument("--artifact-root", required=True) + parser.add_argument("--hermes-repository", required=True) + parser.add_argument("--hermes-commit", required=True) + parser.add_argument("--switchyard-commit", required=True) + parser.add_argument("--relay-wheel-sha256", required=True) + parser.add_argument("--relay-config", type=Path, required=True) + parser.add_argument("--switchyard-manifest", type=Path, required=True) + parser.add_argument("--switchyard-library", type=Path, required=True) + parser.add_argument("--session-handle", default="") + parser.add_argument("--started-at", type=float) + parser.add_argument("--error-type", default="") + return parser.parse_args() + + +def initialize(args: argparse.Namespace, root: Path) -> None: + with args.switchyard_manifest.open("rb") as stream: + manifest = tomllib.load(stream) + plugin_id = manifest.get("plugin", {}).get("id") + if plugin_id != "nvidia.switchyard": + raise ValueError(f"unexpected Switchyard plugin id: {plugin_id!r}") + + config_digest = sha256(args.relay_config) + receipt = { + "schema_version": SCHEMA_VERSION, + "status": "initialized", + "activation_mode": "relay_standard_dynamic", + "session_handle": args.session_handle or None, + "dependencies": { + "nemo_relay": { + "version": importlib.metadata.version("nemo-relay"), + "wheel_sha256": args.relay_wheel_sha256, + }, + "hermes": { + "repository": args.hermes_repository, + "commit": args.hermes_commit, + }, + "switchyard": { + "commit": args.switchyard_commit, + "plugin_id": plugin_id, + "manifest_sha256": sha256(args.switchyard_manifest), + "library_sha256": sha256(args.switchyard_library), + }, + }, + "relay_config_sha256": config_digest, + "dynamic_plugin_ids": [plugin_id], + "routing_contract": { + "relay_outer_lifecycle": True, + "execution_intercept_owner": plugin_id, + "provider_http_client_owner": "switchyard-llm-client", + "separate_switchyard_service": False, + }, + "artifacts": { + "root": str(root), + "atof": str(root / "relay" / "trajectory.atof.jsonl"), + "atif_directory": str(root / "relay" / "atif"), + "bounded_diagnostics": str(root / "diagnostics" / "hermes-tail.txt"), + }, + "cleanup": { + "plugin_host_closed": False, + "exporters_flushed": False, + "completion_marker_written": False, + }, + } + try: + require_supported_version(receipt["dependencies"]["nemo_relay"]["version"]) + except ValueError as error: + raise RuntimeError("Hermes environment did not install a supported nemo-relay release") from error + (root / "relay" / "atif").mkdir(mode=0o700, parents=True, exist_ok=True) + (root / "diagnostics").mkdir(mode=0o700, parents=True, exist_ok=True) + atomic_json(root / "direct-hermes-receipt.json", receipt) + + +def _read_session_messages(path: Path) -> tuple[list[dict[str, Any]], str | None]: + messages: list[dict[str, Any]] = [] + session_id: str | None = None + if not path.is_file(): + return messages, session_id + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(payload, dict): + continue + candidate = payload.get("session_id") or payload.get("id") + if isinstance(candidate, str) and candidate: + session_id = candidate + nested = payload.get("messages") + if isinstance(nested, list): + messages.extend(item for item in nested if isinstance(item, dict)) + elif payload.get("role"): + messages.append(payload) + return messages, session_id + + +def _text_content(content: Any) -> str: + if isinstance(content, str): + return content.strip() + if isinstance(content, list): + parts: list[str] = [] + for item in content: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict) and isinstance(item.get("text"), str): + parts.append(item["text"]) + return "\n".join(parts).strip() + return "" + + +def _last_assistant_response(messages: list[dict[str, Any]]) -> str | None: + for message in reversed(messages): + if message.get("role") == "assistant": + text = _text_content(message.get("content")) + if text: + return text + return None + + +def _response_from_cli_log(text: str) -> tuple[str | None, str | None]: + """Recover quiet-mode output when Hermes produced an empty session export.""" + lines = text.splitlines() + marker_index: int | None = None + session_id: str | None = None + for index, line in enumerate(lines): + match = re.fullmatch(r"\s*session_id:\s*(\S+)\s*", line) + if match: + marker_index = index + session_id = match.group(1) + if marker_index is None: + return None, None + response = "\n".join(lines[marker_index + 1 :]).strip() + return response or None, session_id + + +def _write_bounded_diagnostics(root: Path) -> str: + destination = root / "diagnostics" / "hermes-tail.txt" + if not HERMES_LOG.is_file(): + destination.write_text("", encoding="utf-8") + os.chmod(destination, 0o600) + return "" + size = HERMES_LOG.stat().st_size + with HERMES_LOG.open("rb") as stream: + if size > MAX_DIAGNOSTIC_BYTES: + stream.seek(-MAX_DIAGNOSTIC_BYTES, os.SEEK_END) + content = stream.read(MAX_DIAGNOSTIC_BYTES) + destination.write_bytes(content) + os.chmod(destination, 0o600) + return content.decode("utf-8", errors="replace") + + +def complete(args: argparse.Namespace, root: Path) -> None: + receipt_path = root / "direct-hermes-receipt.json" + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + messages, exported_session_id = _read_session_messages(HERMES_SESSION) + session_response = _last_assistant_response(messages) + diagnostic_text = _write_bounded_diagnostics(root) + log_response, log_session_id = _response_from_cli_log(diagnostic_text) + # Quiet mode sometimes leaves the exported session empty, so successful + # runs recover their response from stdout. A non-zero agent command may + # instead print an error after ``session_id:``; never promote that text to + # a completed response. The deterministic Phase 1 late-failure injection + # happens after a successful inherited run and intentionally exercises the + # bounded stdout recovery path. + allow_log_response = not args.error_type or args.error_type == "InjectedPostResponseFailure" + response = session_response or (log_response if allow_log_response else None) + exported_session_id = exported_session_id or log_session_id + lowered = diagnostic_text.lower() + cleanup_failure = any( + marker in lowered + for marker in ( + "plugin configuration cleanup failed", + "plugin subscriber flush failed", + "exporter flush failed", + "plugin teardown failed", + ) + ) + late_failure = bool(args.error_type or cleanup_failure) + if response and late_failure: + status = "preserved_completed_response" + elif response: + status = "completed" + else: + status = "failed" + + ended_at = time.time() + result = { + "schema_version": SCHEMA_VERSION, + "status": status, + "final_response": response, + "session_id": exported_session_id or args.session_handle or None, + "timing": { + "started_at_unix": args.started_at, + "ended_at_unix": ended_at, + "duration_seconds": (max(0.0, ended_at - args.started_at) if args.started_at is not None else None), + }, + "error": ( + { + "type": args.error_type or "RelayCleanupError", + "phase": ( + "shutdown" + if cleanup_failure or args.error_type == "InjectedPostResponseFailure" + else "agent" + ), + } + if late_failure + else None + ), + } + atomic_json(root / "direct-hermes-result.json", result) + + receipt["status"] = "completed" if status != "failed" else "failed" + receipt["result_status"] = status + receipt["cleanup"] = { + "plugin_host_closed": not cleanup_failure, + "exporters_flushed": not cleanup_failure, + "completion_marker_written": True, + "late_failure": late_failure, + } + atomic_json(receipt_path, receipt) + completion = { + "schema_version": SCHEMA_VERSION, + "status": "passed" if status != "failed" else "failed", + "result_status": status, + "completed_at_unix": ended_at, + } + atomic_json(root / "completion.json", completion) + marker = root / ".complete" + marker.write_text("completed\n", encoding="utf-8") + os.chmod(marker, 0o600) + + +def main() -> int: + args = parse_args() + root = checked_artifact_root(args.artifact_root) + if args.mode == "initialize": + initialize(args, root) + else: + complete(args, root) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh b/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh new file mode 100755 index 000000000..19a27aa4c --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ "$#" -eq 1 ]]; then + env_file="$example_root/.env" + session="$1" +elif [[ "$#" -eq 2 ]]; then + env_file="$1" + session="$2" +else + echo "usage: $0 [env-file] tmux-session-name" >&2 + exit 2 +fi +if [[ ! -f "$env_file" ]]; then + echo "Phase 2 environment file does not exist: $env_file" >&2 + exit 2 +fi +env_file="$(cd "$(dirname "$env_file")" && pwd)/$(basename "$env_file")" +if [[ ! "$session" =~ ^[A-Za-z0-9_.-]+$ ]]; then + echo "tmux session name may contain only letters, digits, dot, underscore, and dash" >&2 + exit 2 +fi +command -v tmux >/dev/null || { echo "tmux is required" >&2; exit 2; } +"$example_root/scripts/validate_phase2_environment.sh" "$env_file" +if tmux has-session -t "$session" 2>/dev/null; then + echo "tmux session already exists: $session" >&2 + exit 21 +fi + +# The caller must not source the protected file. Only its path is projected +# into the detached session; run_phase2_from_env.sh sources it with xtrace off. +tmux new-session -d -s "$session" \ + -e "TERMINAL_BENCH_ENV_FILE=$env_file" \ + "$example_root/scripts/run_phase2_from_env.sh" +echo "started tmux session: $session" diff --git a/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py b/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py new file mode 100755 index 000000000..04cc4fa8e --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py @@ -0,0 +1,152 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise Hermes #77915 and Switchyard #270 against a fake provider.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import threading +import tomllib +from pathlib import Path +from typing import Any + + +async def exercise(model: str) -> tuple[list[dict[str, Any]], dict[str, Any]]: + from agent.relay_runtime import RelayRuntime + + import nemo_relay + + host = RelayRuntime(profile_key="phase2-offline") + downstream_called = False + + async def forbidden_downstream(_request: Any) -> dict[str, Any]: + nonlocal downstream_called + downstream_called = True + raise AssertionError("Switchyard managed request reached Relay downstream callback") + + responses: list[dict[str, Any]] = [] + try: + cases = ( + ("phase2-offline-weak-session", "reply with the smoke marker"), + ("phase2-offline-strong-session", "force strong route and reply with the smoke marker"), + ) + for session_id, prompt in cases: + session = host.ensure_session({"session_id": session_id}) + if session is None: + raise RuntimeError("Hermes Relay runtime did not open a session") + request = nemo_relay.LLMRequest( + {}, + { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "stream": False, + }, + ) + response = await host.run_in_session_async( + session, + nemo_relay.llm.execute, + "openai.chat_completions", + request, + forbidden_downstream, + model_name=model, + response_codec=nemo_relay.codecs.OpenAIChatCodec(), + ) + responses.append(response) + host.close_session({"session_id": session_id}) + active_report = nemo_relay.plugin.report() + if active_report is None: + raise AssertionError("Relay did not expose an active plugin report") + report = active_report.to_dict() if hasattr(active_report, "to_dict") else active_report + finally: + host.shutdown() + if downstream_called: + raise AssertionError("Relay downstream callback was invoked") + contents = [response["choices"][0]["message"]["content"] for response in responses] + if contents != ["OFFLINE_SWITCHYARD_OK", "OFFLINE_SWITCHYARD_OK"]: + raise AssertionError(f"unexpected fake-provider responses: {contents!r}") + return responses, report + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--plugins", type=Path, required=True) + parser.add_argument("--artifacts", type=Path, required=True) + parser.add_argument("--request-log", type=Path, required=True) + parser.add_argument("--model", default="ollama-route-stub") + args = parser.parse_args() + config = tomllib.loads(args.plugins.read_text(encoding="utf-8")) + plugin = config["plugins"]["dynamic"][0]["config"] + algorithm = plugin["algorithm"] + targets = plugin["targets"] + classifier_model = targets[algorithm["classifier_target"]]["model"] + weak_model = targets[algorithm["weak_target"]]["model"] + strong_model = targets[algorithm["strong_target"]]["model"] + artifacts = args.artifacts.resolve() + artifacts.mkdir(mode=0o700, parents=True, exist_ok=True) + os.environ["HERMES_NEMO_RELAY_PLUGINS_TOML"] = str(args.plugins.resolve()) + responses, report = asyncio.run(exercise(args.model)) + + atof = artifacts / "relay" / "trajectory.atof.jsonl" + atif = sorted((artifacts / "relay" / "atif").glob("trajectory-*.atif.json")) + if not atof.is_file() or atof.stat().st_size == 0: + raise AssertionError("ATOF file sink did not emit") + if not atif: + raise AssertionError("ATIF file sink did not emit") + marks = [] + for line in atof.read_text(encoding="utf-8").splitlines(): + event = json.loads(line) + name = event.get("name") + if isinstance(name, str) and name.startswith("switchyard.routing."): + marks.append(name) + if not marks: + raise AssertionError("Switchyard routing marks were not emitted") + requests = [json.loads(line) for line in args.request_log.read_text(encoding="utf-8").splitlines() if line.strip()] + if len(requests) != 4 or not all(item.get("authorization_present") for item in requests): + raise AssertionError("fake provider did not receive exactly four authenticated requests") + request_kinds = [item.get("request_kind") for item in requests] + if request_kinds != ["classifier", "completion", "classifier", "completion"]: + raise AssertionError(f"unexpected provider request sequence: {request_kinds}") + request_models = [item.get("model") for item in requests] + expected_models = [ + classifier_model, + weak_model, + classifier_model, + strong_model, + ] + if request_models != expected_models: + raise AssertionError(f"unexpected provider model sequence: {request_models}") + if args.model in request_models: + raise AssertionError("Hermes caller stub reached the provider") + surviving = [ + thread.name for thread in threading.enumerate() if thread.name.startswith("hermes-nemo-relay-shutdown-") + ] + if surviving: + raise AssertionError(f"Hermes shutdown threads survived: {surviving}") + + result = { + "schema_version": "harbor-hermes-switchyard.offline-smoke.v1", + "status": "passed", + "responses": [response["choices"][0]["message"]["content"] for response in responses], + "provider_requests": len(requests), + "provider_request_kinds": request_kinds, + "provider_models": request_models, + "hermes_caller_model": args.model, + "relay_downstream_callback_called": False, + "switchyard_routing_marks": marks, + "active_plugin_report_before_shutdown": report, + "atof": str(atof), + "atif": [str(path) for path in atif], + "surviving_shutdown_threads": surviving, + } + output = artifacts / "offline-smoke.json" + output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py new file mode 100755 index 000000000..5e8c54b7b --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py @@ -0,0 +1,315 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare one immutable evaluation run root and render its Relay config.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tomllib +from pathlib import Path +from urllib.parse import urlsplit + +import tomli_w +from relay_version import RELAY_REQUIREMENT, require_supported_version, wheel_version + +HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" +HERMES_REF = "feat/relay-native-plugin-init" +HERMES_COMMIT = "a3d472f0e6bdc376df87b1436a461c4796db6747" +SWITCHYARD_REPOSITORY = "https://github.com/bbednarski9/Switchyard.git" +SWITCHYARD_COMMIT = "8daac03edf8544144833af1fd009b3da737715bc" +SAFE_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,127}") + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def checked_url(value: str, name: str) -> str: + parsed = urlsplit(value) + if ( + parsed.scheme not in {"http", "https"} + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.fragment + ): + raise ValueError(f"{name} must be a credential-free HTTP(S) URL") + return value.rstrip("/") + + +def checked_label(value: str, name: str) -> str: + if not SAFE_LABEL.fullmatch(value): + raise ValueError(f"{name} contains unsupported characters") + return value + + +def download_relay_wheel(destination: Path, architecture: str) -> Path: + destination.mkdir(mode=0o700, parents=True) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "download", + "--only-binary=:all:", + "--no-deps", + "--platform", + f"manylinux2014_{architecture}", + "--implementation", + "cp", + "--python-version", + "311", + "--abi", + "abi3", + "--dest", + str(destination), + RELAY_REQUIREMENT, + ], + check=True, + ) + wheels = sorted(destination.glob("nemo_relay-*.whl")) + if len(wheels) != 1: + raise RuntimeError(f"expected one Relay wheel, found {len(wheels)}") + return wheels[0] + + +def verify_relay_wheel(path: Path, architecture: str) -> str: + if not path.is_file() or not path.name.startswith("nemo_relay-"): + raise ValueError("Relay wheel must be a nemo_relay wheel") + if "manylinux" not in path.name or architecture not in path.name: + raise ValueError(f"Relay wheel must target Linux {architecture}") + return wheel_version(path) + + +def verify_native_library(path: Path, architecture: str) -> None: + with path.open("rb") as stream: + header = stream.read(20) + if header[:4] != b"\x7fELF" or len(header) < 20 or header[5] != 1: + raise ValueError("Switchyard native library must be a little-endian ELF artifact") + machine = int.from_bytes(header[18:20], "little") + expected = {"x86_64": 62, "aarch64": 183}[architecture] + if machine != expected: + raise ValueError(f"Switchyard library does not target {architecture}: ELF e_machine={machine}") + + +def plugin_settings(config: dict[str, object]) -> dict[str, str]: + plugins = config.get("plugins") + if not isinstance(plugins, dict) or not isinstance(plugins.get("dynamic"), list): + raise ValueError("plugins.toml.in must define one dynamic plugin") + dynamic = plugins["dynamic"] + if len(dynamic) != 1 or not isinstance(dynamic[0], dict): + raise ValueError("plugins.toml.in must define exactly one dynamic plugin") + plugin_config = dynamic[0].get("config") + if not isinstance(plugin_config, dict) or not isinstance(plugin_config.get("targets"), dict): + raise ValueError("Switchyard dynamic plugin targets are missing") + targets = plugin_config["targets"] + if set(targets) != {"strong", "weak", "judge"}: + raise ValueError("Switchyard must define strong, weak, and judge targets") + settings: dict[str, str] = {} + algorithm = plugin_config.get("algorithm") + classifier_target = algorithm.get("classifier_target") if isinstance(algorithm, dict) else None + if classifier_target not in targets: + raise ValueError("Switchyard classifier_target must reference a configured target") + settings["classifier_target"] = classifier_target + for name in ("strong", "weak", "judge"): + target = targets[name] + if not isinstance(target, dict): + raise ValueError(f"Switchyard target is invalid: {name}") + model = checked_label(str(target.get("model", "")), f"{name}_model") + base_url = checked_url(str(target.get("base_url", "")), f"{name}_base_url") + header_env = target.get("header_env") + if header_env != {"authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION"}: + raise ValueError("plugins.toml.in must reference SWITCHYARD_PROVIDER_AUTHORIZATION") + settings[f"{name}_model"] = model + settings[f"{name}_base_url"] = base_url + if settings["strong_model"] == settings["weak_model"]: + raise ValueError("strong and weak models must be distinct") + components = config.get("components") + if not isinstance(components, list): + raise ValueError("Relay components are missing") + observability = next( + ( + component + for component in components + if isinstance(component, dict) and component.get("kind") == "observability" + ), + None, + ) + if not isinstance(observability, dict): + raise ValueError("Relay observability component is missing") + observation_config = observability.get("config") + if not isinstance(observation_config, dict) or not isinstance(observation_config.get("atif"), dict): + raise ValueError("Relay ATIF configuration is missing") + settings["hermes_caller_model"] = checked_label( + str(observation_config["atif"].get("model_name", "")), "hermes_caller_model" + ) + if settings["hermes_caller_model"] in { + settings["strong_model"], settings["weak_model"], settings["judge_model"] + }: + raise ValueError("Hermes caller model must be distinct from Switchyard targets") + return settings + + +def render_config( + template: Path, + output: Path, + replacements: dict[str, str], + test_overrides: dict[str, str] | None = None, +) -> dict[str, str]: + rendered = template.read_text(encoding="utf-8") + for key, value in replacements.items(): + if "\n" in value or "\r" in value: + raise ValueError(f"replacement {key} contains a newline") + rendered = rendered.replace(f"@{key}@", value) + unresolved = sorted(set(re.findall(r"@[A-Z0-9_]+@", rendered))) + if unresolved: + raise ValueError(f"unresolved Relay config placeholders: {unresolved}") + config = tomllib.loads(rendered) + if test_overrides: + plugin = config["plugins"]["dynamic"][0]["config"] + old_models = {name: plugin["targets"][name]["model"] for name in ("strong", "weak", "judge")} + for name in ("strong", "weak", "judge"): + override = f"{name}_model" + plugin["targets"][name]["model"] = test_overrides[override] + plugin["targets"][name]["base_url"] = test_overrides["provider_base_url"] + pricing = config["components"][0]["config"]["sources"][0]["catalog"]["entries"] + replacement_models = { + old_models[name]: test_overrides[f"{name}_model"] + for name in old_models + } + for entry in pricing: + entry["model_id"] = replacement_models.get(entry["model_id"], entry["model_id"]) + settings = plugin_settings(config) + output.write_text(tomli_w.dumps(config), encoding="utf-8") + os.chmod(output, 0o600) + return settings + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--switchyard-bundle", type=Path, required=True) + parser.add_argument("--relay-wheel", type=Path) + parser.add_argument("--relay-architecture", choices=("x86_64", "aarch64"), default="x86_64") + parser.add_argument("--plugin-config-template", type=Path) + parser.add_argument("--test-provider-base-url") + parser.add_argument("--test-strong-model") + parser.add_argument("--test-weak-model") + parser.add_argument("--test-judge-model") + parser.add_argument("--openinference-endpoint", required=True) + parser.add_argument("--phoenix-project", required=True) + parser.add_argument("--eval-cohort", required=True) + args = parser.parse_args() + + example_root = Path(__file__).resolve().parents[1] + run_root = args.run_root.expanduser().resolve() + if run_root.exists(): + raise FileExistsError(f"run root already exists: {run_root}") + run_root.mkdir(mode=0o700, parents=True) + runtime = run_root / "runtime" + artifacts = run_root / "artifacts" + jobs = run_root / "jobs" + for path in (runtime, artifacts, jobs): + path.mkdir(mode=0o700) + + source_bundle = args.switchyard_bundle.expanduser().resolve() + if not (source_bundle / "relay-plugin.toml").is_file(): + raise FileNotFoundError(source_bundle / "relay-plugin.toml") + bundle = runtime / "switchyard-plugin" + shutil.copytree(source_bundle, bundle) + + if args.relay_wheel: + source_wheel = args.relay_wheel.expanduser().resolve() + relay_version = verify_relay_wheel(source_wheel, args.relay_architecture) + wheel_dir = runtime / "wheels" + wheel_dir.mkdir(mode=0o700) + relay_wheel = wheel_dir / source_wheel.name + shutil.copy2(source_wheel, relay_wheel) + else: + relay_wheel = download_relay_wheel(runtime / "wheels", args.relay_architecture) + relay_version = verify_relay_wheel(relay_wheel, args.relay_architecture) + + openinference_endpoint = checked_url(args.openinference_endpoint, "openinference_endpoint") + phoenix_project = checked_label(args.phoenix_project, "phoenix_project") + eval_cohort = checked_label(args.eval_cohort, "eval_cohort") + plugin_template = (args.plugin_config_template or example_root / "config" / "plugins.toml.in").resolve(strict=True) + test_values = (args.test_provider_base_url, args.test_strong_model, args.test_weak_model, args.test_judge_model) + if any(test_values) and not all(test_values): + raise ValueError("all test provider overrides must be supplied together") + test_overrides = None + if all(test_values): + test_overrides = { + "provider_base_url": checked_url(args.test_provider_base_url, "test_provider_base_url"), + "strong_model": checked_label(args.test_strong_model, "test_strong_model"), + "weak_model": checked_label(args.test_weak_model, "test_weak_model"), + "judge_model": checked_label(args.test_judge_model, "test_judge_model"), + } + + config_path = runtime / "plugins.toml" + routing = render_config( + plugin_template, + config_path, + { + "HERMES_COMMIT": HERMES_COMMIT, + "OPENINFERENCE_ENDPOINT": openinference_endpoint, + "PHOENIX_PROJECT": phoenix_project, + "EVAL_COHORT": eval_cohort, + }, + test_overrides, + ) + + manifest = bundle / "relay-plugin.toml" + libraries = sorted(path for path in bundle.iterdir() if path.is_file() and path.suffix in {".so", ".dylib", ".dll"}) + if len(libraries) != 1: + raise ValueError("Switchyard bundle must contain exactly one native library") + verify_native_library(libraries[0], args.relay_architecture) + provenance = { + "schema_version": "harbor-hermes-switchyard.phase1.v1", + "nemo_relay": { + "version": relay_version, + "architecture": args.relay_architecture, + "wheel": relay_wheel.name, + "wheel_sha256": sha256(relay_wheel), + }, + "hermes": { + "repository": HERMES_REPOSITORY, + "ref": HERMES_REF, + "commit": HERMES_COMMIT, + }, + "switchyard": { + "repository": SWITCHYARD_REPOSITORY, + "commit": SWITCHYARD_COMMIT, + "manifest_sha256": sha256(manifest), + "library": libraries[0].name, + "library_sha256": sha256(libraries[0]), + }, + "relay_config_sha256": sha256(config_path), + "plugin_config_template_sha256": sha256(plugin_template), + "routing": { + "algorithm": "llm_classifier", + **routing, + }, + "phoenix_project": phoenix_project, + "eval_cohort": eval_cohort, + } + provenance_path = runtime / "provenance.json" + provenance_path.write_text(json.dumps(provenance, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.chmod(provenance_path, 0o600) + print(json.dumps({"run_root": str(run_root), "provenance": provenance}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/relay_version.py b/examples/harbor-hermes-switchyard/scripts/relay_version.py new file mode 100644 index 000000000..01e856627 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/relay_version.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Version helpers for the released NeMo Relay wheel used by Phase 2.""" + +from __future__ import annotations + +import re +from pathlib import Path +from zipfile import ZipFile + +RELAY_REQUIREMENT = "nemo-relay>=0.7.0" +RELAY_MIN_VERSION = "0.7.0" + + +def version_tuple(value: str) -> tuple[int, int, int]: + match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", value) + if match is None: + raise ValueError(f"nemo-relay version must be a stable semantic version: {value!r}") + return tuple(int(component) for component in match.groups()) + + +def require_supported_version(value: str) -> str: + if version_tuple(value) < version_tuple(RELAY_MIN_VERSION): + raise ValueError(f"nemo-relay must satisfy {RELAY_REQUIREMENT}; found {value}") + return value + + +def wheel_version(path: Path) -> str: + with ZipFile(path) as wheel: + metadata_names = [name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")] + if len(metadata_names) != 1: + raise ValueError("Relay wheel has an ambiguous METADATA payload") + metadata = wheel.read(metadata_names[0]).decode("utf-8", errors="strict") + name = re.search(r"^Name: (.+)$", metadata, re.MULTILINE) + version = re.search(r"^Version: (.+)$", metadata, re.MULTILINE) + if name is None or name.group(1) != "nemo-relay" or version is None: + raise ValueError("Relay wheel metadata does not identify nemo-relay") + return require_supported_version(version.group(1)) diff --git a/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh b/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh new file mode 100755 index 000000000..a1f156628 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +run_root="${1:-}" +admission_output="${2:-$run_root/artifacts/offline-admission.json}" +image="${OFFLINE_COMPAT_IMAGE:-python:3.11-bookworm}" +platform="${OFFLINE_COMPAT_PLATFORM:-linux/amd64}" +hermes_repository="${HERMES_REPOSITORY:-https://github.com/bbednarski9/hermes-agent.git}" +hermes_ref="${HERMES_REF:-feat/relay-native-plugin-init}" +hermes_commit="${HERMES_COMMIT:-a3d472f0e6bdc376df87b1436a461c4796db6747}" + +if [[ -z "$run_root" || "$run_root" != /* ]]; then + echo "usage: $0 /absolute/prepared-run-root" >&2 + exit 2 +fi +for required in \ + "$run_root/runtime/plugins.toml" \ + "$run_root/runtime/provenance.json" \ + "$run_root/runtime/switchyard-plugin/relay-plugin.toml"; do + [[ -f "$required" ]] || { echo "missing prepared runtime file: $required" >&2; exit 1; } +done +command -v docker >/dev/null || { echo "docker is required" >&2; exit 1; } +docker info >/dev/null +case "$platform" in + linux/amd64) expected_architecture=x86_64 ;; + linux/arm64) expected_architecture=aarch64 ;; + *) echo "OFFLINE_COMPAT_PLATFORM must be linux/amd64 or linux/arm64" >&2; exit 2 ;; +esac +prepared_architecture="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["nemo_relay"].get("architecture", "x86_64"))' "$run_root/runtime/provenance.json")" +[[ "$prepared_architecture" == "$expected_architecture" ]] || { + echo "prepared Relay architecture $prepared_architecture does not match $platform" >&2 + exit 2 +} + +artifacts="$run_root/artifacts/offline-compatibility" +if [[ -e "$artifacts" ]]; then + [[ -d "$artifacts" ]] || { echo "artifact path is not a directory: $artifacts" >&2; exit 1; } + [[ -z "$(find "$artifacts" -mindepth 1 -maxdepth 1 -print -quit)" ]] || { + echo "offline compatibility artifacts already exist: $artifacts" >&2 + exit 1 + } +else + mkdir -m 0700 "$artifacts" +fi + +docker run --rm \ + --platform "$platform" \ + --volume "$example_root:/example:ro" \ + --volume "$run_root/runtime:/runtime:ro" \ + --volume "$run_root/runtime/switchyard-plugin:/opt/relay-plugins/nvidia.switchyard:ro" \ + --volume "$artifacts:/logs/agent/direct-hermes" \ + "$image" \ + bash -lc ' + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + export HERMES_HOME=/tmp/hermes + export HERMES_NEMO_RELAY_PLUGINS_TOML=/runtime/plugins.toml + export SWITCHYARD_PROVIDER_AUTHORIZATION="Bearer phase2-offline-secret-value" + apt-get update + apt-get install -y --no-install-recommends build-essential ca-certificates curl git ripgrep xz-utils + git clone --no-tags --branch "'"$hermes_ref"'" "'"$hermes_repository"'" /tmp/hermes-agent-src + git -C /tmp/hermes-agent-src fetch --depth 1 origin "'"$hermes_commit"'" + git -C /tmp/hermes-agent-src checkout --detach "'"$hermes_commit"'" + # The installer treats ffmpeg as optional, but a root-owned Debian smoke + # otherwise installs ~500 MB of unrelated TTS/video packages. Advertise a + # command only during installation; it is not on PATH for the runtime. + mkdir /tmp/hermes-install-path + ln -s /bin/true /tmp/hermes-install-path/ffmpeg + HERMES_INSTALL_DIR=/tmp/hermes-agent-src \ + PATH=/tmp/hermes-install-path:$PATH \ + bash /tmp/hermes-agent-src/scripts/install.sh \ + --skip-setup --skip-browser --no-skills \ + --dir /tmp/hermes-agent-src \ + --branch "'"$hermes_ref"'" \ + --commit "'"$hermes_commit"'" --force-commit + test "$(git -C /tmp/hermes-agent-src rev-parse HEAD)" = "'"$hermes_commit"'" + cd /tmp/hermes-agent-src + UV_PROJECT_ENVIRONMENT=/tmp/hermes-agent-src/venv \ + /tmp/hermes/bin/uv sync --frozen --extra all + cd / + /tmp/hermes-agent-src/venv/bin/python -c \ + "import importlib.metadata as m; assert tuple(map(int, m.version(\"nemo-relay\").split(\".\"))) >= (0, 7, 0)" + relay_wheel="$(find /runtime/wheels -maxdepth 1 -type f -name "nemo_relay-*.whl" -print)" + test -n "$relay_wheel" + expected_wheel_sha="$(python3 -c "import json; print(json.load(open(\"/runtime/provenance.json\"))[\"nemo_relay\"][\"wheel_sha256\"])")" + test "$(sha256sum "$relay_wheel" | cut -d" " -f1)" = "$expected_wheel_sha" + /tmp/hermes/bin/uv pip install \ + --python /tmp/hermes-agent-src/venv/bin/python \ + --force-reinstall --no-deps "$relay_wheel" + python3 /example/scripts/fake_openai_upstream.py \ + --token phase2-offline-secret-value \ + --request-log /logs/agent/direct-hermes/provider-requests.jsonl & + provider_pid=$! + python3 /example/scripts/fake_otlp_collector.py \ + --request-log /logs/agent/direct-hermes/otlp-requests.jsonl & + otlp_pid=$! + cleanup() { + kill "$provider_pid" "$otlp_pid" >/dev/null 2>&1 || true + wait "$provider_pid" "$otlp_pid" >/dev/null 2>&1 || true + } + trap cleanup EXIT + for endpoint in http://127.0.0.1:8000/healthz http://127.0.0.1:4318/healthz; do + for _ in $(seq 1 50); do + curl --fail --silent "$endpoint" >/dev/null && break + sleep 0.1 + done + curl --fail --silent "$endpoint" >/dev/null + done + PYTHONPATH=/tmp/hermes-agent-src \ + /tmp/hermes-agent-src/venv/bin/python \ + /example/scripts/offline_compatibility_smoke.py \ + --plugins /runtime/plugins.toml \ + --artifacts /logs/agent/direct-hermes \ + --request-log /logs/agent/direct-hermes/provider-requests.jsonl + test -s /logs/agent/direct-hermes/otlp-requests.jsonl + if grep -R -F phase2-offline-secret-value /logs/agent/direct-hermes >/dev/null; then + echo "offline secret leaked into persisted evidence" >&2 + exit 1 + fi + ' + +python3 - "$artifacts" "$run_root/runtime/provenance.json" "$admission_output" <<'PY' +import hashlib +import json +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +provenance_path = pathlib.Path(sys.argv[2]) +output = pathlib.Path(sys.argv[3]) +result = json.loads((root / "offline-smoke.json").read_text()) +if result.get("status") != "passed": + raise SystemExit("offline compatibility smoke did not pass") +provenance = json.loads(provenance_path.read_text()) +admission = { + "schema_version": "harbor-hermes-switchyard.phase2-offline-admission.v1", + "status": "passed", + "hermes_commit": provenance["hermes"]["commit"], + "relay_architecture": provenance["nemo_relay"]["architecture"], + "relay_wheel_sha256": provenance["nemo_relay"]["wheel_sha256"], + "switchyard_library_sha256": provenance["switchyard"]["library_sha256"], + "plugin_config_template_sha256": provenance["plugin_config_template_sha256"], + "offline_relay_config_sha256": provenance["relay_config_sha256"], + "offline_smoke_sha256": hashlib.sha256((root / "offline-smoke.json").read_bytes()).hexdigest(), + "provider_requests": result["provider_requests"], + "surviving_shutdown_threads": result["surviving_shutdown_threads"], +} +output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) +output.write_text(json.dumps(admission, indent=2, sort_keys=True) + "\n") +print(json.dumps(admission, indent=2)) +PY diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py new file mode 100755 index 000000000..f01251867 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py @@ -0,0 +1,1181 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run one resumable, isolated Terminal-Bench cohort for Phase 2.""" + +from __future__ import annotations + +import argparse +import asyncio +import fcntl +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import time +import tomllib +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +_SCRIPT_ROOT = Path(__file__).resolve().parent +if str(_SCRIPT_ROOT) not in sys.path: + sys.path.insert(0, str(_SCRIPT_ROOT)) +import run_setup_admission as setup_admission # noqa: E402 +from relay_version import wheel_version # noqa: E402 + +SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-cohort.v1" +PLAN_SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-plan.v1" +TASK_STATE_SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-task-state.v1" +EXPECTED_HERMES_COMMIT = "a3d472f0e6bdc376df87b1436a461c4796db6747" +HERMETIC_RUNTIME_SCHEMA = "harbor-hermes-switchyard.hermetic-runtime.v1" +INFRASTRUCTURE_PATTERNS = ( + "apt-get update && apt-get install", + "cannot connect to the docker daemon", + "connection refused", + "connection reset", + "connection timed out", + "command failed (exit 137)", + "connecterror", + "context deadline exceeded", + "docker build failed", + "docker is not running", + "failed to resolve source metadata", + "error getting dataset", + "i/o timeout", + "network is unreachable", + "no space left on device", + "phoenix upload", + "provider has been unresponsive", + "provider returned http 408", + "registry-1.docker.io", + "temporary failure in name resolution", + "tls handshake timeout", + "too many requests", +) + + +@dataclass(frozen=True) +class Task: + index: int + name: str + memory_gb: int + + @property + def directory_name(self) -> str: + return f"{self.index:03d}-{self.name}" + + def as_json(self) -> dict[str, Any]: + return {"index": self.index, "name": self.name, "memory_gb": self.memory_gb} + + +def read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"expected a JSON object: {path}") + return value + + +def write_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(path) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def sha256_file_set(root: Path, paths: list[Path]) -> str: + digest = hashlib.sha256() + for path in sorted(paths): + relative = path.relative_to(root).as_posix() + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(sha256_file(path).encode()) + digest.update(b"\n") + return digest.hexdigest() + + +def parse_memory_gb(value: object, task_name: str) -> int: + text = str(value or "2G").strip() + match = re.fullmatch(r"([1-9][0-9]*)G", text, re.IGNORECASE) + if not match: + raise ValueError(f"unsupported memory value for {task_name}: {text}") + return int(match.group(1)) + + +def discover_tasks(dataset_root: Path, sample_count: int, excluded: set[str], canary_task: str | None) -> list[Task]: + if not dataset_root.is_dir(): + raise ValueError(f"dataset root is not a directory: {dataset_root}") + discovered: list[tuple[str, int]] = [] + for task_root in sorted(path for path in dataset_root.iterdir() if path.is_dir()): + if task_root.name in excluded: + continue + task_toml = task_root / "task.toml" + if not task_toml.is_file(): + continue + with task_toml.open("rb") as stream: + config = tomllib.load(stream) + memory = parse_memory_gb((config.get("environment") or {}).get("memory"), task_root.name) + discovered.append((task_root.name, memory)) + selected = discovered[:sample_count] + if len(selected) != sample_count: + raise ValueError(f"dataset contains {len(selected)} selectable tasks; requested {sample_count}") + if canary_task: + canaries = [item for item in selected if item[0] == canary_task] + if len(canaries) != 1: + raise ValueError(f"canary task is not uniquely selectable: {canary_task}") + ordered = canaries + [item for item in selected if item[0] != canary_task] + else: + ordered = selected + return [Task(index, name, memory) for index, (name, memory) in enumerate(ordered, 1)] + + +def task_summary_passed(path: Path) -> bool: + if not path.is_file(): + return False + try: + summary = read_json(path) + except (OSError, ValueError, json.JSONDecodeError): + return False + validation = summary.get("validation") + benchmark = validation.get("benchmark", {}) if isinstance(validation, dict) else {} + benchmark_status = benchmark.get("status", validation.get("status") if isinstance(validation, dict) else None) + return summary.get("status") == "passed" and benchmark_status == "passed" + + +def successful_attempt(task_root: Path) -> Path | None: + attempts = task_root / "attempts" + if not attempts.is_dir(): + return None + for attempt in sorted((path for path in attempts.iterdir() if path.is_dir()), reverse=True): + if task_summary_passed(attempt / "summary.json"): + return attempt + return None + + +def classify_failure(log_text: str) -> str: + lowered = log_text.lower() + if any(pattern in lowered for pattern in INFRASTRUCTURE_PATTERNS): + return "infrastructure" + if re.search(r"http (429|5[0-9][0-9])\b", lowered): + return "infrastructure" + return "harness_or_integration" + + +def classify_attempt_failure(log_text: str, attempt: Path) -> str: + """Classify a failed attempt using the wrapper and Harbor's nested logs.""" + + if classify_failure(log_text) == "infrastructure": + return "infrastructure" + diagnostic_paths = [ + *attempt.rglob("*.log"), + *attempt.rglob("result.json"), + ] + for path in sorted(diagnostic_paths): + try: + with path.open(encoding="utf-8", errors="replace") as stream: + while chunk := stream.read(1024 * 1024): + if classify_failure(chunk) == "infrastructure": + return "infrastructure" + except OSError: + continue + return "harness_or_integration" + + +def plugin_contract(path: Path) -> dict[str, Any]: + with path.open("rb") as stream: + config = tomllib.load(stream) + plugins = config.get("plugins", {}).get("dynamic", []) + if len(plugins) != 1: + raise ValueError("plugin configuration must define exactly one dynamic plugin") + targets = plugins[0].get("config", {}).get("targets", {}) + if set(targets) != {"strong", "weak", "judge"}: + raise ValueError("plugin configuration must define strong, weak, and judge targets") + for target in targets.values(): + if target.get("header_env") != {"authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION"}: + raise ValueError("plugin authorization must reference SWITCHYARD_PROVIDER_AUTHORIZATION") + models = [targets[name].get("model") for name in ("weak", "strong")] + catalog_models = [*models, targets["judge"].get("model")] + base_urls = sorted({targets[name].get("base_url") for name in ("weak", "strong", "judge")}) + if any(not isinstance(value, str) or not value for value in catalog_models + base_urls): + raise ValueError("plugin target models and base URLs must be non-empty") + for base_url in base_urls: + parsed = urllib.parse.urlsplit(base_url) + if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password: + raise ValueError("plugin target base URLs must be credential-free HTTP(S) URLs") + components = {component.get("kind"): component for component in config.get("components", [])} + caller = components.get("observability", {}).get("config", {}).get("atif", {}).get("model_name") + if not isinstance(caller, str) or caller in models: + raise ValueError("Hermes caller model must be distinct from plugin target models") + return { + "required_models": sorted(models), + "catalog_models": sorted(catalog_models), + "strong_model": targets["strong"]["model"], + "weak_model": targets["weak"]["model"], + "judge_model": targets["judge"]["model"], + "hermes_caller_model": caller, + "provider_base_urls": base_urls, + "sha256": sha256_file(path), + } + + +def switchyard_library(bundle: Path) -> Path: + libraries = sorted(bundle.glob("libswitchyard_nemo_relay_plugin.*")) + if len(libraries) != 1: + raise ValueError("Switchyard bundle must contain exactly one native library") + return libraries[0] + + +def load_hermetic_runtime(path: Path, args: argparse.Namespace) -> dict[str, Any]: + payload = setup_admission.load_payload(path) + expected = { + "schema_version": HERMETIC_RUNTIME_SCHEMA, + "status": "passed", + "hermes_commit": EXPECTED_HERMES_COMMIT, + "relay_version": wheel_version(args.relay_wheel), + "relay_wheel_sha256": sha256_file(args.relay_wheel), + "relay_architecture": args.relay_architecture, + } + mismatches = { + key: {"expected": value, "actual": payload.get(key)} + for key, value in expected.items() + if payload.get(key) != value + } + if mismatches: + raise ValueError(f"hermetic runtime does not match cohort inputs: {mismatches}") + digest = payload.get("content_sha256") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ValueError("hermetic runtime content digest is missing or invalid") + for relative in ("bin/hermes", "bin/python", "bin/uv", "hermes-agent-src/venv"): + if not (path / relative).exists(): + raise FileNotFoundError(path / relative) + return payload + + +def ensure_hermetic_runtime(args: argparse.Namespace) -> tuple[Path, dict[str, Any]]: + wheel_digest = sha256_file(args.relay_wheel) + name = f"hermes-{EXPECTED_HERMES_COMMIT[:8]}-relay-070-{args.relay_architecture}-{wheel_digest[:12]}" + output = args.bootstrap_root / name + args.bootstrap_root.mkdir(mode=0o700, parents=True, exist_ok=True) + lock_path = args.bootstrap_root / f".{name}.lock" + with lock_path.open("a+") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + if output.is_dir(): + return output, load_hermetic_runtime(output, args) + if output.exists(): + raise ValueError(f"hermetic runtime cache path is not a directory: {output}") + subprocess.run( + [ + str(args.python_bin), + str(args.hermetic_runtime_builder), + "--output", + str(output), + "--relay-wheel", + str(args.relay_wheel), + "--relay-architecture", + args.relay_architecture, + "--hermes-commit", + EXPECTED_HERMES_COMMIT, + ], + check=True, + ) + return output, load_hermetic_runtime(output, args) + + +def prepare_setup_runtime(args: argparse.Namespace) -> Path: + destination = args.run_root / "setup-runtime" + runtime = destination / "runtime" + provenance = runtime / "provenance.json" + if provenance.is_file(): + observed = read_json(provenance) + if ( + observed.get("nemo_relay", {}).get("wheel_sha256") != sha256_file(args.relay_wheel) + or observed.get("switchyard", {}).get("library_sha256") + != sha256_file(switchyard_library(args.switchyard_bundle)) + or observed.get("relay_config_sha256") != sha256_file(runtime / "plugins.toml") + ): + raise ValueError("existing setup runtime does not match cohort inputs") + return runtime + if destination.exists(): + raise ValueError(f"incomplete setup runtime already exists: {destination}") + temporary = destination.with_name(f".{destination.name}.preparing-{os.getpid()}") + if temporary.exists(): + raise ValueError(f"stale setup runtime preparation exists: {temporary}") + try: + subprocess.run( + [ + str(args.python_bin), + str(args.runtime_preparer), + "--run-root", + str(temporary), + "--switchyard-bundle", + str(args.switchyard_bundle), + "--relay-wheel", + str(args.relay_wheel), + "--relay-architecture", + args.relay_architecture, + "--plugin-config-template", + str(args.plugin_config_template), + "--openinference-endpoint", + "http://127.0.0.1:9/v1/traces", + "--phoenix-project", + args.phoenix_project, + "--eval-cohort", + args.eval_cohort, + ], + check=True, + ) + temporary.replace(destination) + except BaseException: + shutil.rmtree(temporary, ignore_errors=True) + raise + return runtime + + +def bootstrap_preflight(args: argparse.Namespace) -> dict[str, Any]: + clock = setup_admission.run_clock_preflight() + if clock["status"] != "passed": + raise RuntimeError("wall-clock preflight failed; synchronize the host and Docker clocks") + compatibility_plan = { + "inputs": { + "relay_architecture": args.relay_architecture, + "switchyard_bundle": str(args.switchyard_bundle), + "switchyard_library_sha256": sha256_file(switchyard_library(args.switchyard_bundle)), + } + } + plugin = setup_admission.run_plugin_compatibility_preflight(compatibility_plan) + if plugin["status"] != "passed": + raise RuntimeError("Switchyard plugin does not load in the oldest supported task base") + evidence = {"status": "passed", "clock": clock, "plugin_compatibility": plugin} + write_json(args.run_root / "bootstrap-preflight.json", evidence) + return evidence + + +def validate_smoke_evidence( + path: Path, + expected_count: int, + dataset_root: Path, + concurrency: int, + relay_architecture: str, + relay_wheel: Path, + switchyard_bundle: Path, + plugin_config_template: Path, +) -> None: + evidence = read_json(path) + dataset_root = dataset_root.resolve() + task_tomls = sorted(dataset_root.glob("*/task.toml")) + expected_names = [task_toml.parent.name for task_toml in task_tomls] + observed_tasks = evidence.get("tasks") + observed_names = ( + [task.get("name") for task in observed_tasks] + if isinstance(observed_tasks, list) and all(isinstance(task, dict) for task in observed_tasks) + else [] + ) + records_valid = len(observed_names) == expected_count + if records_valid: + for record in observed_tasks: + name = record.get("name") + try: + instruction = (dataset_root / record["instruction_path"]).resolve(strict=True) + verifier = (dataset_root / record["verifier_path"]).resolve(strict=True) + instruction.relative_to(dataset_root) + verifier.relative_to(dataset_root) + except (KeyError, OSError, RuntimeError, TypeError, ValueError): + records_valid = False + break + task_toml = dataset_root / str(name) / "task.toml" + if ( + instruction.parent != task_toml.parent + or verifier.parent.parent != task_toml.parent + or record.get("task_toml_sha256") != sha256_file(task_toml) + or record.get("instruction_sha256") != sha256_file(instruction) + or record.get("test_sha256") != sha256_file(verifier) + ): + records_valid = False + break + if ( + evidence.get("schema_version") != "harbor-hermes-switchyard.phase2-smoke.v1" + or evidence.get("status") != "passed" + or evidence.get("task_count") != expected_count + or len(task_tomls) != expected_count + or observed_names != expected_names + or not records_valid + or evidence.get("dataset_task_definitions_sha256") != sha256_file_set(dataset_root, task_tomls) + or evidence.get("registry_network_attempts") != 0 + or evidence.get("concurrency") != concurrency + or evidence.get("relay_architecture") != relay_architecture + or not isinstance(evidence.get("relay_runtime"), dict) + or evidence["relay_runtime"].get("status") != "passed" + or evidence["relay_runtime"].get("relay_wheel_sha256") != sha256_file(relay_wheel) + or evidence["relay_runtime"].get("switchyard_library_sha256") + != sha256_file(switchyard_library(switchyard_bundle)) + or evidence["relay_runtime"].get("plugin_config_template_sha256") != sha256_file(plugin_config_template) + ): + raise ValueError(f"Phase 2 all-task smoke evidence is not passed: {path}") + + +def validate_offline_evidence( + path: Path, + relay_architecture: str, + relay_wheel: Path, + switchyard_bundle: Path, + plugin_config_template: Path, +) -> None: + evidence = read_json(path) + if ( + evidence.get("schema_version") != "harbor-hermes-switchyard.phase2-offline-admission.v1" + or evidence.get("status") != "passed" + or evidence.get("hermes_commit") != EXPECTED_HERMES_COMMIT + or evidence.get("relay_architecture") != relay_architecture + or evidence.get("relay_wheel_sha256") != sha256_file(relay_wheel) + or evidence.get("switchyard_library_sha256") != sha256_file(switchyard_library(switchyard_bundle)) + or evidence.get("plugin_config_template_sha256") != sha256_file(plugin_config_template) + or evidence.get("provider_requests", 0) <= 0 + or evidence.get("surviving_shutdown_threads") != [] + ): + raise ValueError(f"Phase 2 offline admission evidence is not passed: {path}") + + +def probe_url(url: str, label: str, attempts: int = 3) -> None: + for attempt in range(1, attempts + 1): + request = urllib.request.Request(url, method="GET") + try: + with urllib.request.urlopen(request, timeout=10) as response: + if response.status >= 500: + raise RuntimeError(f"{label} returned HTTP {response.status}") + return + except urllib.error.HTTPError as error: + if error.code < 500: + return + failure: Exception = error + except (OSError, urllib.error.URLError) as error: + failure = error + if attempt < attempts: + time.sleep(2 ** (attempt - 1)) + raise RuntimeError(f"{label} is unreachable after {attempts} attempts: {failure}") from failure + + +def verify_provider_catalog( + base_url: str, authorization: str, required_models: list[str], attempts: int = 3 +) -> list[str]: + catalog_url = f"{base_url.rstrip('/')}/models" + for attempt in range(1, attempts + 1): + request = urllib.request.Request(catalog_url, headers={"Authorization": authorization}, method="GET") + try: + with urllib.request.urlopen(request, timeout=30) as response: + if response.status != 200: + raise RuntimeError(f"provider model catalog returned HTTP {response.status}") + payload = json.load(response) + break + except urllib.error.HTTPError as error: + if error.code < 500 and error.code != 429: + raise RuntimeError(f"provider model catalog returned HTTP {error.code}") from error + failure: Exception = error + except (OSError, ValueError, urllib.error.URLError) as error: + failure = error + if attempt < attempts: + time.sleep(2 ** (attempt - 1)) + else: + raise RuntimeError( + f"provider model catalog is unavailable or invalid after {attempts} attempts: {failure}" + ) from failure + records = payload.get("data") if isinstance(payload, dict) else None + available = { + record.get("id") for record in records or [] if isinstance(record, dict) and isinstance(record.get("id"), str) + } + missing = sorted(set(required_models) - available) + if missing: + raise RuntimeError(f"provider model catalog is missing configured model(s): {', '.join(missing)}") + return sorted(required_models) + + +def normalize_architecture(value: str) -> str: + normalized = value.strip().lower() + aliases = {"amd64": "x86_64", "x86_64": "x86_64", "arm64": "aarch64", "aarch64": "aarch64"} + if normalized not in aliases: + raise RuntimeError(f"unsupported Docker architecture: {value}") + return aliases[normalized] + + +def capacity_requirement_gb(args: argparse.Namespace, tasks: list[Task]) -> int: + parallel = args.concurrency * args.parallel_max_memory_gb + largest = max(task.memory_gb for task in tasks) + return max(parallel, largest) + args.docker_memory_reserve_gb + + +def shared_preflight(args: argparse.Namespace, tasks: list[Task]) -> dict[str, Any]: + free_bytes = shutil.disk_usage(args.run_root.parent).free + if free_bytes < args.minimum_free_gb * 1024**3: + raise RuntimeError(f"fewer than {args.minimum_free_gb} GiB remain on the run volume") + docker = json.loads( + subprocess.run( + ["docker", "info", "--format", "{{json .}}"], + check=True, + capture_output=True, + text=True, + ).stdout + ) + docker_cpus = int(docker["NCPU"]) + docker_memory_bytes = int(docker["MemTotal"]) + docker_memory_gb = docker_memory_bytes // 1024**3 + docker_architecture = normalize_architecture(str(docker["Architecture"])) + required_memory_gb = capacity_requirement_gb(args, tasks) + if docker_architecture != args.relay_architecture: + raise RuntimeError( + f"Docker architecture {docker_architecture} does not match Relay architecture {args.relay_architecture}" + ) + if args.concurrency > docker_cpus: + raise RuntimeError(f"concurrency {args.concurrency} exceeds Docker CPU count {docker_cpus}") + if required_memory_gb > docker_memory_gb: + raise RuntimeError(f"Phase 2 requires {required_memory_gb} GiB but Docker exposes {docker_memory_gb} GiB") + version = subprocess.run( + [str(args.harbor_bin), "--version"], check=True, capture_output=True, text=True + ).stdout.strip() + if version != "0.18.0": + raise RuntimeError(f"Harbor 0.18.0 is required; {args.harbor_bin} reports {version}") + python_version = subprocess.run( + [ + str(args.python_bin), + "-c", + 'import importlib.metadata; print(importlib.metadata.version("harbor"))', + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if python_version != "0.18.0": + raise RuntimeError(f"Harbor Python 0.18.0 is required; {args.python_bin} provides {python_version}") + probe_url(args.phoenix_url, "Phoenix") + provider_authorization = os.environ.get("SWITCHYARD_PROVIDER_AUTHORIZATION") + if not provider_authorization: + raise RuntimeError("SWITCHYARD_PROVIDER_AUTHORIZATION is unset") + verified_models: set[str] = set() + for provider_url in args.plugin_contract["provider_base_urls"]: + verified_models.update( + verify_provider_catalog( + provider_url, + provider_authorization, + args.plugin_contract["catalog_models"], + ) + ) + if not args.switchyard_bundle.is_dir(): + raise RuntimeError(f"Switchyard bundle is missing: {args.switchyard_bundle}") + if not args.relay_wheel.is_file(): + raise RuntimeError(f"Relay wheel is missing: {args.relay_wheel}") + return { + "harbor_version": version, + "minimum_free_gb": args.minimum_free_gb, + "disk_free_gb": free_bytes // 1024**3, + "concurrency": args.concurrency, + "parallel_task_memory_gb": args.parallel_max_memory_gb, + "largest_task_memory_gb": max(task.memory_gb for task in tasks), + "docker_memory_reserve_gb": args.docker_memory_reserve_gb, + "required_docker_memory_gb": required_memory_gb, + "docker_memory_gb": docker_memory_gb, + "docker_cpus": docker_cpus, + "docker_architecture": docker_architecture, + "relay_architecture": args.relay_architecture, + "provider_endpoints": args.plugin_contract["provider_base_urls"], + "provider_catalog_models_verified": sorted(verified_models), + "phoenix_endpoint": args.phoenix_url, + "phoenix_reachable": True, + "provider_reachable": True, + "docker_healthy": True, + } + + +def make_plan(args: argparse.Namespace, tasks: list[Task]) -> dict[str, Any]: + manifest = args.switchyard_bundle / "relay-plugin.toml" + library_candidates = sorted(args.switchyard_bundle.glob("libswitchyard_nemo_relay_plugin.*")) + if not manifest.is_file() or len(library_candidates) != 1: + raise ValueError("Switchyard bundle must contain one manifest and one native library") + example_root = args.task_runner.parent + runtime_sources = [ + args.task_runner, + example_root / "run_phase2_cohort.sh", + example_root / "supervise_phase2_cohort.sh", + ] + for relative in ("agents", "config", "scripts"): + runtime_sources.extend( + path + for path in (example_root / relative).rglob("*") + if path.is_file() and path.suffix in {".py", ".sh", ".toml", ".yaml"} + ) + task_definitions = [args.dataset_root / task.name / "task.toml" for task in tasks] + return { + "schema_version": PLAN_SCHEMA_VERSION, + "dataset": args.dataset, + "dataset_root": str(args.dataset_root.resolve()), + "sample_count": args.sample_count, + "canary_task": args.canary_task, + "concurrency": args.concurrency, + "setup_concurrency": args.setup_concurrency, + "setup_batch_size": args.setup_batch_size, + "setup_max_infra_attempts": args.setup_max_infra_attempts, + "parallel_max_memory_gb": args.parallel_max_memory_gb, + "docker_memory_reserve_gb": args.docker_memory_reserve_gb, + "minimum_free_gb": args.minimum_free_gb, + "phoenix_project": args.phoenix_project, + "evaluation_cohort": args.eval_cohort, + "timeout_multipliers": {"agent": 3, "agent_setup": 6, "environment_build": 6}, + "required_models": args.plugin_contract["required_models"], + "require_cache_hit": args.require_cache_hit, + "routing": { + "strong_model": args.plugin_contract["strong_model"], + "weak_model": args.plugin_contract["weak_model"], + "hermes_caller_model": args.plugin_contract["hermes_caller_model"], + "provider_base_urls": args.plugin_contract["provider_base_urls"], + }, + "inputs": { + "runner_sha256": sha256_file(args.task_runner), + "runtime_sources_sha256": sha256_file_set(example_root, runtime_sources), + "dataset_task_definitions_sha256": sha256_file_set(args.dataset_root, task_definitions), + "phase2_smoke_evidence_sha256": sha256_file(args.smoke_evidence), + "phase2_offline_evidence_sha256": sha256_file(args.offline_evidence), + "plugin_config_template_sha256": sha256_file(args.plugin_config_template), + "relay_wheel_sha256": sha256_file(args.relay_wheel), + "switchyard_manifest_sha256": sha256_file(manifest), + "switchyard_library_sha256": sha256_file(library_candidates[0]), + "hermetic_runtime_sha256": args.hermetic_runtime_payload["content_sha256"], + "setup_runtime_provenance_sha256": sha256_file(args.setup_runtime / "provenance.json"), + }, + "tasks": [task.as_json() for task in tasks], + } + + +def load_or_create_plan(path: Path, plan: dict[str, Any]) -> None: + if path.is_file(): + existing = read_json(path) + if existing != plan: + raise ValueError(f"existing Phase 2 plan does not match requested configuration: {path}") + return + write_json(path, plan) + + +def task_record(task: Task, attempt: Path | None) -> dict[str, Any]: + record: dict[str, Any] = task.as_json() + record["status"] = "pending" + record["attempt_count"] = 0 + task_root = attempt.parents[1] if attempt else None + if task_root: + record["attempt_count"] = len([path for path in (task_root / "attempts").iterdir() if path.is_dir()]) + if not attempt: + return record + summary = read_json(attempt / "summary.json") + validation = summary["validation"] + upload = summary["phoenix_upload"] + integration = validation.get("integration", {"status": validation.get("status"), "errors": []}) + record.update( + { + "status": "completed", + "successful_attempt": attempt.name, + "attempt_root": str(attempt), + "benchmark_task_passed": validation.get("benchmark_task_passed"), + "benchmark_completion": validation.get("benchmark", {"status": validation.get("status")}), + "integration_validation": integration, + "direct_result_status": validation.get("direct_result_status"), + "switchyard_decision_count": validation.get("switchyard_decision_count", 0), + "routed_models": validation.get("routed_models", []), + "routed_targets": validation.get("routed_targets", []), + "cache_read_tokens": validation.get("cache_read_tokens", 0), + "cache_write_tokens": validation.get("cache_write_tokens", 0), + "secret_findings": validation.get("secret_findings", []), + "uploaded_spans": upload.get("uploaded_spans", 0), + "phoenix_upload": upload.get("status"), + } + ) + return record + + +def aggregate_summary(args: argparse.Namespace, tasks: list[Task]) -> dict[str, Any]: + records = [] + for task in tasks: + task_root = args.run_root / "tasks" / task.directory_name + records.append(task_record(task, successful_attempt(task_root))) + complete = all(record["status"] == "completed" for record in records) + cache_read_tokens = sum(int(record.get("cache_read_tokens") or 0) for record in records) + observed_models = sorted( + {model for record in records for model in record.get("routed_models", []) if isinstance(model, str)} + ) + missing_models = sorted(set(args.required_model).difference(observed_models)) + completed_records = [record for record in records if record["status"] == "completed"] + secrets_clean = all(not record.get("secret_findings") for record in completed_records) + integration_failures = [ + { + "task": record["name"], + "errors": record.get("integration_validation", {}).get("errors", []), + } + for record in completed_records + if record.get("integration_validation", {}).get("status") != "passed" + ] + gates = { + "task_outputs": {"passed": complete, "completed": sum(r["status"] == "completed" for r in records)}, + "integration_validation": { + "passed": not integration_failures, + "failed_task_count": len(integration_failures), + "failures": integration_failures, + }, + "cache_hit": { + "required": args.require_cache_hit, + "passed": not args.require_cache_hit or cache_read_tokens > 0, + "cache_read_tokens": cache_read_tokens, + }, + "route_diversity": { + "required_models": sorted(args.required_model), + "observed_models": observed_models, + "missing_models": missing_models, + "passed": not missing_models, + }, + "secret_scan": {"passed": secrets_clean}, + } + passed = complete and all(gate["passed"] for gate in gates.values()) + return { + "schema_version": SCHEMA_VERSION, + "status": "passed" if passed else "partial", + "dataset": args.dataset, + "phoenix_project": args.phoenix_project, + "evaluation_cohort": args.eval_cohort, + "planned_tasks": len(records), + "completed_tasks": sum(record["status"] == "completed" for record in records), + "benchmark_pass_count": sum(record.get("benchmark_task_passed") is True for record in records), + "benchmark_nonpass_count": sum(record.get("benchmark_task_passed") is False for record in records), + "uploaded_spans": sum(int(record.get("uploaded_spans") or 0) for record in records), + "cohort_gates": gates, + "tasks": records, + } + + +def write_report(root: Path, summary: dict[str, Any]) -> None: + gates = summary["cohort_gates"] + lines = [ + "# Harbor + Hermes + Switchyard Phase 2 cohort", + "", + f"- Status: `{summary['status']}`", + f"- Completed: {summary['completed_tasks']}/{summary['planned_tasks']}", + f"- Benchmark pass/non-pass: {summary['benchmark_pass_count']}/{summary['benchmark_nonpass_count']}", + f"- Uploaded spans: {summary['uploaded_spans']}", + f"- Cache-read tokens: {gates['cache_hit']['cache_read_tokens']}", + f"- Observed provider models: {', '.join(gates['route_diversity']['observed_models']) or 'none'}", + "", + "| # | Task | Memory | Evidence | Benchmark | Attempts | Spans |", + "|---:|---|---:|---|---|---:|---:|", + ] + for task in summary["tasks"]: + benchmark = task.get("benchmark_task_passed") + benchmark_text = "pass" if benchmark is True else "non-pass" if benchmark is False else "pending" + lines.append( + f"| {task['index']:03d} | `{task['name']}` | {task['memory_gb']}G | {task['status']} | " + f"{benchmark_text} | {task['attempt_count']} | {task.get('uploaded_spans', 0)} |" + ) + (root / "report.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + + +class CohortRunner: + def __init__(self, args: argparse.Namespace, tasks: list[Task]): + self.args = args + self.tasks = tasks + self.stop_scheduling = asyncio.Event() + self.summary_lock = asyncio.Lock() + + async def refresh_summary(self) -> None: + async with self.summary_lock: + summary = aggregate_summary(self.args, self.tasks) + write_json(self.args.run_root / "summary.json", summary) + write_report(self.args.run_root, summary) + + def provision_environments(self) -> bool: + output = self.args.run_root / "setup-admission" + command = [ + str(self.args.python_bin), + str(self.args.setup_admission_runner), + "--dataset", + str(self.args.dataset_root), + "--runtime-root", + str(self.args.setup_runtime), + "--hermetic-runtime", + str(self.args.hermetic_runtime), + "--output", + str(output), + "--harbor", + str(self.args.harbor_bin), + "--concurrency", + str(self.args.setup_concurrency), + "--batch-size", + str(self.args.setup_batch_size), + "--max-infra-attempts", + str(self.args.setup_max_infra_attempts), + "--backoff-seconds", + str(self.args.backoff_seconds), + "--force-build", + "--no-preserve-containers", + ] + log_path = self.args.run_root / "setup-admission-coordinator.log" + with log_path.open("a", encoding="utf-8") as log: + process = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT) + summary_path = output / "summary.json" + summary = read_json(summary_path) if summary_path.is_file() else {} + passed = ( + process.returncode == 0 + and summary.get("status") == "passed" + and summary.get("planned") == len(self.tasks) + and summary.get("passed") == len(self.tasks) + ) + write_json( + self.args.run_root / "setup-state.json", + { + "schema_version": "harbor-hermes-switchyard.phase2-setup-state.v1", + "status": "passed" if passed else "failed", + "failure_class": ( + None + if passed + else "harness_or_integration" + if process.returncode == 20 or summary.get("integration_failures") + else "infrastructure" + ), + "exit_code": process.returncode, + "summary": summary, + "hermetic_runtime_sha256": self.args.hermetic_runtime_payload["content_sha256"], + "force_build": True, + "setup_concurrency": self.args.setup_concurrency, + }, + ) + return passed + + async def run_attempt(self, task: Task, attempt_number: int) -> tuple[int, Path, str]: + task_root = self.args.run_root / "tasks" / task.directory_name + attempts_root = task_root / "attempts" + attempts_root.mkdir(mode=0o700, parents=True, exist_ok=True) + attempt = attempts_root / f"{attempt_number:03d}" + log_path = task_root / f"attempt-{attempt_number:03d}.log" + if attempt.exists(): + raise RuntimeError(f"attempt root already exists: {attempt}") + env = os.environ.copy() + env.update( + { + "TASK_NAME": task.name, + "EVAL_PHASE": "phase2", + "TBENCH_DATASET_PATH": str(self.args.dataset_root), + "PHOENIX_PROJECT": self.args.phoenix_project, + "EVAL_COHORT": self.args.eval_cohort, + "PHOENIX_BASE_URL": self.args.phoenix_url, + "SWITCHYARD_BUNDLE": str(self.args.switchyard_bundle), + "RELAY_WHEEL": str(self.args.relay_wheel), + "RELAY_ARCHITECTURE": self.args.relay_architecture, + "PLUGIN_CONFIG_TEMPLATE": str(self.args.plugin_config_template), + "HARBOR_BIN": str(self.args.harbor_bin), + "EVAL_PYTHON": str(self.args.python_bin), + "AGENT_TIMEOUT_MULTIPLIER": "3", + "AGENT_SETUP_TIMEOUT_MULTIPLIER": "6", + "ENVIRONMENT_BUILD_TIMEOUT_MULTIPLIER": "6", + "HERMETIC_RUNTIME_DIR": str(self.args.hermetic_runtime), + "HERMETIC_RUNTIME_SHA256": self.args.hermetic_runtime_payload["content_sha256"], + # Harbor assigns trial-specific local image tags. Rebuild from the + # pinned task Dockerfile so an incompatible published prebuilt image + # cannot replace the architecture-validated setup-admission image. + # The setup lane has already populated Docker's layer cache. + "HARBOR_FORCE_BUILD": "true", + } + ) + with log_path.open("wb") as log: + process = await asyncio.create_subprocess_exec( + str(self.args.task_runner), str(attempt), env=env, stdout=log, stderr=asyncio.subprocess.STDOUT + ) + status = await process.wait() + log_text = log_path.read_text(encoding="utf-8", errors="replace") + return status, attempt, log_text + + async def run_task(self, task: Task) -> bool: + task_root = self.args.run_root / "tasks" / task.directory_name + passed = successful_attempt(task_root) + if passed: + print(f"[phase2] already complete: {task.directory_name}", flush=True) + return True + state_path = task_root / "task-state.json" + if state_path.is_file(): + state = read_json(state_path) + if state.get("status") == "failed" and state.get("failure_class") == "harness_or_integration": + print(f"[phase2] preserved integration blocker: {task.directory_name}", flush=True) + self.stop_scheduling.set() + return False + attempts_root = task_root / "attempts" + existing_attempts = len(list(attempts_root.glob("[0-9][0-9][0-9]"))) + attempt_number = existing_attempts + 1 + infrastructure_failures = 0 + retry_preflight_failures = 0 + needs_retry_preflight = False + while infrastructure_failures < self.args.max_infra_attempts: + if self.stop_scheduling.is_set(): + return False + if needs_retry_preflight: + try: + preflight = shared_preflight(self.args, self.tasks) + except (OSError, RuntimeError, subprocess.SubprocessError, ValueError) as error: + retry_preflight_failures += 1 + write_json( + state_path, + { + "schema_version": TASK_STATE_SCHEMA_VERSION, + "status": "waiting_for_infrastructure", + "task": task.as_json(), + "failure_class": "infrastructure", + "preflight_error": str(error), + "infrastructure_failure_count": infrastructure_failures, + "retry_preflight_failure_count": retry_preflight_failures, + }, + ) + print( + f"[phase2] retry preflight blocked {task.directory_name} " + f"({retry_preflight_failures}/{self.args.max_infra_attempts}): {error}", + flush=True, + ) + await self.refresh_summary() + if retry_preflight_failures >= self.args.max_infra_attempts: + self.stop_scheduling.set() + return False + await asyncio.sleep(self.args.backoff_seconds * (2 ** (retry_preflight_failures - 1))) + continue + write_json(self.args.run_root / "preflight.json", {"status": "passed", **preflight}) + needs_retry_preflight = False + retry_preflight_failures = 0 + print(f"[phase2] starting {task.directory_name} attempt {attempt_number:03d}", flush=True) + status, attempt, log_text = await self.run_attempt(task, attempt_number) + if status == 0 and task_summary_passed(attempt / "summary.json"): + write_json( + state_path, + { + "schema_version": TASK_STATE_SCHEMA_VERSION, + "status": "passed", + "task": task.as_json(), + "successful_attempt": attempt.name, + }, + ) + await self.refresh_summary() + print(f"[phase2] completed {task.directory_name} attempt {attempt_number:03d}", flush=True) + return True + failure_class = classify_attempt_failure(log_text, attempt) + write_json( + state_path, + { + "schema_version": TASK_STATE_SCHEMA_VERSION, + "status": "failed", + "task": task.as_json(), + "latest_attempt": attempt.name, + "failure_class": failure_class, + "exit_code": status, + }, + ) + await self.refresh_summary() + print( + f"[phase2] failed {task.directory_name} attempt {attempt_number:03d}: {failure_class}", + flush=True, + ) + if failure_class != "infrastructure": + self.stop_scheduling.set() + return False + infrastructure_failures += 1 + attempt_number += 1 + needs_retry_preflight = True + if infrastructure_failures < self.args.max_infra_attempts: + await asyncio.sleep(self.args.backoff_seconds * (2 ** (infrastructure_failures - 1))) + self.stop_scheduling.set() + return False + + async def run_parallel_lane(self, tasks: list[Task]) -> bool: + semaphore = asyncio.Semaphore(self.args.concurrency) + + async def guarded(task: Task) -> bool: + async with semaphore: + if self.stop_scheduling.is_set(): + return False + return await self.run_task(task) + + results = await asyncio.gather(*(guarded(task) for task in tasks)) + return all(results) + + async def run(self) -> bool: + preflight = shared_preflight(self.args, self.tasks) + write_json(self.args.run_root / "preflight.json", {"status": "passed", **preflight}) + await self.refresh_summary() + if not self.provision_environments(): + return False + remaining = self.tasks + if self.args.canary_task: + first = self.tasks[0] + if not await self.run_task(first): + return False + remaining = self.tasks[1:] + parallel = [task for task in remaining if task.memory_gb <= self.args.parallel_max_memory_gb] + serial = [task for task in remaining if task.memory_gb > self.args.parallel_max_memory_gb] + if not await self.run_parallel_lane(parallel): + return False + for task in serial: + if not await self.run_task(task): + return False + await self.refresh_summary() + return aggregate_summary(self.args, self.tasks)["status"] == "passed" + + +def parse_args() -> argparse.Namespace: + example_root = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--dataset", default="terminal-bench@2.0") + parser.add_argument("--dataset-root", type=Path, required=True) + parser.add_argument("--sample-count", type=int, default=89) + parser.add_argument("--exclude-task", action="append", default=[]) + parser.add_argument( + "--canary-task", + default="", + help="task to run alone before the cohort; pass an empty value to disable the canary", + ) + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--setup-concurrency", type=int, default=2) + parser.add_argument("--setup-batch-size", type=int, default=89) + parser.add_argument("--setup-max-infra-attempts", type=int, default=4) + parser.add_argument("--parallel-max-memory-gb", type=int, default=2) + parser.add_argument("--docker-memory-reserve-gb", type=int, default=4) + parser.add_argument("--max-infra-attempts", type=int, default=3) + parser.add_argument("--backoff-seconds", type=float, default=30) + parser.add_argument("--minimum-free-gb", type=int, default=100) + parser.add_argument("--smoke-evidence", type=Path, required=True) + parser.add_argument("--offline-evidence", type=Path, required=True) + parser.add_argument("--plugin-config-template", type=Path, required=True) + parser.add_argument("--task-runner", type=Path, default=example_root / "run_terminal_bench.sh") + parser.add_argument( + "--setup-admission-runner", + type=Path, + default=example_root / "scripts" / "run_setup_admission.py", + ) + parser.add_argument( + "--hermetic-runtime-builder", + type=Path, + default=example_root / "scripts" / "build_hermetic_runtime.py", + ) + parser.add_argument( + "--runtime-preparer", + type=Path, + default=example_root / "scripts" / "prepare_runtime.py", + ) + parser.add_argument( + "--bootstrap-root", + type=Path, + help="shared content-addressed bootstrap cache; generated automatically when absent", + ) + parser.add_argument("--harbor-bin", type=Path, default=example_root / ".venv" / "bin" / "harbor") + parser.add_argument("--python-bin", type=Path, default=example_root / ".venv" / "bin" / "python") + parser.add_argument("--phoenix-url", required=True) + parser.add_argument("--phoenix-project", required=True) + parser.add_argument("--eval-cohort", required=True) + parser.add_argument("--switchyard-bundle", type=Path, required=True) + parser.add_argument("--relay-wheel", type=Path, required=True) + parser.add_argument("--relay-architecture", choices=("x86_64", "aarch64"), default="x86_64") + parser.add_argument("--require-cache-hit", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--plan-only", action="store_true") + parser.add_argument("--preflight-only", action="store_true") + args = parser.parse_args() + for name in ( + "sample_count", + "concurrency", + "setup_concurrency", + "setup_batch_size", + "setup_max_infra_attempts", + "parallel_max_memory_gb", + "docker_memory_reserve_gb", + "max_infra_attempts", + "minimum_free_gb", + ): + if getattr(args, name) <= 0: + parser.error(f"--{name.replace('_', '-')} must be positive") + if not args.run_root.is_absolute(): + parser.error("--run-root must be absolute") + args.bootstrap_root = ( + (args.bootstrap_root or args.run_root.parent / "harbor-hermes-switchyard-bootstrap").expanduser().resolve() + ) + if args.plan_only and args.preflight_only: + parser.error("--plan-only and --preflight-only are mutually exclusive") + return args + + +def main() -> int: + args = parse_args() + args.plugin_contract = plugin_contract(args.plugin_config_template) + args.required_model = args.plugin_contract["required_models"] + validate_smoke_evidence( + args.smoke_evidence, + args.sample_count, + args.dataset_root, + args.concurrency, + args.relay_architecture, + args.relay_wheel, + args.switchyard_bundle, + args.plugin_config_template, + ) + validate_offline_evidence( + args.offline_evidence, + args.relay_architecture, + args.relay_wheel, + args.switchyard_bundle, + args.plugin_config_template, + ) + args.canary_task = args.canary_task or None + tasks = discover_tasks(args.dataset_root, args.sample_count, set(args.exclude_task), args.canary_task) + args.run_root.mkdir(mode=0o700, parents=True, exist_ok=True) + lock_path = args.run_root / ".phase2.lock" + with lock_path.open("a+") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise SystemExit(f"another Phase 2 supervisor owns {args.run_root}") from None + bootstrap_preflight(args) + args.hermetic_runtime, args.hermetic_runtime_payload = ensure_hermetic_runtime(args) + args.setup_runtime = prepare_setup_runtime(args) + plan = make_plan(args, tasks) + load_or_create_plan(args.run_root / "plan.json", plan) + if args.plan_only: + summary = aggregate_summary(args, tasks) + write_json(args.run_root / "summary.json", summary) + write_report(args.run_root, summary) + print(json.dumps(plan, indent=2)) + return 0 + if args.preflight_only: + preflight = shared_preflight(args, tasks) + write_json(args.run_root / "preflight.json", {"status": "passed", **preflight}) + summary = aggregate_summary(args, tasks) + write_json(args.run_root / "summary.json", summary) + write_report(args.run_root, summary) + print(json.dumps({"status": "passed", "preflight": preflight}, indent=2)) + return 0 + passed = asyncio.run(CohortRunner(args, tasks).run()) + summary = aggregate_summary(args, tasks) + print(json.dumps(summary, indent=2)) + if passed: + return 0 + if ( + summary["completed_tasks"] == summary["planned_tasks"] + and not summary["cohort_gates"]["integration_validation"]["passed"] + ): + return 20 + states = [read_json(path) for path in (args.run_root / "tasks").glob("*/task-state.json") if path.is_file()] + setup_state = args.run_root / "setup-state.json" + if setup_state.is_file(): + states.append(read_json(setup_state)) + if any(state.get("failure_class") == "harness_or_integration" for state in states): + return 20 + return 75 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh b/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh new file mode 100755 index 000000000..04b8be508 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail +set +x + +example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +env_file="${TERMINAL_BENCH_ENV_FILE:-${1:-}}" +if [[ -z "$env_file" || ! -f "$env_file" ]]; then + echo "TERMINAL_BENCH_ENV_FILE must reference an existing environment file" >&2 + exit 2 +fi +env_file="$(cd "$(dirname "$env_file")" && pwd)/$(basename "$env_file")" + +"$example_root/scripts/validate_phase2_environment.sh" "$env_file" +set -a +# shellcheck disable=SC1090 +source "$env_file" +set +a +set +x + +mkdir -p "$TERMINAL_BENCH_RUN_ROOT" +chmod 0700 "$TERMINAL_BENCH_RUN_ROOT" +runtime_harness="$TERMINAL_BENCH_RUN_ROOT/runtime-harness" +"$EVAL_PYTHON" "$example_root/scripts/stage_phase2_runtime.py" \ + --source "$example_root" \ + --destination "$runtime_harness" \ + --plan "$TERMINAL_BENCH_RUN_ROOT/plan.json" +exec "$runtime_harness/supervise_phase2_cohort.sh" "$TERMINAL_BENCH_RUN_ROOT" \ + >>"$TERMINAL_BENCH_RUN_ROOT/supervisor.log" 2>&1 diff --git a/examples/harbor-hermes-switchyard/scripts/run_setup_admission.py b/examples/harbor-hermes-switchyard/scripts/run_setup_admission.py new file mode 100755 index 000000000..b87255c81 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/run_setup_admission.py @@ -0,0 +1,649 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Provision task environments through Harbor's provider-free install lifecycle. + +The Phase 2 coordinator and the standalone diagnostic share this implementation. +It stops before agent execution and the verifier, uses Harbor's real Docker and +agent setup paths, reuses successful task evidence by content hash, and never +loads provider authorization. +""" + +from __future__ import annotations + +import argparse +import email.utils +import hashlib +import json +import os +import subprocess +import time +import urllib.request +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from harbor import __version__ as harbor_version +from harbor.models.task.task import Task + +PLAN_SCHEMA = "harbor-hermes-switchyard.setup-admission-plan.v2" +RESULT_SCHEMA = "harbor-hermes-switchyard.setup-admission-task.v2" +SUMMARY_SCHEMA = "harbor-hermes-switchyard.setup-admission-summary.v2" +PAYLOAD_SCHEMA = "harbor-hermes-switchyard.hermetic-runtime.v1" +CLOCK_PREFLIGHT_SCHEMA = "harbor-hermes-switchyard.clock-preflight.v1" +CLOCK_REFERENCE_URL = "https://deb.debian.org/debian-security/dists/bookworm-security/InRelease" +CLOCK_MAX_OFFSET_SECONDS = 300 +CLOCK_PROBE_IMAGE = "python:3.11-bullseye" +PLUGIN_COMPATIBILITY_SCHEMA = "harbor-hermes-switchyard.plugin-compatibility.v1" +SETUP_AGENT_PATH = Path(__file__).resolve().parents[1] / "agents" / "harbor_hermes_agent.py" +INFRASTRUCTURE_PATTERNS = ( + "apt-get update", + "connection refused", + "connection reset", + "connection timed out", + "context deadline exceeded", + "docker build failed", + "failed to resolve source metadata", + "failed to authorize", + "i/o timeout", + "network is unreachable", + "no space left on device", + "registry-1.docker.io", + "temporary failure in name resolution", + "remote end closed connection", + "tls handshake timeout", + "too many requests", + "unexpected eof", +) + + +def classify_setup_failure(trial_root: Path, diagnostic: str) -> str: + lowered = diagnostic.lower() + if any(pattern in lowered for pattern in INFRASTRUCTURE_PATTERNS): + return "infrastructure" + for path in sorted(trial_root.rglob("*.log")): + try: + text = path.read_text(encoding="utf-8", errors="replace").lower() + except OSError: + continue + if any(pattern in text for pattern in INFRASTRUCTURE_PATTERNS): + return "infrastructure" + return "harness_or_integration" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def sha256_tree(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(candidate for candidate in root.rglob("*") if candidate.is_file()): + relative = path.relative_to(root).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(4, "big")) + digest.update(relative) + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def hermetic_content_sha256(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(candidate for candidate in root.rglob("*") if candidate.is_file()): + if path.name == "payload.json": + continue + relative = path.relative_to(root).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(4, "big")) + digest.update(relative) + digest.update(path.stat().st_mode.to_bytes(4, "big")) + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def canonical_sha256(value: Any) -> str: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def write_json(path: Path, value: Any) -> None: + temporary = path.with_suffix(f"{path.suffix}.tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(path) + + +def evaluate_clock_preflight(*, host_epoch: float, docker_epoch: float, reference_epoch: float) -> dict[str, Any]: + host_reference_offset = round(reference_epoch - host_epoch, 3) + docker_host_offset = round(docker_epoch - host_epoch, 3) + passed = ( + abs(host_reference_offset) <= CLOCK_MAX_OFFSET_SECONDS and abs(docker_host_offset) <= CLOCK_MAX_OFFSET_SECONDS + ) + return { + "schema_version": CLOCK_PREFLIGHT_SCHEMA, + "status": "passed" if passed else "failed", + "checked_at": datetime.now(UTC).isoformat(), + "reference_url": CLOCK_REFERENCE_URL, + "maximum_offset_seconds": CLOCK_MAX_OFFSET_SECONDS, + "host_reference_offset_seconds": host_reference_offset, + "docker_host_offset_seconds": docker_host_offset, + } + + +def run_clock_preflight() -> dict[str, Any]: + date_header: str | None = None + for attempt in range(1, 5): + request = urllib.request.Request( + CLOCK_REFERENCE_URL, + method="HEAD", + headers={"User-Agent": "harbor-hermes-switchyard-setup-admission/2"}, + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + date_header = response.headers.get("Date") + break + except OSError: + if attempt == 4: + raise + time.sleep(2 ** (attempt - 1)) + if not date_header: + raise RuntimeError("clock reference response omitted its Date header") + reference_datetime = email.utils.parsedate_to_datetime(date_header) + docker_output = "" + for attempt in range(1, 5): + try: + docker_output = subprocess.check_output( + [ + "docker", + "run", + "--rm", + "--pull=missing", + CLOCK_PROBE_IMAGE, + "date", + "+%s", + ], + text=True, + ).strip() + break + except subprocess.CalledProcessError: + if attempt == 4: + raise + time.sleep(2 ** (attempt - 1)) + return evaluate_clock_preflight( + host_epoch=time.time(), + docker_epoch=float(docker_output), + reference_epoch=reference_datetime.timestamp(), + ) + + +def plugin_compatibility_command(plan: dict[str, Any]) -> list[str]: + inputs = plan["inputs"] + platform = { + "aarch64": "linux/arm64", + "x86_64": "linux/amd64", + }[inputs["relay_architecture"]] + return [ + "docker", + "run", + "--rm", + "--pull=missing", + "--platform", + platform, + "--volume", + f"{inputs['switchyard_bundle']}:/bundle:ro", + CLOCK_PROBE_IMAGE, + "python", + "-c", + ( + "import ctypes; " + "library=ctypes.CDLL('/bundle/libswitchyard_nemo_relay_plugin.so'); " + "getattr(library, 'nemo_relay_register_plugin')" + ), + ] + + +def run_plugin_compatibility_preflight(plan: dict[str, Any]) -> dict[str, Any]: + for attempt in range(1, 5): + process = subprocess.run( + plugin_compatibility_command(plan), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=120, + ) + if process.returncode == 0 or attempt == 4: + break + time.sleep(2 ** (attempt - 1)) + inputs = plan["inputs"] + return { + "schema_version": PLUGIN_COMPATIBILITY_SCHEMA, + "status": "passed" if process.returncode == 0 else "failed", + "checked_at": datetime.now(UTC).isoformat(), + "probe_image": CLOCK_PROBE_IMAGE, + "relay_architecture": inputs["relay_architecture"], + "switchyard_library_sha256": inputs["switchyard_library_sha256"], + "registration_symbol": "nemo_relay_register_plugin", + "exit_code": process.returncode, + } + + +def load_payload(path: Path) -> dict[str, Any]: + marker = path / "payload.json" + payload = json.loads(marker.read_text(encoding="utf-8")) + if payload.get("schema_version") != PAYLOAD_SCHEMA or payload.get("status") != "passed": + raise ValueError(f"invalid hermetic runtime marker: {marker}") + observed = hermetic_content_sha256(path) + if payload.get("content_sha256") != observed: + raise ValueError( + "hermetic runtime content does not match its marker: " + f"expected {payload.get('content_sha256')}, observed {observed}" + ) + return payload + + +def discover_tasks(dataset: Path) -> list[Task]: + tasks = [Task(path) for path in sorted(dataset.iterdir()) if Task.is_valid_dir(path)] + names = [task.name for task in tasks] + if len(tasks) != 89 or len(set(names)) != 89: + raise ValueError(f"expected 89 unique tasks, found {len(tasks)} tasks and {len(set(names))} names") + return sorted(tasks, key=lambda task: task.name) + + +def task_record(task: Task) -> dict[str, Any]: + task_dir = task.task_dir + environment = task.paths.environment_dir + tests = task.paths.tests_dir + record = { + "name": task.name, + "task_dir": str(task_dir), + "task_sha256": sha256_tree(task_dir), + "environment_sha256": sha256_tree(environment), + "instruction_sha256": sha256_file(task.paths.instruction_path), + "verifier_sha256": sha256_tree(tests), + "docker_image": task.config.environment.docker_image, + "cpus": task.config.environment.cpus, + "memory_mb": task.config.environment.memory_mb, + "build_timeout_sec": task.config.environment.build_timeout_sec, + } + return record + + +def build_plan(args: argparse.Namespace, tasks: list[Task], payload: dict[str, Any]) -> dict[str, Any]: + runtime = args.runtime_root + provenance_path = runtime / "provenance.json" + provenance = json.loads(provenance_path.read_text(encoding="utf-8")) + relay_wheels = sorted((runtime / "wheels").glob("nemo_relay-*.whl")) + if len(relay_wheels) != 1: + raise ValueError(f"expected one Relay wheel below {runtime / 'wheels'}") + libraries = sorted((runtime / "switchyard-plugin").glob("*.so")) + if len(libraries) != 1: + raise ValueError("expected one Linux Switchyard library") + records = [task_record(task) for task in tasks] + inputs = { + "dataset_root": str(args.dataset), + "dataset_sha256": canonical_sha256( + [{key: value for key, value in record.items() if key != "task_dir"} for record in records] + ), + "hermetic_runtime_root": str(args.hermetic_runtime), + "hermetic_runtime_sha256": payload["content_sha256"], + "hermes_commit": payload["hermes_commit"], + "relay_architecture": payload["relay_architecture"], + "relay_wheel": str(relay_wheels[0]), + "relay_wheel_sha256": sha256_file(relay_wheels[0]), + "relay_config": str(runtime / "plugins.toml"), + "relay_config_sha256": sha256_file(runtime / "plugins.toml"), + "switchyard_bundle": str(runtime / "switchyard-plugin"), + "switchyard_library_sha256": sha256_file(libraries[0]), + "runtime_provenance_sha256": sha256_file(provenance_path), + "runtime_provenance_schema": provenance.get("schema_version"), + "setup_agent_sha256": sha256_file(SETUP_AGENT_PATH), + "harbor_version": harbor_version, + "concurrency": args.concurrency, + "batch_size": args.batch_size, + "maximum_infrastructure_attempts": args.max_infra_attempts, + "force_build": args.force_build, + "preserve_containers": args.preserve_containers, + } + if inputs["relay_wheel_sha256"] != payload["relay_wheel_sha256"]: + raise ValueError("prepared runtime Relay wheel does not match hermetic runtime") + if inputs["relay_architecture"] != provenance["nemo_relay"]["architecture"]: + raise ValueError("prepared runtime architecture does not match hermetic runtime") + return { + "schema_version": PLAN_SCHEMA, + "status": "planned", + "created_at": datetime.now(UTC).isoformat(), + "inputs": inputs, + "tasks": records, + } + + +def result_path(root: Path, task_name: str) -> Path: + return root / "task-results" / f"{task_name}.json" + + +def completed_names(root: Path, plan: dict[str, Any]) -> set[str]: + bindings = { + task["name"]: canonical_sha256( + { + "task": task, + "inputs": plan["inputs"], + } + ) + for task in plan["tasks"] + } + completed: set[str] = set() + for name, binding in bindings.items(): + path = result_path(root, name) + if not path.is_file(): + continue + result = json.loads(path.read_text(encoding="utf-8")) + if ( + result.get("schema_version") == RESULT_SCHEMA + and result.get("status") == "passed" + and result.get("binding_sha256") == binding + ): + completed.add(name) + return completed + + +def task_bindings(plan: dict[str, Any]) -> dict[str, str]: + return {task["name"]: canonical_sha256({"task": task, "inputs": plan["inputs"]}) for task in plan["tasks"]} + + +def parse_job_results(root: Path, plan: dict[str, Any]) -> None: + bindings = task_bindings(plan) + known = set(bindings) + for path in sorted((root / "jobs").glob("*/*/result.json")): + result = json.loads(path.read_text(encoding="utf-8")) + task_name = result.get("task_name") + if task_name not in known: + continue + exception = result.get("exception_info") + environment = result.get("environment_setup") + setup = result.get("agent_setup") + execution = result.get("agent_execution") + verifier = result.get("verifier") + passed = ( + exception is None + and isinstance(environment, dict) + and environment.get("finished_at") + and isinstance(setup, dict) + and setup.get("finished_at") + and execution is None + and verifier is None + ) + output = { + "schema_version": RESULT_SCHEMA, + "status": "passed" if passed else "failed", + "task_name": task_name, + "binding_sha256": bindings[task_name], + "trial_result": str(path.relative_to(root)), + "environment_setup": environment, + "agent_setup": setup, + "agent_execution_skipped": execution is None, + "verifier_skipped": verifier is None, + "exception_type": exception.get("exception_type") if isinstance(exception, dict) else None, + "exception_message": exception.get("exception_message") if isinstance(exception, dict) else None, + } + diagnostic = " ".join(str(value or "") for value in (output["exception_type"], output["exception_message"])) + output["failure_class"] = None if passed else classify_setup_failure(path.parent, diagnostic) + destination = result_path(root, task_name) + # Paths are scanned in job-name order, so always replacing the cached + # result preserves the newest attempt. A later pass overwrites an older + # failure, and a repeated failure retains its current diagnosis. + write_json(destination, output) + + +def write_summary(root: Path, plan: dict[str, Any]) -> dict[str, Any]: + passed = completed_names(root, plan) + failed: list[str] = [] + for task in plan["tasks"]: + path = result_path(root, task["name"]) + if path.is_file() and task["name"] not in passed: + failed.append(task["name"]) + failed_classes: dict[str, str] = {} + for name in failed: + result = json.loads(result_path(root, name).read_text(encoding="utf-8")) + failed_classes[name] = str(result.get("failure_class") or "harness_or_integration") + summary = { + "schema_version": SUMMARY_SCHEMA, + "status": "passed" if len(passed) == len(plan["tasks"]) else "partial", + "plan_sha256": canonical_sha256(plan), + "planned": len(plan["tasks"]), + "passed": len(passed), + "failed": len(failed), + "pending": len(plan["tasks"]) - len(passed) - len(failed), + "failed_tasks": sorted(failed), + "infrastructure_failures": sorted( + name for name, failure_class in failed_classes.items() if failure_class == "infrastructure" + ), + "integration_failures": sorted( + name for name, failure_class in failed_classes.items() if failure_class != "infrastructure" + ), + } + write_json(root / "summary.json", summary) + return summary + + +def run_harbor(args: argparse.Namespace, plan: dict[str, Any], pending: list[str]) -> int: + inputs = plan["inputs"] + job_name = f"setup-admission-{datetime.now(UTC).strftime('%Y%m%dT%H%M%S%fZ')}" + mounts = json.dumps( + [ + { + "type": "bind", + "source": str(args.hermetic_runtime), + "target": "/opt/hermes-runtime", + "read_only": True, + "bind": {"create_host_path": False}, + } + ], + separators=(",", ":"), + ) + command = [ + str(args.harbor), + "run", + "--path", + str(args.dataset), + "--n-tasks", + str(len(pending)), + "--agent", + "harbor_hermes_agent:HarborHermesAgent", + "--model", + "openai/setup-admission-stub", + "--ak", + f"commit={inputs['hermes_commit']}", + "--ak", + f"relay_config_path={inputs['relay_config']}", + "--ak", + f"switchyard_bundle_dir={inputs['switchyard_bundle']}", + "--ak", + f"relay_wheel_path={inputs['relay_wheel']}", + "--ak", + f"relay_wheel_sha256={inputs['relay_wheel_sha256']}", + "--ak", + f"relay_architecture={inputs['relay_architecture']}", + "--ak", + f"hermetic_runtime_dir={inputs['hermetic_runtime_root']}", + "--ak", + f"hermetic_runtime_sha256={inputs['hermetic_runtime_sha256']}", + "--mounts", + mounts, + "--install-only", + "--disable-verification", + "--n-concurrent", + str(args.concurrency), + "--n-attempts", + "1", + "--agent-setup-timeout-multiplier", + "6", + "--environment-build-timeout-multiplier", + "6", + "--job-name", + job_name, + "--jobs-dir", + str(args.output / "jobs"), + "--yes", + ] + if args.force_build: + command.append("--force-build") + if args.preserve_containers: + command.append("--no-delete") + for task_name in pending: + command.extend(["--include-task-name", task_name]) + env = os.environ.copy() + agent_path = Path(__file__).resolve().parents[1] / "agents" + env["PYTHONPATH"] = f"{agent_path}{os.pathsep}{env.get('PYTHONPATH', '')}".rstrip(os.pathsep) + with (args.output / "admission.log").open("a", encoding="utf-8") as log: + log.write(f"[{datetime.now(UTC).isoformat()}] starting {len(pending)} task(s)\n") + log.flush() + process = subprocess.run(command, env=env, stdout=log, stderr=subprocess.STDOUT) + log.write(f"[{datetime.now(UTC).isoformat()}] Harbor exit={process.returncode}\n") + return process.returncode + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=Path, required=True) + parser.add_argument("--runtime-root", type=Path, required=True) + parser.add_argument("--hermetic-runtime", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--harbor", type=Path, required=True) + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--batch-size", type=int, default=89) + parser.add_argument("--max-infra-attempts", type=int, default=4) + parser.add_argument("--backoff-seconds", type=float, default=15) + parser.add_argument( + "--force-build", + action=argparse.BooleanOptionalAction, + default=True, + help="exercise Docker's task-image build path even when an image already exists", + ) + parser.add_argument( + "--preserve-containers", + action=argparse.BooleanOptionalAction, + default=False, + help="retain admission containers for debugging (off by default)", + ) + parser.add_argument( + "--task-name", + action="append", + default=[], + help="Run only this task in the immutable all-89 plan; repeat as needed.", + ) + parser.add_argument("--plan-only", action="store_true") + args = parser.parse_args() + if args.concurrency < 1 or args.batch_size < 1 or args.max_infra_attempts < 1: + parser.error("concurrency, batch size, and infrastructure attempts must be positive") + if args.backoff_seconds < 0: + parser.error("--backoff-seconds cannot be negative") + for name in ("dataset", "runtime_root", "hermetic_runtime", "harbor"): + value = getattr(args, name).expanduser().resolve(strict=True) + setattr(args, name, value) + args.output = args.output.expanduser().resolve() + args.output.mkdir(mode=0o700, parents=True, exist_ok=True) + (args.output / "jobs").mkdir(mode=0o700, exist_ok=True) + (args.output / "task-results").mkdir(mode=0o700, exist_ok=True) + + if harbor_version != "0.18.0": + raise RuntimeError(f"setup admission requires Harbor 0.18.0, found {harbor_version}") + payload = load_payload(args.hermetic_runtime) + tasks = discover_tasks(args.dataset) + known_names = {task.name for task in tasks} + selected_names = set(args.task_name) + unknown_names = selected_names - known_names + if unknown_names: + parser.error(f"unknown --task-name values: {', '.join(sorted(unknown_names))}") + candidate = build_plan(args, tasks, payload) + plan_path = args.output / "plan.json" + if plan_path.exists(): + plan = json.loads(plan_path.read_text(encoding="utf-8")) + comparable = dict(candidate) + comparable["created_at"] = plan.get("created_at") + if plan != comparable: + raise ValueError("existing setup-admission plan does not match current inputs") + else: + plan = candidate + write_json(plan_path, plan) + if args.plan_only: + print(json.dumps(write_summary(args.output, plan), indent=2, sort_keys=True)) + return 0 + + # Import completed/failed results before an infrastructure gate can stop a + # resumed invocation. This preserves every finished task even when the + # machine is not currently healthy enough to launch more work. + parse_job_results(args.output, plan) + clock_preflight = run_clock_preflight() + write_json(args.output / "clock-preflight.json", clock_preflight) + if clock_preflight["status"] != "passed": + write_summary(args.output, plan) + raise RuntimeError( + "wall-clock preflight failed; synchronize the host clock before building " + f"task environments (evidence: {args.output / 'clock-preflight.json'})" + ) + plugin_preflight = run_plugin_compatibility_preflight(plan) + write_json(args.output / "plugin-compatibility.json", plugin_preflight) + if plugin_preflight["status"] != "passed": + write_summary(args.output, plan) + raise RuntimeError( + "Switchyard plugin compatibility preflight failed in the oldest " + "supported task base (evidence: " + f"{args.output / 'plugin-compatibility.json'})" + ) + + passed = completed_names(args.output, plan) + pending = [ + task["name"] + for task in plan["tasks"] + if task["name"] not in passed and (not selected_names or task["name"] in selected_names) + ] + integration_blockers: set[str] = set() + for offset in range(0, len(pending), args.batch_size): + batch = pending[offset : offset + args.batch_size] + remaining = list(batch) + for attempt in range(1, args.max_infra_attempts + 1): + if not remaining: + break + run_harbor(args, plan, remaining) + # Harbor writes result files atomically enough for a completed process; + # a resumed invocation re-scans every prior job before selecting work. + parse_job_results(args.output, plan) + passed = completed_names(args.output, plan) + remaining = [name for name in remaining if name not in passed] + blockers: set[str] = set() + for name in remaining: + path = result_path(args.output, name) + if not path.is_file(): + continue + result = json.loads(path.read_text(encoding="utf-8")) + if result.get("failure_class") == "harness_or_integration": + blockers.add(name) + if blockers: + integration_blockers.update(blockers) + break + if remaining and attempt < args.max_infra_attempts: + delay = args.backoff_seconds * (2 ** (attempt - 1)) + with (args.output / "admission.log").open("a", encoding="utf-8") as log: + log.write( + f"[{datetime.now(UTC).isoformat()}] retrying {len(remaining)} " + f"infrastructure setup failure(s) after {delay:g}s\n" + ) + time.sleep(delay) + if integration_blockers: + break + summary = write_summary(args.output, plan) + print(json.dumps(summary, indent=2, sort_keys=True)) + if selected_names: + passed = completed_names(args.output, plan) + return 0 if selected_names <= passed else 1 + if integration_blockers: + return 20 + return 0 if summary["status"] == "passed" else 75 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py b/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py new file mode 100755 index 000000000..56b829c52 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise all Phase 2 dataset and runtime wiring without provider work.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import importlib.metadata +import json +import socket +import subprocess +import sys +import tempfile +import tomllib +from pathlib import Path +from typing import Any +from unittest.mock import patch + +_SCRIPT_ROOT = Path(__file__).resolve().parent +if str(_SCRIPT_ROOT) not in sys.path: + sys.path.insert(0, str(_SCRIPT_ROOT)) +from relay_version import wheel_version + +from harbor.job import Job +from harbor.models.job.config import DatasetConfig, JobConfig +from harbor.models.task.task import Task as HarborTask +from harbor.models.trial.config import AgentConfig, EnvironmentConfig + +SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-smoke.v1" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def combined_digest(root: Path, paths: list[Path]) -> str: + digest = hashlib.sha256() + for path in sorted(paths): + digest.update(path.relative_to(root).as_posix().encode()) + digest.update(b"\0") + digest.update(sha256_file(path).encode()) + digest.update(b"\n") + return digest.hexdigest() + + +def parse_memory_gb(task: HarborTask) -> int: + memory_mb = task.config.environment.memory_mb or 2048 + if memory_mb % 1024: + raise ValueError(f"unsupported memory for {task.name}: {memory_mb} MB") + return memory_mb // 1024 + + +async def validate_local_dataset( + dataset_root: Path, + expected_count: int, + jobs_dir: Path, + concurrency: int, + relay_architecture: str, + authorization_file: Path, +) -> tuple[list[dict[str, Any]], int]: + network_attempts = 0 + + def deny_network(*_args: Any, **_kwargs: Any) -> None: + nonlocal network_attempts + network_attempts += 1 + raise AssertionError("local dataset smoke attempted network access") + + dataset = DatasetConfig(path=dataset_root) + with ( + patch("socket.create_connection", side_effect=deny_network), + patch.object(socket.socket, "connect", side_effect=deny_network), + patch( + "harbor.registry.client.factory.RegistryClientFactory.create", + side_effect=deny_network, + ), + ): + configs = await dataset.get_task_configs() + if len(configs) != expected_count: + raise ValueError(f"Harbor resolved {len(configs)} tasks; expected {expected_count}") + task_records: list[dict[str, Any]] = [] + expected_names = sorted(path.parent.name for path in dataset_root.glob("*/task.toml")) + resolved_names = sorted(config.get_local_path().name for config in configs) + if resolved_names != expected_names: + raise ValueError("Harbor local dataset resolution did not preserve the exported task set") + for name in resolved_names: + selected = await DatasetConfig( + path=dataset_root, + task_names=[name], + n_tasks=1, + ).get_task_configs() + if len(selected) != 1 or selected[0].get_local_path().name != name: + raise ValueError(f"Harbor did not uniquely select local task {name}") + task = HarborTask(selected[0].get_local_path()) + if not task.instruction.strip(): + raise ValueError(f"Harbor loaded an empty instruction for {name}") + test_path = task.paths.discovered_test_path + if not test_path.is_file(): + raise ValueError(f"Harbor did not discover a verifier test for {name}") + task_records.append( + { + "name": name, + "memory_gb": parse_memory_gb(task), + "instruction_path": task.paths.instruction_path.relative_to(dataset_root).as_posix(), + "verifier_path": test_path.relative_to(dataset_root).as_posix(), + "task_toml_sha256": sha256_file(task.paths.config_path), + "instruction_sha256": sha256_file(task.paths.instruction_path), + "test_sha256": sha256_file(test_path), + "has_steps": task.has_steps, + } + ) + + job = await Job.create( + JobConfig( + job_name="phase2-all-task-smoke", + jobs_dir=jobs_dir, + n_attempts=1, + n_concurrent_trials=concurrency, + agent_timeout_multiplier=3, + agent_setup_timeout_multiplier=6, + environment_build_timeout_multiplier=6, + agents=[ + AgentConfig( + import_path="harbor_hermes_agent:HarborHermesAgent", + model_name="openai/ollama-route-stub", + kwargs={ + "repository_url": "https://github.com/bbednarski9/hermes-agent.git", + "repository_ref": "feat/relay-native-plugin-init", + "commit": "a3d472f0e6bdc376df87b1436a461c4796db6747", + "relay_config_path": "/smoke/runtime/plugins.toml", + "switchyard_bundle_dir": "/smoke/runtime/switchyard-plugin", + "relay_wheel_path": "/smoke/runtime/nemo-relay.whl", + "relay_architecture": relay_architecture, + }, + env={ + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "http://127.0.0.1:9/v1", + "OPENROUTER_API_KEY": "relay-intercepted", + "OPENROUTER_BASE_URL": "http://127.0.0.1:9/v1", + }, + ) + ], + environment=EnvironmentConfig( + mounts=[ + { + "type": "bind", + "source": str(authorization_file), + "target": "/run/secrets/switchyard-provider-authorization", + "read_only": True, + "bind": {"create_host_path": False}, + } + ] + ), + datasets=[dataset], + artifacts=["/logs/agent/direct-hermes"], + ) + ) + if len(job) != expected_count: + raise ValueError(f"Harbor constructed {len(job)} trials; expected {expected_count}") + return task_records, network_attempts + + +def validate_cli_local_projection( + harbor_bin: Path, dataset_root: Path, task_name: str, authorization_file: Path +) -> None: + mounts = json.dumps( + [ + { + "type": "bind", + "source": str(authorization_file), + "target": "/run/secrets/switchyard-provider-authorization", + "read_only": True, + "bind": {"create_host_path": False}, + } + ], + separators=(",", ":"), + ) + result = subprocess.run( + [ + str(harbor_bin), + "run", + "--path", + str(dataset_root), + "--include-task-name", + task_name, + "--n-tasks", + "1", + "--agent", + "harbor_hermes_agent:HarborHermesAgent", + "--model", + "openai/ollama-route-stub", + "--mounts", + mounts, + "--print-config", + ], + check=True, + capture_output=True, + text=True, + ) + config = json.loads(result.stdout) + datasets = config.get("datasets") + if not isinstance(datasets, list) or len(datasets) != 1: + raise ValueError("Harbor CLI did not render one local dataset") + dataset = datasets[0] + if ( + Path(dataset.get("path", "")).resolve() != dataset_root + or dataset.get("task_names") != [task_name] + or dataset.get("n_tasks") != 1 + or dataset.get("name") is not None + ): + raise ValueError("Harbor CLI local dataset projection is incorrect") + if config.get("environment", {}).get("mounts") != json.loads(mounts): + raise ValueError("Harbor CLI did not preserve the protected authorization mount") + + +def validate_relay_runtime( + example_root: Path, + temporary_root: Path, + switchyard_bundle: Path, + relay_wheel: Path, + relay_architecture: str, + plugin_config_template: Path, +) -> dict[str, Any]: + run_root = temporary_root / "runtime-smoke" + subprocess.run( + [ + sys.executable, + str(example_root / "scripts" / "prepare_runtime.py"), + "--run-root", + str(run_root), + "--switchyard-bundle", + str(switchyard_bundle), + "--relay-wheel", + str(relay_wheel), + "--relay-architecture", + relay_architecture, + "--plugin-config-template", + str(plugin_config_template), + "--openinference-endpoint", + "http://127.0.0.1:4318/v1/traces", + "--phoenix-project", + "phase2-smoke", + "--eval-cohort", + "phase2-smoke", + ], + check=True, + stdout=subprocess.DEVNULL, + ) + compatibility_path = run_root / "artifacts" / "harbor-hermes-compatibility.json" + subprocess.run( + [ + sys.executable, + str(example_root / "scripts" / "verify_harbor_hermes_compat.py"), + "--bridge", + str(example_root / "agents" / "harbor_hermes_agent.py"), + "--relay-config", + str(run_root / "runtime" / "plugins.toml"), + "--output", + str(compatibility_path), + ], + check=True, + stdout=subprocess.DEVNULL, + ) + provenance = json.loads((run_root / "runtime" / "provenance.json").read_text()) + compatibility = json.loads(compatibility_path.read_text()) + with (run_root / "runtime" / "plugins.toml").open("rb") as stream: + relay_config = tomllib.load(stream) + with (switchyard_bundle / "relay-plugin.toml").open("rb") as stream: + switchyard_manifest = tomllib.load(stream) + dynamic_plugins = relay_config.get("plugins", {}).get("dynamic", []) + components = {component["kind"]: component for component in relay_config.get("components", [])} + plugin_id = switchyard_manifest.get("plugin", {}).get("id") + if ( + provenance.get("nemo_relay", {}).get("version") != wheel_version(relay_wheel) + or provenance.get("nemo_relay", {}).get("wheel_sha256") != sha256_file(relay_wheel) + or provenance.get("switchyard", {}).get("library_sha256") is None + or compatibility.get("status") != "passed" + or len(dynamic_plugins) != 1 + or dynamic_plugins[0].get("manifest") != "/opt/relay-plugins/nvidia.switchyard/relay-plugin.toml" + or plugin_id != "nvidia.switchyard" + or components.get("observability", {}).get("config", {}).get("version") != 3 + ): + raise ValueError("Relay/Hermes/Switchyard smoke wiring did not pass") + return { + "status": "passed", + "relay_version": provenance["nemo_relay"]["version"], + "relay_wheel_sha256": provenance["nemo_relay"]["wheel_sha256"], + "switchyard_library_sha256": provenance["switchyard"]["library_sha256"], + "plugin_config_template_sha256": provenance["plugin_config_template_sha256"], + "relay_config_sha256": provenance["relay_config_sha256"], + "relay_architecture": provenance["nemo_relay"]["architecture"], + "routing": provenance["routing"], + "dynamic_plugin_id": plugin_id, + "observability_version": 3, + "compatibility_status": compatibility["status"], + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--dataset-root", type=Path, required=True) + parser.add_argument("--expected-count", type=int, default=89) + parser.add_argument("--harbor-bin", type=Path, required=True) + parser.add_argument("--switchyard-bundle", type=Path, required=True) + parser.add_argument("--relay-wheel", type=Path, required=True) + parser.add_argument("--relay-architecture", choices=("x86_64", "aarch64"), required=True) + parser.add_argument("--plugin-config-template", type=Path, required=True) + parser.add_argument("--concurrency", type=int, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + example_root = Path(__file__).resolve().parents[1] + dataset_root = args.dataset_root.expanduser().resolve(strict=True) + if args.concurrency <= 0: + raise ValueError("concurrency must be positive") + if importlib.metadata.version("harbor") != "0.18.0": + raise RuntimeError("Phase 2 smoke requires Harbor 0.18.0") + if ( + subprocess.run([str(args.harbor_bin), "--version"], check=True, capture_output=True, text=True).stdout.strip() + != "0.18.0" + ): + raise RuntimeError("Phase 2 smoke Harbor CLI is not 0.18.0") + + with tempfile.TemporaryDirectory(prefix="harbor-phase2-smoke-") as directory: + temporary_root = Path(directory) + authorization_file = temporary_root / "switchyard-provider-authorization" + authorization_file.write_text("Bearer offline-placeholder", encoding="utf-8") + authorization_file.chmod(0o600) + task_records, network_attempts = asyncio.run( + validate_local_dataset( + dataset_root, + args.expected_count, + temporary_root / "jobs", + args.concurrency, + args.relay_architecture, + authorization_file, + ) + ) + validate_cli_local_projection(args.harbor_bin, dataset_root, task_records[0]["name"], authorization_file) + relay_runtime = validate_relay_runtime( + example_root, + temporary_root, + args.switchyard_bundle.expanduser().resolve(strict=True), + args.relay_wheel.expanduser().resolve(strict=True), + args.relay_architecture, + args.plugin_config_template.expanduser().resolve(strict=True), + ) + + task_tomls = [dataset_root / record["name"] / "task.toml" for record in task_records] + result = { + "schema_version": SCHEMA_VERSION, + "status": "passed", + "harbor_version": "0.18.0", + "dataset_name": dataset_root.name, + "task_count": len(task_records), + "dataset_task_definitions_sha256": combined_digest(dataset_root, task_tomls), + "registry_network_attempts": network_attempts, + "local_cli_projection": "passed", + "job_trial_count": len(task_records), + "concurrency": args.concurrency, + "relay_architecture": args.relay_architecture, + "relay_runtime": relay_runtime, + "memory_lanes": { + "2G": sum(record["memory_gb"] == 2 for record in task_records), + "4G": sum(record["memory_gb"] == 4 for record in task_records), + "8G": sum(record["memory_gb"] == 8 for record in task_records), + }, + "tasks": task_records, + } + args.output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({key: value for key, value in result.items() if key != "tasks"}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/stage_phase2_runtime.py b/examples/harbor-hermes-switchyard/scripts/stage_phase2_runtime.py new file mode 100644 index 000000000..4bb70cf75 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/stage_phase2_runtime.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Stage and verify the immutable Phase 2 runtime harness.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +from pathlib import Path + +RUNTIME_SUFFIXES = {".py", ".sh", ".toml", ".yaml"} +RUNTIME_TOP_LEVEL = ( + "run_terminal_bench.sh", + "run_phase2_cohort.sh", + "supervise_phase2_cohort.sh", +) +RUNTIME_DIRECTORIES = ("agents", "config", "scripts") + + +def runtime_files(root: Path) -> list[Path]: + files = [root / name for name in RUNTIME_TOP_LEVEL] + for relative in RUNTIME_DIRECTORIES: + files.extend( + path + for path in (root / relative).rglob("*") + if path.is_file() and path.suffix in RUNTIME_SUFFIXES + ) + missing = [path for path in files if not path.is_file()] + if missing: + raise FileNotFoundError(f"runtime source is missing: {missing[0]}") + return sorted(files) + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def runtime_digest(root: Path, files: list[Path]) -> str: + digest = hashlib.sha256() + for path in sorted(files): + digest.update(path.relative_to(root).as_posix().encode()) + digest.update(b"\0") + digest.update(file_sha256(path).encode()) + digest.update(b"\n") + return digest.hexdigest() + + +def expected_digest(plan_path: Path) -> str: + plan = json.loads(plan_path.read_text(encoding="utf-8")) + value = plan.get("inputs", {}).get("runtime_sources_sha256") + if not isinstance(value, str) or not value: + raise ValueError("plan does not contain inputs.runtime_sources_sha256") + return value + + +def verify_runtime(root: Path, expected: str) -> tuple[str, int]: + files = runtime_files(root) + observed = runtime_digest(root, files) + if observed != expected: + raise ValueError(f"runtime harness hash mismatch: expected {expected}, observed {observed}") + return observed, len(files) + + +def stage_runtime(source: Path, destination: Path, plan_path: Path) -> dict[str, object]: + source = source.resolve() + destination = destination.resolve() + expected = expected_digest(plan_path) + if destination.is_dir(): + observed, file_count = verify_runtime(destination, expected) + return {"status": "verified", "runtime_sources_sha256": observed, "file_count": file_count} + if destination.exists(): + raise ValueError(f"runtime harness destination is not a directory: {destination}") + + files = runtime_files(source) + observed = runtime_digest(source, files) + if observed != expected: + raise ValueError(f"live runtime differs from immutable plan: expected {expected}, observed {observed}") + + temporary = destination.with_name(f"{destination.name}.tmp-{os.getpid()}") + if temporary.exists(): + shutil.rmtree(temporary) + temporary.mkdir(mode=0o700, parents=False) + try: + for path in files: + target = temporary / path.relative_to(source) + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + shutil.copy2(path, target) + (temporary / "snapshot.json").write_text( + json.dumps( + { + "schema_version": "harbor-hermes-switchyard.phase2-runtime-snapshot.v1", + "runtime_sources_sha256": observed, + "file_count": len(files), + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + temporary.replace(destination) + except BaseException: + shutil.rmtree(temporary, ignore_errors=True) + raise + return {"status": "staged", "runtime_sources_sha256": observed, "file_count": len(files)} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--destination", type=Path, required=True) + parser.add_argument("--plan", type=Path, required=True) + args = parser.parse_args() + result = stage_runtime(args.source, args.destination, args.plan) + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/upload_openinference.py b/examples/harbor-hermes-switchyard/scripts/upload_openinference.py new file mode 100755 index 000000000..b1dcddb1e --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/upload_openinference.py @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Stream bounded OTLP JSON batches into a Phoenix HTTP receiver.""" + +from __future__ import annotations + +import argparse +import base64 +import json +import re +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + +from google.protobuf import json_format +from opentelemetry.proto.collector.trace.v1 import trace_service_pb2 + + +def _serialize(resource_spans: list[dict[str, Any]]) -> bytes: + request = trace_service_pb2.ExportTraceServiceRequest() + json_format.ParseDict({"resourceSpans": resource_spans}, request) + return request.SerializeToString() + + +def _hex_id_to_base64(span: dict[str, Any], field: str, width: int) -> None: + value = span.get(field) + if isinstance(value, str) and re.fullmatch(rf"[0-9a-fA-F]{{{width}}}", value): + span[field] = base64.b64encode(bytes.fromhex(value)).decode("ascii") + + +def _normalize_ids(payload: dict[str, Any]) -> None: + for resource_span in payload.get("resourceSpans", []): + for scope_span in resource_span.get("scopeSpans", []): + for span in scope_span.get("spans", []): + _hex_id_to_base64(span, "traceId", 32) + _hex_id_to_base64(span, "spanId", 16) + _hex_id_to_base64(span, "parentSpanId", 16) + + +def _set_project(resource_span: dict[str, Any], project: str) -> None: + attributes = resource_span.setdefault("resource", {}).setdefault("attributes", []) + for attribute in attributes: + if attribute.get("key") == "openinference.project.name": + attribute["value"] = {"stringValue": project} + return + attributes.append( + { + "key": "openinference.project.name", + "value": {"stringValue": project}, + } + ) + + +def _post(endpoint: str, body: bytes, timeout: float, attempts: int) -> int: + for attempt in range(1, attempts + 1): + request = urllib.request.Request( + endpoint, + data=body, + headers={"content-type": "application/x-protobuf"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + if response.status != 200: + raise RuntimeError(f"Phoenix returned HTTP {response.status}") + return attempt - 1 + except (TimeoutError, urllib.error.URLError): + if attempt == attempts: + raise + time.sleep(2 ** (attempt - 1)) + raise AssertionError("unreachable") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--openinference", type=Path, required=True) + parser.add_argument("--phoenix-url", required=True) + parser.add_argument("--project", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--max-batch-bytes", type=int, default=1024 * 1024) + parser.add_argument("--timeout-seconds", type=float, default=60) + parser.add_argument("--max-attempts", type=int, default=3) + args = parser.parse_args() + if args.batch_size < 1 or args.max_batch_bytes < 1 or args.max_attempts < 1: + raise ValueError("batch and retry limits must be positive") + + endpoint = args.phoenix_url.rstrip("/") + "/v1/traces" + pending: list[dict[str, Any]] = [] + pending_documents = 0 + uploaded_documents = 0 + uploaded_batches = 0 + uploaded_spans = 0 + retries = 0 + + def upload(items: list[dict[str, Any]]) -> None: + nonlocal uploaded_batches, retries + retries += _post( + endpoint, + _serialize(items), + args.timeout_seconds, + args.max_attempts, + ) + uploaded_batches += 1 + + with args.openinference.open(encoding="utf-8") as stream: + for line in stream: + if not line.strip(): + continue + payload: dict[str, Any] = json.loads(line) + _normalize_ids(payload) + resource_spans = list(payload.get("resourceSpans", [])) + for resource_span in resource_spans: + _set_project(resource_span, args.project) + uploaded_spans += sum(len(scope.get("spans", [])) for scope in resource_span.get("scopeSpans", [])) + candidate = pending + resource_spans + if pending and (pending_documents >= args.batch_size or len(_serialize(candidate)) > args.max_batch_bytes): + upload(pending) + pending = resource_spans + pending_documents = 1 + else: + pending = candidate + pending_documents += 1 + uploaded_documents += 1 + if pending: + upload(pending) + if uploaded_documents == 0 or uploaded_spans == 0: + raise RuntimeError("OpenInference artifact did not contain any spans") + + result = { + "status": "passed", + "project": args.project, + "endpoint": endpoint, + "uploaded_documents": uploaded_documents, + "uploaded_batches": uploaded_batches, + "upload_retries": retries, + "uploaded_spans": uploaded_spans, + } + args.output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh new file mode 100755 index 000000000..9ff693512 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail +set +x + +env_file="${1:-}" +if [[ -z "$env_file" || ! -f "$env_file" ]]; then + echo "usage: $0 .env" >&2 + exit 2 +fi +env_file="$(cd "$(dirname "$env_file")" && pwd)/$(basename "$env_file")" +if mode="$(stat -f '%Lp' "$env_file" 2>/dev/null)"; then + : +else + mode="$(stat -c '%a' "$env_file")" +fi +if [[ "$mode" != "600" ]]; then + echo "Phase 2 environment file must have mode 0600: $env_file" >&2 + exit 2 +fi +if grep -Eq '^(INFERENCE_SECRETS_FILE|NV_INFERENCEHUB_ENDPOINT|NV_INFERENCEHUB_KEY|STRONG_MODEL|WEAK_MODEL|UPSTREAM_BASE_URL|UPSTREAM_AUTH_ENV)=' "$env_file"; then + echo "legacy secret or provider-routing overrides are not supported in Phase 2" >&2 + exit 2 +fi + +set -a +# shellcheck disable=SC1090 +source "$env_file" +set +a + +required_values=( + EXAMPLE_ROOT TERMINAL_BENCH_RUN_ID TERMINAL_BENCH_RUN_ROOT TERMINAL_BENCH_ADMISSION_ROOT + TERMINAL_BENCH_BOOTSTRAP_ROOT + HARBOR_BIN EVAL_PYTHON TBENCH_DATASET_PATH SWITCHYARD_BUNDLE RELAY_WHEEL + RELAY_ARCHITECTURE PLUGIN_CONFIG_TEMPLATE TERMINAL_BENCH_SMOKE_EVIDENCE + TERMINAL_BENCH_OFFLINE_EVIDENCE PHOENIX_BASE_URL PHOENIX_PROJECT EVAL_COHORT + TBENCH_SAMPLE_COUNT TBENCH_CANARY_TASK TBENCH_CONCURRENCY + TBENCH_SETUP_CONCURRENCY TBENCH_SETUP_BATCH_SIZE TBENCH_SETUP_MAX_INFRA_ATTEMPTS + TBENCH_PARALLEL_MAX_MEMORY_GB TBENCH_DOCKER_MEMORY_RESERVE_GB + TBENCH_MINIMUM_FREE_GB SWITCHYARD_PROVIDER_AUTHORIZATION +) +for name in "${required_values[@]}"; do + if [[ -z "${!name:-}" ]]; then + echo "required Phase 2 variable is unset: $name" >&2 + exit 2 + fi +done +for name in EXAMPLE_ROOT TERMINAL_BENCH_RUN_ROOT TERMINAL_BENCH_ADMISSION_ROOT \ + TERMINAL_BENCH_BOOTSTRAP_ROOT HARBOR_BIN EVAL_PYTHON \ + TBENCH_DATASET_PATH SWITCHYARD_BUNDLE RELAY_WHEEL PLUGIN_CONFIG_TEMPLATE \ + TERMINAL_BENCH_SMOKE_EVIDENCE TERMINAL_BENCH_OFFLINE_EVIDENCE; do + if [[ "${!name}" != /* ]]; then + echo "Phase 2 path must be absolute: $name" >&2 + exit 2 + fi +done +for name in TBENCH_SAMPLE_COUNT TBENCH_CONCURRENCY TBENCH_SETUP_CONCURRENCY \ + TBENCH_SETUP_BATCH_SIZE TBENCH_SETUP_MAX_INFRA_ATTEMPTS TBENCH_PARALLEL_MAX_MEMORY_GB \ + TBENCH_DOCKER_MEMORY_RESERVE_GB TBENCH_MINIMUM_FREE_GB; do + if [[ ! "${!name}" =~ ^[1-9][0-9]*$ ]]; then + echo "Phase 2 capacity value must be a positive integer: $name" >&2 + exit 2 + fi +done +case "$RELAY_ARCHITECTURE" in + x86_64|aarch64) ;; + *) echo "RELAY_ARCHITECTURE must be x86_64 or aarch64" >&2; exit 2 ;; +esac +for path in "$EXAMPLE_ROOT" "$TBENCH_DATASET_PATH" "$SWITCHYARD_BUNDLE"; do + [[ -d "$path" ]] || { echo "required Phase 2 directory is missing" >&2; exit 2; } +done +for path in "$HARBOR_BIN" "$EVAL_PYTHON" "$RELAY_WHEEL" "$PLUGIN_CONFIG_TEMPLATE"; do + [[ -f "$path" ]] || { echo "required Phase 2 file is missing" >&2; exit 2; } +done + +"$EVAL_PYTHON" - "$PLUGIN_CONFIG_TEMPLATE" <<'PY' +import sys +import tomllib + +with open(sys.argv[1], "rb") as stream: + config = tomllib.load(stream) +plugins = config.get("plugins", {}).get("dynamic", []) +if len(plugins) != 1: + raise SystemExit("plugin config must contain one dynamic plugin") +targets = plugins[0].get("config", {}).get("targets", {}) +if set(targets) != {"strong", "weak", "judge"}: + raise SystemExit("plugin config must contain strong, weak, and judge targets") +for target in targets.values(): + if target.get("header_env") != {"authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION"}: + raise SystemExit("plugin config must reference SWITCHYARD_PROVIDER_AUTHORIZATION") +PY + +echo "Phase 2 environment validation passed (secret values withheld)" diff --git a/examples/harbor-hermes-switchyard/scripts/validate_run.py b/examples/harbor-hermes-switchyard/scripts/validate_run.py new file mode 100755 index 000000000..1be9bcf10 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/validate_run.py @@ -0,0 +1,555 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate one Harbor/Hermes/Switchyard Phase 1 task evidence set.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +_SCRIPT_ROOT = Path(__file__).resolve().parent +if str(_SCRIPT_ROOT) not in sys.path: + sys.path.insert(0, str(_SCRIPT_ROOT)) +from relay_version import require_supported_version +from typing import Any, Iterable + +SCHEMA_VERSION = "harbor-hermes-switchyard.validation.v1" + + +def read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"expected a JSON object: {path}") + return value + + +def is_trial_result(value: dict[str, Any]) -> bool: + return "task_name" in value and "verifier_result" in value + + +def read_benchmark_passed(value: dict[str, Any]) -> bool | None: + reward = value.get("reward") + if isinstance(reward, dict): + candidate = reward.get("task_passed") + if isinstance(candidate, bool): + return candidate + verifier = value.get("verifier_result") + rewards = verifier.get("rewards") if isinstance(verifier, dict) else None + if isinstance(rewards, dict): + candidate = rewards.get("task_passed") + if isinstance(candidate, bool): + return candidate + candidate = rewards.get("reward") + if isinstance(candidate, (int, float)) and not isinstance(candidate, bool): + return candidate > 0 + return None + + +def is_verifier_backed_agent_timeout_nonpass( + direct_result: dict[str, Any], harbor_result: dict[str, Any] +) -> bool: + """Recognize a completed benchmark non-pass caused by Harbor's agent deadline. + + Harbor cancels the agent subprocess at its configured deadline, so the + direct adapter records ``CancelledError`` and cannot emit a final response + or terminal ATIF/AGENT span. The outcome is complete only when Harbor + independently records ``AgentTimeoutError`` and the verifier produces a + non-passing reward. Other cancellation and missing-artifact cases remain + integration failures. + """ + error = direct_result.get("error") + exception = harbor_result.get("exception_info") + return ( + direct_result.get("status") == "failed" + and isinstance(error, dict) + and error.get("phase") == "agent" + and error.get("type") == "CancelledError" + and isinstance(exception, dict) + and exception.get("exception_type") == "AgentTimeoutError" + and read_benchmark_passed(harbor_result) is False + ) + + +def validate_harbor_job_config(job_dir: Path) -> tuple[dict[str, float], list[str]]: + """Validate the timeout multipliers serialized by Harbor for this job.""" + errors: list[str] = [] + path = job_dir / "config.json" + expected = { + "agent_timeout_multiplier": 3.0, + "agent_setup_timeout_multiplier": 6.0, + "environment_build_timeout_multiplier": 6.0, + } + if not path.is_file(): + return {}, [f"missing Harbor job config: {path}"] + config = read_json(path) + observed: dict[str, float] = {} + for key, expected_value in expected.items(): + value = config.get(key) + if not isinstance(value, (int, float)) or isinstance(value, bool): + errors.append(f"Harbor job config has no numeric {key}") + continue + observed[key] = float(value) + if observed[key] != expected_value: + errors.append(f"Harbor job config {key}={observed[key]} does not match required {expected_value}") + return observed, errors + + +def contained_files(root: Path) -> list[Path]: + resolved_root = root.resolve(strict=True) + files: list[Path] = [] + for path in root.rglob("*"): + if path.is_symlink(): + raise ValueError(f"artifact symlink is forbidden: {path}") + if not path.is_file(): + continue + resolved = path.resolve(strict=True) + if resolved_root not in resolved.parents: + raise ValueError(f"artifact escaped its root: {path}") + files.append(path) + return files + + +def scan_files(root: Path) -> list[Path]: + """Return regular files below a scan root without following symlinks.""" + if not root.is_dir(): + return [] + files: list[Path] = [] + for path in root.rglob("*"): + if path.is_symlink(): + continue + if path.is_file(): + files.append(path) + return files + + +def scan_secrets(files: Iterable[Path], values: list[bytes]) -> list[str]: + findings: list[str] = [] + for path in files: + with path.open("rb") as stream: + overlap = b"" + while True: + block = stream.read(1024 * 1024) + if not block: + break + haystack = overlap + block + for index, secret in enumerate(values): + if secret and secret in haystack: + findings.append(f"{path.name}:secret[{index}]") + max_width = max((len(secret) for secret in values), default=1) + overlap = haystack[-max_width:] + return sorted(set(findings)) + + +def validate_receipt_provenance(receipt: dict[str, Any], provenance: dict[str, Any]) -> list[str]: + errors: list[str] = [] + dependencies = receipt.get("dependencies", {}) + relay = dependencies.get("nemo_relay", {}) + hermes = dependencies.get("hermes", {}) + switchyard = dependencies.get("switchyard", {}) + provenance_relay = provenance.get("nemo_relay", {}) + provenance_hermes = provenance.get("hermes", {}) + provenance_switchyard = provenance.get("switchyard", {}) + if relay.get("version") != provenance_relay.get("version"): + errors.append("receipt Relay version does not match runtime provenance") + else: + try: + require_supported_version(relay.get("version", "")) + except ValueError: + errors.append("receipt did not record nemo-relay>=0.7.0") + if relay.get("wheel_sha256") != provenance_relay.get("wheel_sha256"): + errors.append("Relay wheel digest does not match runtime provenance") + if receipt.get("relay_config_sha256") != provenance.get("relay_config_sha256"): + errors.append("Relay config digest does not match runtime provenance") + if hermes.get("commit") != provenance_hermes.get("commit"): + errors.append("Hermes commit does not match runtime provenance") + if switchyard.get("commit") != provenance_switchyard.get("commit"): + errors.append("Switchyard commit does not match runtime provenance") + if switchyard.get("manifest_sha256") != provenance_switchyard.get("manifest_sha256"): + errors.append("Switchyard manifest digest does not match runtime provenance") + if switchyard.get("library_sha256") != provenance_switchyard.get("library_sha256"): + errors.append("Switchyard library digest does not match runtime provenance") + if receipt.get("dynamic_plugin_ids") != ["nvidia.switchyard"]: + errors.append("receipt did not record only nvidia.switchyard") + if receipt.get("activation_mode") != "relay_standard_dynamic": + errors.append("receipt did not record standard dynamic activation") + if receipt.get("routing_contract") != { + "relay_outer_lifecycle": True, + "execution_intercept_owner": "nvidia.switchyard", + "provider_http_client_owner": "switchyard-llm-client", + "separate_switchyard_service": False, + }: + errors.append("receipt did not record the expected Relay/Switchyard ownership contract") + cleanup = receipt.get("cleanup", {}) + if not cleanup.get("plugin_host_closed") or not cleanup.get("exporters_flushed"): + errors.append("receipt did not prove plugin close and exporter flush") + return errors + + +def _otlp_attribute_value(attribute: dict[str, Any]) -> Any: + value = attribute.get("value") + if not isinstance(value, dict): + return None + for key in ("stringValue", "boolValue", "intValue", "doubleValue"): + if key in value: + return value[key] + return None + + +def inspect_openinference(path: Path) -> dict[str, Any]: + documents = 0 + spans = 0 + span_kinds: set[str] = set() + scope_names: set[str] = set() + lineage_spans = 0 + resource_attributes: dict[str, set[Any]] = {} + with path.open(encoding="utf-8") as stream: + for line_number, line in enumerate(stream, 1): + if not line.strip(): + continue + payload = json.loads(line) + if not isinstance(payload, dict): + raise ValueError(f"OpenInference line {line_number} is not an object") + documents += 1 + for resource_span in payload.get("resourceSpans", []): + for attribute in resource_span.get("resource", {}).get("attributes", []): + key = attribute.get("key") + value = _otlp_attribute_value(attribute) + if isinstance(key, str) and value is not None: + resource_attributes.setdefault(key, set()).add(value) + for scope_span in resource_span.get("scopeSpans", []): + scope_name = scope_span.get("scope", {}).get("name") + if isinstance(scope_name, str) and scope_name: + scope_names.add(scope_name) + for span in scope_span.get("spans", []): + spans += 1 + attributes = { + attribute.get("key"): _otlp_attribute_value(attribute) + for attribute in span.get("attributes", []) + if isinstance(attribute, dict) + } + kind = attributes.get("openinference.span.kind") + if isinstance(kind, str) and kind: + span_kinds.add(kind) + scope_lineage = attributes.get("nemo_relay.uuid") and attributes.get( + "nemo_relay.scope_type" + ) + mark_lineage = ( + attributes.get("nemo_relay.mark.uuid") + and attributes.get("nemo_relay.mark.parent_uuid") + and span.get("parentSpanId") + ) + if scope_lineage or mark_lineage: + lineage_spans += 1 + return { + "documents": documents, + "spans": spans, + "span_kinds": sorted(span_kinds), + "scope_names": sorted(scope_names), + "lineage_spans": lineage_spans, + "resource_attributes": {key: sorted(values, key=str) for key, values in sorted(resource_attributes.items())}, + } + + +def inspect_atof(path: Path) -> dict[str, Any]: + count = 0 + marks: list[str] = [] + models: list[str] = [] + targets: list[str] = [] + cache_read_tokens = 0 + cache_write_tokens = 0 + decision_count = 0 + with path.open(encoding="utf-8") as stream: + for line_number, line in enumerate(stream, 1): + if not line.strip(): + continue + payload = json.loads(line) + if not isinstance(payload, dict): + raise ValueError(f"ATOF line {line_number} is not an object") + count += 1 + name = payload.get("name") + if isinstance(name, str) and name.startswith("switchyard.routing."): + marks.append(name) + if name == "switchyard.routing.decision": + decision_count += 1 + for container in (payload.get("data"), payload.get("metadata")): + if isinstance(container, dict): + for key in ("model", "selected_model", "target_model"): + value = container.get(key) + if isinstance(value, str) and value: + models.append(value) + value = container.get("selected_target") + if isinstance(value, str) and value: + targets.append(value) + profile = payload.get("category_profile") + response = profile.get("annotated_response") if isinstance(profile, dict) else None + data = payload.get("data") + if isinstance(response, dict): + model = response.get("model") + if isinstance(model, str) and model: + models.append(model) + if name == "openai.chat_completions" and payload.get("scope_category") == "end": + model = data.get("model") if isinstance(data, dict) else None + if isinstance(model, str) and model: + models.append(model) + if name == "llm.chunk" and isinstance(data, dict): + usage = data.get("usage") + if isinstance(usage, dict): + cache_read = usage.get("cache_read_tokens") + cache_write = usage.get("cache_write_tokens") + if isinstance(cache_read, int) and not isinstance(cache_read, bool): + cache_read_tokens += cache_read + if isinstance(cache_write, int) and not isinstance(cache_write, bool): + cache_write_tokens += cache_write + return { + "count": count, + "marks": sorted(set(marks)), + "models": sorted(set(models)), + "targets": sorted(set(targets)), + "decision_count": decision_count, + "cache_read_tokens": cache_read_tokens, + "cache_write_tokens": cache_write_tokens, + } + + +def read_atof(path: Path) -> tuple[int, list[str], list[str], list[str]]: + """Retain the Phase 1 tuple API while Phase 2 consumes richer evidence.""" + evidence = inspect_atof(path) + return evidence["count"], evidence["marks"], evidence["models"], evidence["targets"] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--artifacts", type=Path, required=True) + parser.add_argument("--provenance", type=Path, required=True) + parser.add_argument("--openinference", type=Path, required=True) + parser.add_argument("--harbor-job-dir", type=Path) + parser.add_argument("--scan-root", type=Path, action="append", default=[]) + parser.add_argument("--expect-late-failure", action="store_true") + parser.add_argument("--secret-env", action="append", default=[]) + parser.add_argument("--secret-file", type=Path, action="append", default=[]) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + # Keep benchmark completion distinct from the optional-but-audited + # Relay/Switchyard evidence contract. A verifier-backed task result is + # still a completed Terminal-Bench evaluation if observability evidence is + # incomplete; the cohort summary decides whether that evidence is strong + # enough for this integration example to pass as a whole. + integration_errors: list[str] = [] + benchmark_errors: list[str] = [] + integration_warnings: list[str] = [] + root = args.artifacts.resolve() + try: + files = contained_files(root) + except Exception as error: + files = [] + integration_errors.append(str(error)) + + required = { + "result": root / "direct-hermes-result.json", + "receipt": root / "direct-hermes-receipt.json", + "completion": root / "completion.json", + "atof": root / "relay" / "trajectory.atof.jsonl", + } + for name, path in required.items(): + if not path.is_file(): + target = benchmark_errors if name == "result" else integration_errors + target.append(f"missing {name}: {path}") + atif_files = sorted((root / "relay" / "atif").glob("trajectory-*.atif.json")) + if not atif_files: + integration_errors.append("missing ATIF trajectory") + if not args.openinference.is_file() or args.openinference.stat().st_size == 0: + integration_errors.append("missing OpenInference OTLP artifact") + + result: dict[str, Any] = {} + receipt: dict[str, Any] = {} + provenance: dict[str, Any] = {} + if required["result"].is_file(): + result = read_json(required["result"]) + if result.get("status") not in {"completed", "preserved_completed_response"}: + benchmark_errors.append(f"invalid direct result status: {result.get('status')!r}") + if not isinstance(result.get("final_response"), str) or not result["final_response"]: + benchmark_errors.append("direct result has no normalized final response") + if args.expect_late_failure: + if result.get("status") != "preserved_completed_response": + benchmark_errors.append("expected a preserved completed response after late failure") + if result.get("error", {}).get("type") != "InjectedPostResponseFailure": + benchmark_errors.append("deterministic post-response failure was not recorded") + if required["receipt"].is_file(): + receipt = read_json(required["receipt"]) + if args.provenance.is_file(): + provenance = read_json(args.provenance) + else: + integration_errors.append("missing runtime provenance") + + if receipt: + integration_errors.extend(validate_receipt_provenance(receipt, provenance)) + + openinference_evidence = { + "documents": 0, + "spans": 0, + "span_kinds": [], + "scope_names": [], + "lineage_spans": 0, + "resource_attributes": {}, + } + if args.openinference.is_file() and args.openinference.stat().st_size > 0: + try: + openinference_evidence = inspect_openinference(args.openinference) + except Exception as error: + integration_errors.append(f"invalid OpenInference OTLP artifact: {error}") + if openinference_evidence["documents"] == 0 or openinference_evidence["spans"] == 0: + integration_errors.append("OpenInference artifact contains no spans") + if not {"AGENT", "LLM"}.issubset(openinference_evidence["span_kinds"]): + integration_errors.append("OpenInference artifact does not contain both AGENT and LLM span kinds") + if openinference_evidence["scope_names"] != ["harbor-hermes-switchyard"]: + integration_errors.append("OpenInference instrumentation scope does not match the example") + if openinference_evidence["lineage_spans"] != openinference_evidence["spans"]: + integration_errors.append("OpenInference spans are missing Relay UUID or scope-type lineage") + expected_resources = { + "openinference.project.name": provenance.get("phoenix_project"), + "evaluation.cohort": provenance.get("eval_cohort"), + "service.name": "harbor-hermes-switchyard", + "service.namespace": "nemo-relay-examples", + } + for key, expected in expected_resources.items(): + if openinference_evidence["resource_attributes"].get(key) != [expected]: + integration_errors.append(f"OpenInference resource attribute {key!r} does not match runtime provenance") + + event_count = 0 + routing_marks: list[str] = [] + routed_models: list[str] = [] + routed_targets: list[str] = [] + switchyard_decision_count = 0 + cache_read_tokens = 0 + cache_write_tokens = 0 + if required["atof"].is_file(): + atof_evidence = inspect_atof(required["atof"]) + event_count = atof_evidence["count"] + routing_marks = atof_evidence["marks"] + routed_models = atof_evidence["models"] + routed_targets = atof_evidence["targets"] + switchyard_decision_count = atof_evidence["decision_count"] + cache_read_tokens = atof_evidence["cache_read_tokens"] + cache_write_tokens = atof_evidence["cache_write_tokens"] + if event_count == 0: + integration_errors.append("ATOF artifact is empty") + if not routing_marks: + integration_errors.append("ATOF artifact has no Switchyard routing evidence") + if not routed_targets: + integration_errors.append("ATOF artifact has no selected Switchyard target") + unexpected_targets = sorted(set(routed_targets) - {"strong", "weak"}) + if unexpected_targets: + integration_errors.append(f"ATOF artifact selected unexpected targets: {unexpected_targets}") + + caller_model = provenance.get("routing", {}).get("hermes_caller_model") + target_models = { + "strong": provenance.get("routing", {}).get("strong_model"), + "weak": provenance.get("routing", {}).get("weak_model"), + } + routed_models = sorted( + { + *routed_models, + *(target_models[target] for target in routed_targets if target_models.get(target)), + } + ) + if caller_model and caller_model in routed_models: + integration_errors.append("Hermes caller stub appeared as a routed provider model") + + secret_values: list[bytes] = [] + for name in args.secret_env: + value = os.environ.get(name) + if value: + secret_values.append(value.encode()) + if value.startswith("Bearer ") and value[7:]: + secret_values.append(value[7:].encode()) + for path in args.secret_file: + for line in path.read_bytes().splitlines(): + value = line.split(b"=", 1)[-1].strip() + if value: + secret_values.append(value) + files_to_scan = list(files) + if args.openinference.is_file(): + files_to_scan.append(args.openinference) + for scan_root in args.scan_root: + files_to_scan.extend(scan_files(scan_root.resolve())) + findings = scan_secrets(sorted(set(files_to_scan)), secret_values) + if findings: + integration_errors.append(f"secret scan found {len(findings)} persisted value(s)") + + harbor_results: list[Path] = [] + harbor_timeout_multipliers: dict[str, float] = {} + benchmark_passed: bool | None = None + harbor_result: dict[str, Any] = {} + if args.harbor_job_dir: + harbor_timeout_multipliers, timeout_errors = validate_harbor_job_config(args.harbor_job_dir) + integration_errors.extend(timeout_errors) + harbor_results = [ + path for path in sorted(args.harbor_job_dir.glob("**/result.json")) if is_trial_result(read_json(path)) + ] + if len(harbor_results) != 1: + benchmark_errors.append(f"expected one Harbor trial result, found {len(harbor_results)}") + elif harbor_results: + harbor_result = read_json(harbor_results[0]) + benchmark_passed = read_benchmark_passed(harbor_result) + if benchmark_passed is None: + benchmark_errors.append("Harbor trial did not contain a normalized benchmark reward") + else: + benchmark_errors.append("missing Harbor job directory for benchmark completion") + + terminal_timeout_nonpass = is_verifier_backed_agent_timeout_nonpass(result, harbor_result) + if terminal_timeout_nonpass: + tolerated_benchmark_errors = { + "invalid direct result status: 'failed'", + "direct result has no normalized final response", + } + benchmark_errors = [error for error in benchmark_errors if error not in tolerated_benchmark_errors] + tolerated_integration_errors = { + "missing ATIF trajectory", + } + if "LLM" in openinference_evidence["span_kinds"]: + tolerated_integration_errors.add("OpenInference artifact does not contain both AGENT and LLM span kinds") + for error in integration_errors: + if error in tolerated_integration_errors: + integration_warnings.append(error) + integration_errors = [error for error in integration_errors if error not in tolerated_integration_errors] + + validation = { + "schema_version": SCHEMA_VERSION, + "status": "passed" if not benchmark_errors else "failed", + "errors": benchmark_errors, + "benchmark": {"status": "passed" if not benchmark_errors else "failed", "errors": benchmark_errors}, + "integration": { + "status": "passed" if not integration_errors else "failed", + "errors": integration_errors, + "warnings": integration_warnings, + }, + "direct_result_status": result.get("status"), + "harbor_trial_count": len(harbor_results) if args.harbor_job_dir else None, + "harbor_timeout_multipliers": harbor_timeout_multipliers, + "benchmark_task_passed": benchmark_passed, + "terminal_agent_timeout_nonpass": terminal_timeout_nonpass, + "atof_event_count": event_count, + "atif_trajectory_count": len(atif_files), + "openinference": openinference_evidence, + "switchyard_routing_marks": routing_marks, + "switchyard_decision_count": switchyard_decision_count, + "routed_models": routed_models, + "routed_targets": routed_targets, + "cache_read_tokens": cache_read_tokens, + "cache_write_tokens": cache_write_tokens, + "secret_values_scanned": len(secret_values), + "secret_findings": findings, + } + args.output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + args.output.write_text(json.dumps(validation, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(validation, indent=2)) + return 0 if not benchmark_errors else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/verify_harbor_hermes_compat.py b/examples/harbor-hermes-switchyard/scripts/verify_harbor_hermes_compat.py new file mode 100755 index 000000000..92fede137 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/verify_harbor_hermes_compat.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Verify that the temporary agent is a narrow Harbor Hermes compatibility bridge.""" + +from __future__ import annotations + +import argparse +import ast +import importlib.metadata +import importlib.util +import json +import tempfile +from pathlib import Path + +from harbor.agents.installed.hermes import Hermes + + +def load_bridge(path: Path): + spec = importlib.util.spec_from_file_location("harbor_hermes_agent", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not import {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def run_mixed_mode_rejection(module, valid_config: Path) -> str: + text = valid_config.read_text(encoding="utf-8") + text += """ + +[[dynamic_plugins]] +plugin_id = "invalid.worker" +kind = "worker" +manifest_ref = "worker/relay-plugin.toml" +environment_ref = "worker-env" +""" + with tempfile.TemporaryDirectory(prefix="harbor-hermes-mixed-") as directory: + path = Path(directory) / "plugins.toml" + path.write_text(text, encoding="utf-8") + try: + module._validate_relay_config(path) + except ValueError as error: + return str(error) + raise AssertionError("mixed standard and worker plugin modes were accepted") + + +def verify_run_wrapper(source: str) -> None: + tree = ast.parse(source) + classes = [node for node in tree.body if isinstance(node, ast.ClassDef)] + bridge = next(node for node in classes if node.name == "HarborHermesAgent") + run_method = next( + node for node in bridge.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "run" + ) + super_run_calls = [ + node + for node in ast.walk(run_method) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "run" + and isinstance(node.func.value, ast.Call) + and isinstance(node.func.value.func, ast.Name) + and node.func.value.func.id == "super" + ] + if len(super_run_calls) != 1: + raise AssertionError("bridge run() must delegate exactly once to super().run()") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--bridge", type=Path, required=True) + parser.add_argument("--relay-config", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + module = load_bridge(args.bridge.resolve()) + bridge = module.HarborHermesAgent + if not issubclass(bridge, Hermes): + raise AssertionError("HarborHermesAgent must subclass Harbor's built-in Hermes") + if bridge._build_config_yaml is not Hermes._build_config_yaml: + raise AssertionError("bridge must inherit Harbor's Hermes config.yaml behavior") + if bridge.populate_context_post_run is not Hermes.populate_context_post_run: + raise AssertionError("bridge must inherit Harbor's ATIF conversion behavior") + verify_run_wrapper(args.bridge.read_text(encoding="utf-8")) + module._validate_relay_config(args.relay_config.resolve()) + mixed_error = run_mixed_mode_rejection(module, args.relay_config.resolve()) + + result = { + "schema_version": "harbor-hermes-switchyard.compatibility.v1", + "status": "passed", + "harbor_version": importlib.metadata.version("harbor"), + "bridge_base": "harbor.agents.installed.hermes.Hermes", + "inherited_contracts": [ + "task lifecycle", + "provider environment", + "prompt rendering", + "Hermes session export", + "Harbor ATIF conversion", + ], + "overrides": ["installation", "configuration staging", "artifact framing"], + "mixed_mode_rejection": mixed_error, + } + args.output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/supervise_phase2_cohort.sh b/examples/harbor-hermes-switchyard/supervise_phase2_cohort.sh new file mode 100755 index 000000000..7c9dbdae6 --- /dev/null +++ b/examples/harbor-hermes-switchyard/supervise_phase2_cohort.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -uo pipefail + +example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +run_root="${1:-}" +if [[ -z "$run_root" || "$run_root" != /* ]]; then + echo "usage: $0 /absolute/phase2-run-root [coordinator options]" >&2 + exit 2 +fi +shift + +child_pid="" +terminate_group() { + if [[ -z "$child_pid" ]]; then + return + fi + kill -TERM -- "-$child_pid" >/dev/null 2>&1 || true + for _ in {1..10}; do + if ! kill -0 -- "-$child_pid" >/dev/null 2>&1; then + return + fi + sleep 1 + done + kill -KILL -- "-$child_pid" >/dev/null 2>&1 || true +} +terminate() { + terminate_group + [[ -z "$child_pid" ]] || wait "$child_pid" >/dev/null 2>&1 || true + exit 143 +} +trap terminate INT TERM + +backoff_seconds="${TERMINAL_BENCH_SUPERVISOR_BACKOFF_SECONDS:-60}" +maximum_backoff_seconds="${TERMINAL_BENCH_SUPERVISOR_MAX_BACKOFF_SECONDS:-900}" +if [[ ! "$backoff_seconds" =~ ^[1-9][0-9]*$ || ! "$maximum_backoff_seconds" =~ ^[1-9][0-9]*$ ]]; then + echo "Phase 2 supervisor backoffs must be positive integers" >&2 + exit 2 +fi + +while true; do + python_bin="${EVAL_PYTHON:-$example_root/.venv/bin/python}" + "$python_bin" "$example_root/scripts/exec_process_group.py" \ + "$example_root/run_phase2_cohort.sh" "$run_root" "$@" & + child_pid=$! + wait "$child_pid" + status=$? + terminate_group + child_pid="" + case "$status" in + 0) + exit 0 + ;; + 20) + echo "[phase2-supervisor] stopping on preserved harness/integration blocker" >&2 + exit 20 + ;; + *) + echo "[phase2-supervisor] coordinator exited $status; resuming in ${backoff_seconds}s" >&2 + sleep "$backoff_seconds" + if ((backoff_seconds < maximum_backoff_seconds)); then + backoff_seconds=$((backoff_seconds * 2)) + if ((backoff_seconds > maximum_backoff_seconds)); then + backoff_seconds=$maximum_backoff_seconds + fi + fi + ;; + esac +done diff --git a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py new file mode 100644 index 000000000..fe41de1bc --- /dev/null +++ b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import importlib.util +import json +from pathlib import Path + +EXAMPLE_ROOT = Path(__file__).resolve().parents[1] + + +def load_finalizer(): + path = EXAMPLE_ROOT / "scripts" / "finalize_artifacts.py" + spec = importlib.util.spec_from_file_location("phase1_finalizer", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def make_args(tmp_path: Path, *, error_type: str = "") -> argparse.Namespace: + config = tmp_path / "plugins.toml" + config.write_text("version = 1\n", encoding="utf-8") + manifest = tmp_path / "relay-plugin.toml" + manifest.write_text('[plugin]\nid = "nvidia.switchyard"\n', encoding="utf-8") + library = tmp_path / "libswitchyard.so" + library.write_bytes(b"native-plugin-test") + return argparse.Namespace( + relay_config=config, + switchyard_manifest=manifest, + switchyard_library=library, + relay_wheel_sha256="a" * 64, + hermes_repository="https://github.com/bbednarski9/hermes-agent.git", + hermes_commit="a3d472f0e6bdc376df87b1436a461c4796db6747", + switchyard_commit="8daac03edf8544144833af1fd009b3da737715bc", + session_handle="phase1-session", + started_at=1.0, + error_type=error_type, + ) + + +def test_completed_response_is_preserved_after_post_response_failure(tmp_path: Path, monkeypatch) -> None: + module = load_finalizer() + root = tmp_path / "artifacts" + root.mkdir() + session = tmp_path / "hermes-session.jsonl" + session.write_text( + json.dumps( + { + "session_id": "phase1-session", + "messages": [ + {"role": "user", "content": "solve"}, + {"role": "assistant", "content": "completed answer"}, + ], + } + ) + + "\n", + encoding="utf-8", + ) + log = tmp_path / "hermes.txt" + log.write_text("normal shutdown\n", encoding="utf-8") + monkeypatch.setattr(module, "HERMES_SESSION", session) + monkeypatch.setattr(module, "HERMES_LOG", log) + monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.1") + + args = make_args(tmp_path, error_type="InjectedPostResponseFailure") + module.initialize(args, root) + module.complete(args, root) + + result = json.loads((root / "direct-hermes-result.json").read_text()) + receipt = json.loads((root / "direct-hermes-receipt.json").read_text()) + assert result["status"] == "preserved_completed_response" + assert result["final_response"] == "completed answer" + assert result["error"]["type"] == "InjectedPostResponseFailure" + assert receipt["cleanup"]["plugin_host_closed"] is True + assert receipt["cleanup"]["exporters_flushed"] is True + assert (root / ".complete").read_text() == "completed\n" + + +def test_no_response_never_creates_a_passed_completion(tmp_path: Path, monkeypatch) -> None: + module = load_finalizer() + root = tmp_path / "artifacts" + root.mkdir() + session = tmp_path / "hermes-session.jsonl" + session.write_text('{"messages": []}\n', encoding="utf-8") + log = tmp_path / "hermes.txt" + log.write_text("agent stopped\n", encoding="utf-8") + monkeypatch.setattr(module, "HERMES_SESSION", session) + monkeypatch.setattr(module, "HERMES_LOG", log) + monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.1") + + args = make_args(tmp_path) + module.initialize(args, root) + module.complete(args, root) + + completion = json.loads((root / "completion.json").read_text()) + assert completion["status"] == "failed" + + +def test_empty_session_uses_bounded_quiet_cli_output(tmp_path: Path, monkeypatch) -> None: + module = load_finalizer() + root = tmp_path / "artifacts" + root.mkdir() + session = tmp_path / "hermes-session.jsonl" + session.write_text("", encoding="utf-8") + log = tmp_path / "hermes.txt" + log.write_text("startup warning\n\nsession_id: cli-session\ncompleted\nanswer\n", encoding="utf-8") + monkeypatch.setattr(module, "HERMES_SESSION", session) + monkeypatch.setattr(module, "HERMES_LOG", log) + monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.1") + + args = make_args(tmp_path) + module.initialize(args, root) + module.complete(args, root) + + result = json.loads((root / "direct-hermes-result.json").read_text()) + assert result["status"] == "completed" + assert result["session_id"] == "cli-session" + assert result["final_response"] == "completed\nanswer" + + +def test_failed_agent_output_is_not_promoted_to_a_completed_response(tmp_path: Path, monkeypatch) -> None: + module = load_finalizer() + root = tmp_path / "artifacts" + root.mkdir() + session = tmp_path / "hermes-session.jsonl" + session.write_text("", encoding="utf-8") + log = tmp_path / "hermes.txt" + log.write_text( + "session_id: failed-session\nAPI call failed after 3 retries: provider returned HTTP 400\n", + encoding="utf-8", + ) + monkeypatch.setattr(module, "HERMES_SESSION", session) + monkeypatch.setattr(module, "HERMES_LOG", log) + monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.1") + + args = make_args(tmp_path, error_type="NonZeroAgentExitCodeError") + module.initialize(args, root) + module.complete(args, root) + + result = json.loads((root / "direct-hermes-result.json").read_text()) + completion = json.loads((root / "completion.json").read_text()) + assert result["status"] == "failed" + assert result["final_response"] is None + assert result["error"] == {"type": "NonZeroAgentExitCodeError", "phase": "agent"} + assert completion["status"] == "failed" diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py new file mode 100644 index 000000000..f8613855c --- /dev/null +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -0,0 +1,567 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import importlib.util +import json +import sys +from io import BytesIO +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +EXAMPLE_ROOT = Path(__file__).resolve().parents[1] + + +def load_coordinator(): + path = EXAMPLE_ROOT / "scripts" / "run_phase2_cohort.py" + spec = importlib.util.spec_from_file_location("phase2_coordinator", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def write_task(dataset: Path, name: str, memory: str) -> None: + task = dataset / name + task.mkdir(parents=True) + (task / "task.toml").write_text(f'[environment]\nmemory = "{memory}"\n', encoding="utf-8") + (task / "instruction.md").write_text(f"instruction for {name}\n", encoding="utf-8") + (task / "tests").mkdir() + (task / "tests" / "test.sh").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + + +def write_passed_attempt( + root: Path, + task, + *, + model: str, + cache_read: int, + benchmark_passed: bool, +) -> None: + attempt = root / "tasks" / task.directory_name / "attempts" / "001" + attempt.mkdir(parents=True) + summary = { + "status": "passed", + "validation": { + "status": "passed", + "benchmark_task_passed": benchmark_passed, + "direct_result_status": "completed", + "switchyard_decision_count": 2, + "routed_models": [model], + "routed_targets": ["strong"], + "cache_read_tokens": cache_read, + "cache_write_tokens": 0, + "secret_findings": [], + }, + "phoenix_upload": {"status": "passed", "uploaded_spans": 10}, + } + (attempt / "summary.json").write_text(json.dumps(summary), encoding="utf-8") + + +def cohort_args(root: Path) -> argparse.Namespace: + return argparse.Namespace( + run_root=root, + dataset="terminal-bench@2.0", + phoenix_project="phase2-project", + eval_cohort="phase2-cohort", + required_model=["sonnet", "opus"], + require_cache_hit=True, + ) + + +def test_task_discovery_places_explicit_canary_before_lexical_lane(tmp_path: Path) -> None: + module = load_coordinator() + write_task(tmp_path, "task-z", "4G") + write_task(tmp_path, "task-a", "2G") + tasks = module.discover_tasks(tmp_path, 2, set(), "task-z") + assert [(task.index, task.name, task.memory_gb) for task in tasks] == [ + (1, "task-z", 4), + (2, "task-a", 2), + ] + + +def test_task_discovery_without_a_canary_preserves_dataset_order(tmp_path: Path) -> None: + module = load_coordinator() + write_task(tmp_path, "task-z", "4G") + write_task(tmp_path, "task-a", "2G") + tasks = module.discover_tasks(tmp_path, 2, set(), None) + assert [(task.index, task.name, task.memory_gb) for task in tasks] == [ + (1, "task-a", 2), + (2, "task-z", 4), + ] + + +def test_failed_attempt_is_preserved_and_passed_attempt_wins(tmp_path: Path) -> None: + module = load_coordinator() + task = module.Task(1, "task", 2) + attempts = tmp_path / task.directory_name / "attempts" + failed = attempts / "001" + failed.mkdir(parents=True) + (failed / "summary.json").write_text('{"status":"failed"}', encoding="utf-8") + passed = attempts / "002" + passed.mkdir() + (passed / "summary.json").write_text( + json.dumps( + { + "status": "passed", + "validation": {"status": "passed"}, + "phoenix_upload": {"status": "passed"}, + } + ), + encoding="utf-8", + ) + assert module.successful_attempt(tmp_path / task.directory_name) == passed + assert failed.is_dir() + + +def test_cohort_summary_requires_completion_cache_routes_and_secret_scan(tmp_path: Path) -> None: + module = load_coordinator() + tasks = [module.Task(1, "one", 2), module.Task(2, "two", 2)] + write_passed_attempt(tmp_path, tasks[0], model="sonnet", cache_read=12, benchmark_passed=True) + write_passed_attempt(tmp_path, tasks[1], model="opus", cache_read=0, benchmark_passed=False) + summary = module.aggregate_summary(cohort_args(tmp_path), tasks) + assert summary["status"] == "passed" + assert summary["completed_tasks"] == 2 + assert summary["benchmark_pass_count"] == 1 + assert summary["benchmark_nonpass_count"] == 1 + assert summary["cohort_gates"]["cache_hit"]["cache_read_tokens"] == 12 + assert summary["cohort_gates"]["route_diversity"]["observed_models"] == ["opus", "sonnet"] + + +def test_cohort_summary_blocks_missing_route_even_when_tasks_pass(tmp_path: Path) -> None: + module = load_coordinator() + tasks = [module.Task(1, "one", 2)] + write_passed_attempt(tmp_path, tasks[0], model="opus", cache_read=12, benchmark_passed=True) + summary = module.aggregate_summary(cohort_args(tmp_path), tasks) + assert summary["status"] == "partial" + assert summary["cohort_gates"]["route_diversity"]["missing_models"] == ["sonnet"] + + +def test_integration_failure_does_not_erase_completed_benchmark_output(tmp_path: Path) -> None: + module = load_coordinator() + task = module.Task(1, "one", 2) + attempt = tmp_path / "tasks" / task.directory_name / "attempts" / "001" + attempt.mkdir(parents=True) + attempt_summary = { + "status": "passed", + "validation": { + "status": "passed", + "benchmark": {"status": "passed", "errors": []}, + "integration": { + "status": "failed", + "errors": ["missing route mark", "Phoenix upload did not pass"], + "warnings": [], + "phoenix_upload": {"status": "failed"}, + }, + "benchmark_task_passed": True, + "routed_models": ["sonnet"], + "routed_targets": ["weak"], + "cache_read_tokens": 12, + "secret_findings": [], + }, + "phoenix_upload": {"status": "failed", "error": "Phoenix upload command failed"}, + } + (attempt / "summary.json").write_text(json.dumps(attempt_summary), encoding="utf-8") + + assert module.task_summary_passed(attempt / "summary.json") is True + args = cohort_args(tmp_path) + args.required_model = ["sonnet"] + summary = module.aggregate_summary(args, [task]) + assert summary["completed_tasks"] == 1 + assert summary["tasks"][0]["status"] == "completed" + assert summary["cohort_gates"]["integration_validation"] == { + "passed": False, + "failed_task_count": 1, + "failures": [ + {"task": "one", "errors": ["missing route mark", "Phoenix upload did not pass"]} + ], + } + assert summary["status"] == "partial" + + +def test_failure_classifier_retries_only_known_infrastructure_failures() -> None: + module = load_coordinator() + assert module.classify_failure("TLS handshake timeout contacting registry-1.docker.io") == "infrastructure" + assert module.classify_failure("ConnectError: Error getting dataset terminal-bench@2.0") == "infrastructure" + assert module.classify_failure("Command failed (exit 100): apt-get update && apt-get install") == "infrastructure" + assert ( + module.classify_failure( + "Provider has been unresponsive (no response received) for 11 consecutive stale attempts" + ) + == "infrastructure" + ) + assert module.classify_failure("trusted fallback: provider returned HTTP 408") == "infrastructure" + assert module.classify_failure("provider returned HTTP 400") == "harness_or_integration" + assert module.classify_failure("Command failed (exit 137): agent process") == "infrastructure" + assert module.classify_failure("receipt did not prove plugin close") == "harness_or_integration" + + +def test_failure_classifier_reads_nested_harbor_logs(tmp_path: Path) -> None: + module = load_coordinator() + attempt = tmp_path / "attempt" + nested = attempt / "jobs" / "task" / "trial.log" + nested.parent.mkdir(parents=True) + nested.write_text( + 'failed to do request: Head "https://registry-1.docker.io/v2/library/debian/manifests/13.0-slim": ' + "context deadline exceeded\n", + encoding="utf-8", + ) + + assert module.classify_attempt_failure("expected one direct Hermes result, found 0", attempt) == "infrastructure" + + +def test_failure_classifier_reads_nested_harbor_result_json(tmp_path: Path) -> None: + module = load_coordinator() + attempt = tmp_path / "attempt" + result = attempt / "jobs" / "task" / "result.json" + result.parent.mkdir(parents=True) + result.write_text( + json.dumps( + { + "exception_info": { + "exception_message": ( + "Provider has been unresponsive (no response received) " + "for 10 consecutive stale attempts" + ) + } + } + ), + encoding="utf-8", + ) + + assert module.classify_attempt_failure("invalid direct result status", attempt) == "infrastructure" + + +def test_smoke_evidence_is_bound_to_exact_local_dataset(tmp_path: Path) -> None: + module = load_coordinator() + dataset = tmp_path / "dataset" + write_task(dataset, "task-a", "2G") + write_task(dataset, "task-b", "4G") + task_tomls = sorted(dataset.glob("*/task.toml")) + relay_wheel = tmp_path / "relay.whl" + relay_wheel.write_bytes(b"relay") + bundle = tmp_path / "bundle" + bundle.mkdir() + library = bundle / "libswitchyard_nemo_relay_plugin.so" + library.write_bytes(b"switchyard") + plugin_template = EXAMPLE_ROOT / "config" / "plugins.toml.in" + evidence_path = tmp_path / "smoke.json" + evidence = { + "schema_version": "harbor-hermes-switchyard.phase2-smoke.v1", + "status": "passed", + "task_count": 2, + "dataset_task_definitions_sha256": module.sha256_file_set(dataset, task_tomls), + "registry_network_attempts": 0, + "concurrency": 3, + "relay_architecture": "aarch64", + "relay_runtime": { + "status": "passed", + "relay_wheel_sha256": module.sha256_file(relay_wheel), + "switchyard_library_sha256": module.sha256_file(library), + "plugin_config_template_sha256": module.sha256_file(plugin_template), + }, + "tasks": [ + { + "name": name, + "instruction_path": f"{name}/instruction.md", + "verifier_path": f"{name}/tests/test.sh", + "task_toml_sha256": module.sha256_file(dataset / name / "task.toml"), + "instruction_sha256": module.sha256_file(dataset / name / "instruction.md"), + "test_sha256": module.sha256_file(dataset / name / "tests" / "test.sh"), + } + for name in ("task-a", "task-b") + ], + } + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + module.validate_smoke_evidence(evidence_path, 2, dataset, 3, "aarch64", relay_wheel, bundle, plugin_template) + + (dataset / "task-b" / "task.toml").write_text('[environment]\nmemory = "8G"\n', encoding="utf-8") + try: + module.validate_smoke_evidence(evidence_path, 2, dataset, 3, "aarch64", relay_wheel, bundle, plugin_template) + except ValueError: + pass + else: + raise AssertionError("stale smoke evidence accepted a changed dataset") + + (dataset / "task-b" / "task.toml").write_text('[environment]\nmemory = "4G"\n', encoding="utf-8") + (dataset / "task-a" / "instruction.md").write_text("changed instruction\n", encoding="utf-8") + try: + module.validate_smoke_evidence(evidence_path, 2, dataset, 3, "aarch64", relay_wheel, bundle, plugin_template) + except ValueError: + pass + else: + raise AssertionError("stale smoke evidence accepted a changed instruction") + (dataset / "task-a" / "instruction.md").write_text("instruction for task-a\n", encoding="utf-8") + + try: + module.validate_smoke_evidence(evidence_path, 2, dataset, 4, "aarch64", relay_wheel, bundle, plugin_template) + except ValueError: + pass + else: + raise AssertionError("smoke evidence accepted changed concurrency") + + +def test_durable_supervisor_owns_the_coordinator_process_group() -> None: + supervisor = (EXAMPLE_ROOT / "supervise_phase2_cohort.sh").read_text(encoding="utf-8") + assert "scripts/exec_process_group.py" in supervisor + assert 'kill -TERM -- "-$child_pid"' in supervisor + assert 'kill -KILL -- "-$child_pid"' in supervisor + + +def test_tmux_launcher_projects_only_the_protected_file_path() -> None: + launcher = (EXAMPLE_ROOT / "scripts" / "launch_phase2_tmux.sh").read_text(encoding="utf-8") + child = (EXAMPLE_ROOT / "scripts" / "run_phase2_from_env.sh").read_text(encoding="utf-8") + assert 'env_file="$example_root/.env"' in launcher + assert 'usage: $0 [env-file] tmux-session-name' in launcher + assert '-e "TERMINAL_BENCH_ENV_FILE=$env_file"' in launcher + assert 'source "$env_file"' not in launcher + assert "tmux has-session" in launcher + assert 'source "$env_file"' in child + assert "set +x" in child + assert "stage_phase2_runtime.py" in child + assert 'runtime_harness="$TERMINAL_BENCH_RUN_ROOT/runtime-harness"' in child + assert 'exec "$runtime_harness/supervise_phase2_cohort.sh"' in child + assert "supervisor.log" in child + + +def test_runtime_snapshot_remains_bound_after_checkout_changes(tmp_path: Path) -> None: + import importlib.util + + script = EXAMPLE_ROOT / "scripts" / "stage_phase2_runtime.py" + spec = importlib.util.spec_from_file_location("stage_phase2_runtime", script) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + source = tmp_path / "source" + for relative in ("agents", "config", "scripts"): + (source / relative).mkdir(parents=True) + for relative in module.RUNTIME_TOP_LEVEL: + (source / relative).write_text(f"{relative}\n", encoding="utf-8") + (source / "scripts" / "helper.py").write_text("VALUE = 1\n", encoding="utf-8") + digest = module.runtime_digest(source, module.runtime_files(source)) + plan = tmp_path / "plan.json" + plan.write_text(json.dumps({"inputs": {"runtime_sources_sha256": digest}}), encoding="utf-8") + destination = tmp_path / "runtime-harness" + + assert module.stage_runtime(source, destination, plan)["status"] == "staged" + (source / "scripts" / "helper.py").write_text("VALUE = 2\n", encoding="utf-8") + assert module.stage_runtime(source, destination, plan)["status"] == "verified" + assert (destination / "scripts" / "helper.py").read_text(encoding="utf-8") == "VALUE = 1\n" + + +def test_phase2_launcher_is_local_dataset_only() -> None: + launcher = (EXAMPLE_ROOT / "run_phase2_cohort.sh").read_text(encoding="utf-8") + assert 'dataset_root="${TBENCH_DATASET_PATH:-$dataset_export_root/$dataset_name}"' in launcher + assert "datasets download" not in launcher + assert "never downloads or resolves a dataset through the Harbor registry" in launcher + assert '--smoke-evidence "$smoke_evidence"' in launcher + assert '--offline-evidence "$offline_evidence"' in launcher + assert "PHASE1_EVIDENCE_ROOT" not in launcher + assert "INFERENCE_SECRETS_FILE" not in launcher + + +def test_coordinator_owns_bounded_force_build_setup_lane( + tmp_path: Path, +) -> None: + module = load_coordinator() + args = argparse.Namespace( + run_root=tmp_path / "run", + python_bin=tmp_path / "python", + setup_admission_runner=tmp_path / "run_setup_admission.py", + dataset_root=tmp_path / "dataset", + setup_runtime=tmp_path / "setup-runtime", + hermetic_runtime=tmp_path / "hermetic-runtime", + harbor_bin=tmp_path / "harbor", + setup_concurrency=2, + setup_batch_size=89, + setup_max_infra_attempts=4, + backoff_seconds=0, + hermetic_runtime_payload={"content_sha256": "a" * 64}, + ) + args.run_root.mkdir() + captured: list[str] = [] + + def fake_run(command: list[str], **_: object) -> SimpleNamespace: + captured.extend(command) + output = Path(command[command.index("--output") + 1]) + output.mkdir(parents=True) + (output / "summary.json").write_text( + json.dumps({"status": "passed", "planned": 2, "passed": 2}), + encoding="utf-8", + ) + return SimpleNamespace(returncode=0) + + tasks = [module.Task(1, "one", 2), module.Task(2, "two", 2)] + with patch.object(module.subprocess, "run", side_effect=fake_run): + assert module.CohortRunner(args, tasks).provision_environments() + assert captured[captured.index("--concurrency") + 1] == "2" + assert captured[captured.index("--batch-size") + 1] == "89" + assert "--force-build" in captured + assert "--no-preserve-containers" in captured + rendered = " ".join(captured) + assert "SWITCHYARD_PROVIDER_AUTHORIZATION" not in rendered + assert "Bearer " not in rendered + + +def test_provider_attempt_rebuilds_from_cached_layers_and_uses_hermetic_runtime() -> None: + runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") + coordinator = (EXAMPLE_ROOT / "scripts" / "run_phase2_cohort.py").read_text(encoding="utf-8") + assert 'HARBOR_FORCE_BUILD": "true"' in coordinator + assert 'HERMETIC_RUNTIME_DIR": str(self.args.hermetic_runtime)' in coordinator + assert 'harbor_force_build="${HARBOR_FORCE_BUILD:-true}"' in runner + assert "harbor_build_args+=(--force-build)" in runner + assert '--ak "hermetic_runtime_sha256=$hermetic_runtime_sha256"' in runner + assert '"target": "/opt/hermes-runtime"' in runner + assert "--ae 'OPENROUTER_API_KEY=relay-intercepted'" in runner + assert '--ae "OPENROUTER_BASE_URL=$fail_closed_openai_base_url"' in runner + + +def test_all_task_smoke_fail_closes_parent_and_delegated_provider_urls() -> None: + smoke = (EXAMPLE_ROOT / "scripts" / "smoke_phase2_dataset.py").read_text(encoding="utf-8") + assert '"OPENAI_BASE_URL": "http://127.0.0.1:9/v1"' in smoke + assert '"OPENROUTER_API_KEY": "relay-intercepted"' in smoke + assert '"OPENROUTER_BASE_URL": "http://127.0.0.1:9/v1"' in smoke + + +def test_runner_does_not_expand_an_empty_validation_expectations_array() -> None: + runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") + assert "validation_expectations=()" not in runner + assert '"${validation_expectations[@]}"' not in runner + assert "validation_args+=(--expect-late-failure)" in runner + + +def test_runner_keeps_benchmark_completion_separate_from_integration_validation() -> None: + runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") + assert 'summary["benchmark_completion"] = summary["validation"].get("benchmark", {})' in runner + assert 'integration["phoenix_upload"] = summary["phoenix_upload"]' in runner + assert 'summary["integration_validation"] = integration' in runner + assert "benchmark completion did not pass" in runner + assert "benchmark completion or Phoenix upload did not pass" not in runner + + +def test_plugin_contract_owns_routes_and_authorization_name() -> None: + module = load_coordinator() + contract = module.plugin_contract(EXAMPLE_ROOT / "config" / "plugins.toml.in") + assert contract["strong_model"] == "aws/anthropic/bedrock-claude-opus-5" + assert contract["weak_model"] == "aws/anthropic/bedrock-claude-sonnet-5" + assert contract["judge_model"] == "aws/anthropic/bedrock-claude-sonnet-4-6" + assert contract["hermes_caller_model"] == "ollama-route-stub" + assert contract["provider_base_urls"] == ["https://inference-api.nvidia.com/v1"] + + +def test_provider_catalog_requires_every_configured_route_without_persisting_authorization() -> None: + module = load_coordinator() + + class Response(BytesIO): + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + models = ["provider/sonnet", "provider/opus"] + response = Response(json.dumps({"data": [{"id": model} for model in models]}).encode()) + with patch.object(module.urllib.request, "urlopen", return_value=response) as request: + assert module.verify_provider_catalog("https://provider.example/v1", "Bearer secret", models) == sorted(models) + projected = request.call_args.args[0] + assert projected.full_url == "https://provider.example/v1/models" + assert projected.headers["Authorization"] == "Bearer secret" + + missing = Response(json.dumps({"data": [{"id": models[0]}]}).encode()) + with patch.object(module.urllib.request, "urlopen", return_value=missing): + try: + module.verify_provider_catalog("https://provider.example/v1", "Bearer secret", models) + except RuntimeError as error: + assert models[1] in str(error) + assert "Bearer secret" not in str(error) + else: + raise AssertionError("provider catalog accepted a missing configured model") + + +def test_capacity_requirement_covers_parallel_lane_and_largest_serial_task() -> None: + module = load_coordinator() + args = argparse.Namespace(concurrency=6, parallel_max_memory_gb=2, docker_memory_reserve_gb=4) + tasks = [module.Task(1, "canary", 2), module.Task(2, "large", 8)] + assert module.capacity_requirement_gb(args, tasks) == 16 + args.concurrency = 2 + assert module.capacity_requirement_gb(args, tasks) == 12 + assert module.normalize_architecture("amd64") == "x86_64" + assert module.normalize_architecture("arm64") == "aarch64" + + +def test_preflight_hard_rejects_capacity_above_docker_memory(tmp_path: Path) -> None: + module = load_coordinator() + args = argparse.Namespace( + run_root=tmp_path / "run", + minimum_free_gb=100, + concurrency=4, + parallel_max_memory_gb=2, + docker_memory_reserve_gb=4, + relay_architecture="aarch64", + ) + docker_info = json.dumps({"NCPU": 8, "MemTotal": 11 * 1024**3, "Architecture": "arm64"}) + completed = SimpleNamespace(stdout=docker_info) + tasks = [module.Task(1, "canary", 2)] + with ( + patch.object(module.shutil, "disk_usage", return_value=SimpleNamespace(free=200 * 1024**3)), + patch.object(module.subprocess, "run", return_value=completed), + ): + try: + module.shared_preflight(args, tasks) + except RuntimeError as error: + assert "requires 12 GiB" in str(error) + else: + raise AssertionError("unsafe Docker memory capacity was accepted") + + +def test_existing_plan_is_immutable(tmp_path: Path) -> None: + module = load_coordinator() + path = tmp_path / "plan.json" + module.load_or_create_plan(path, {"concurrency": 4}) + module.load_or_create_plan(path, {"concurrency": 4}) + try: + module.load_or_create_plan(path, {"concurrency": 6}) + except ValueError: + pass + else: + raise AssertionError("existing plan accepted changed concurrency") + + +def test_offline_evidence_is_bound_to_runtime_inputs(tmp_path: Path) -> None: + module = load_coordinator() + relay_wheel = tmp_path / "relay.whl" + relay_wheel.write_bytes(b"relay") + bundle = tmp_path / "bundle" + bundle.mkdir() + library = bundle / "libswitchyard_nemo_relay_plugin.so" + library.write_bytes(b"switchyard") + plugin_template = EXAMPLE_ROOT / "config" / "plugins.toml.in" + evidence_path = tmp_path / "offline.json" + evidence = { + "schema_version": "harbor-hermes-switchyard.phase2-offline-admission.v1", + "status": "passed", + "hermes_commit": module.EXPECTED_HERMES_COMMIT, + "relay_architecture": "x86_64", + "relay_wheel_sha256": module.sha256_file(relay_wheel), + "switchyard_library_sha256": module.sha256_file(library), + "plugin_config_template_sha256": module.sha256_file(plugin_template), + "provider_requests": 4, + "surviving_shutdown_threads": [], + } + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + module.validate_offline_evidence(evidence_path, "x86_64", relay_wheel, bundle, plugin_template) + evidence["relay_architecture"] = "aarch64" + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + try: + module.validate_offline_evidence(evidence_path, "x86_64", relay_wheel, bundle, plugin_template) + except ValueError: + pass + else: + raise AssertionError("offline evidence accepted a changed architecture") diff --git a/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py b/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py new file mode 100644 index 000000000..a433eb699 --- /dev/null +++ b/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import ast +from pathlib import Path + +EXAMPLE_ROOT = Path(__file__).resolve().parents[1] +BRIDGE = EXAMPLE_ROOT / "agents" / "harbor_hermes_agent.py" + + +def bridge_class() -> ast.ClassDef: + tree = ast.parse(BRIDGE.read_text(encoding="utf-8")) + return next(node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "HarborHermesAgent") + + +def method(name: str) -> ast.AsyncFunctionDef: + node = next(item for item in bridge_class().body if isinstance(item, ast.AsyncFunctionDef) and item.name == name) + return node + + +def super_calls(node: ast.AST, attribute: str) -> list[ast.Call]: + return [ + item + for item in ast.walk(node) + if isinstance(item, ast.Call) + and isinstance(item.func, ast.Attribute) + and item.func.attr == attribute + and isinstance(item.func.value, ast.Call) + and isinstance(item.func.value.func, ast.Name) + and item.func.value.func.id == "super" + ] + + +def test_bridge_delegates_exactly_once_to_each_inherited_lifecycle_phase() -> None: + assert len(super_calls(method("setup"), "setup")) == 1 + assert len(super_calls(method("run"), "run")) == 1 + + +def test_run_frames_artifacts_in_finally_after_inherited_run() -> None: + run = method("run") + tries = [item for item in run.body if isinstance(item, ast.Try)] + assert len(tries) == 1 + lifecycle = tries[0] + assert super_calls(lifecycle, "run") + assert any( + isinstance(item, ast.Call) and isinstance(item.func, ast.Attribute) and item.func.attr == "exec_as_agent" + for final in lifecycle.finalbody + for item in ast.walk(final) + ) + + +def test_install_verifies_detached_commit_and_relay_release() -> None: + source = ast.unparse(method("install")) + assert "checkout --detach" in source + assert "rev-parse HEAD" in source + assert "/tmp/hermes-install-path/ffmpeg" in source + assert "uv sync --frozen --extra all" in source + assert "m.version('nemo-relay').split('.')" in source + + +def test_setup_uploads_finalizer_with_its_relay_version_dependency() -> None: + uploads = { + (ast.unparse(call.args[0]), ast.literal_eval(call.args[1])) + for call in ast.walk(method("setup")) + if isinstance(call, ast.Call) + and isinstance(call.func, ast.Attribute) + and call.func.attr == "upload_file" + and len(call.args) == 2 + and isinstance(call.args[1], ast.Constant) + } + assert ("self._finalizer_path", "/installed-agent/finalize_artifacts.py") in uploads + assert ("self._relay_version_path", "/installed-agent/relay_version.py") in uploads diff --git a/examples/harbor-hermes-switchyard/tests/test_setup_admission.py b/examples/harbor-hermes-switchyard/tests/test_setup_admission.py new file mode 100644 index 000000000..e0cc5f5d2 --- /dev/null +++ b/examples/harbor-hermes-switchyard/tests/test_setup_admission.py @@ -0,0 +1,382 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import subprocess +import tomllib +from pathlib import Path +from types import ModuleType + +import pytest + +EXAMPLE_ROOT = Path(__file__).resolve().parents[1] + + +def load_module(name: str, path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +agent_module = load_module( + "setup_admission_agent", + EXAMPLE_ROOT / "agents" / "harbor_hermes_agent.py", +) +admission_module = load_module( + "setup_admission_runner", + EXAMPLE_ROOT / "scripts" / "run_setup_admission.py", +) +builder_module = load_module( + "hermetic_runtime_builder", + EXAMPLE_ROOT / "scripts" / "build_hermetic_runtime.py", +) +runtime_preparer_module = load_module( + "phase2_runtime_preparer", + EXAMPLE_ROOT / "scripts" / "prepare_runtime.py", +) + + +def make_payload(root: Path, *, digest: str = "a" * 64) -> dict[str, object]: + for relative in ( + "bin/hermes", + "bin/python", + "bin/uv", + ): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("stub", encoding="utf-8") + (root / "hermes-agent-src" / "venv").mkdir(parents=True) + ca_bundle = root / agent_module._HERMETIC_CA_BUNDLE_RELATIVE + ca_bundle.parent.mkdir(parents=True, exist_ok=True) + ca_bundle.write_text("test CA bundle", encoding="utf-8") + marker = { + "schema_version": agent_module._HERMETIC_RUNTIME_SCHEMA, + "status": "passed", + "content_sha256": digest, + "hermes_commit": "b" * 40, + "relay_wheel_sha256": "c" * 64, + "relay_architecture": "aarch64", + } + (root / "payload.json").write_text(json.dumps(marker), encoding="utf-8") + return marker + + +def test_hermetic_runtime_contract_accepts_bound_payload(tmp_path: Path) -> None: + marker = make_payload(tmp_path) + actual = agent_module._load_hermetic_runtime( + tmp_path, + expected_digest=str(marker["content_sha256"]), + hermes_commit=str(marker["hermes_commit"]), + relay_wheel_sha256=str(marker["relay_wheel_sha256"]), + relay_architecture=str(marker["relay_architecture"]), + ) + assert actual == marker + + +def test_hermetic_runtime_contract_rejects_changed_architecture(tmp_path: Path) -> None: + marker = make_payload(tmp_path) + with pytest.raises(ValueError, match="metadata mismatch"): + agent_module._load_hermetic_runtime( + tmp_path, + expected_digest=str(marker["content_sha256"]), + hermes_commit=str(marker["hermes_commit"]), + relay_wheel_sha256=str(marker["relay_wheel_sha256"]), + relay_architecture="x86_64", + ) + + +def test_hermetic_runtime_readiness_retries_nested_entrypoints(tmp_path: Path) -> None: + runtime = tmp_path / "runtime" + bin_dir = runtime / "bin" + bin_dir.mkdir(parents=True) + counter = tmp_path / "attempts" + (bin_dir / "python").write_text( + "#!/bin/sh\n" + f'counter="{counter}"\n' + 'attempts="$(cat "$counter" 2>/dev/null || printf 0)"\n' + 'attempts="$((attempts + 1))"\n' + 'printf "%s" "$attempts" > "$counter"\n' + '[ "$attempts" -ge 3 ]\n', + encoding="utf-8", + ) + (bin_dir / "hermes").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + os.chmod(bin_dir / "python", 0o755) + os.chmod(bin_dir / "hermes", 0o755) + command = agent_module._hermetic_runtime_readiness_command(str(runtime), attempts=4, delay_seconds=0) + completed = subprocess.run(["bash", "-c", f"set -euo pipefail; {command}"], check=False) + assert completed.returncode == 0 + assert counter.read_text(encoding="utf-8") == "3" + + +def test_setup_admission_binds_agent_source() -> None: + expected = EXAMPLE_ROOT / "agents" / "harbor_hermes_agent.py" + assert admission_module.SETUP_AGENT_PATH == expected + assert admission_module.sha256_file(expected) == agent_module._sha256(expected) + + +def test_bridge_accepts_current_classifier_routing_contract() -> None: + agent_module._validate_relay_config(EXAMPLE_ROOT / "config" / "plugins.toml.in") + + +def test_runtime_provenance_derives_classifier_target_from_plugin_config() -> None: + with (EXAMPLE_ROOT / "config" / "plugins.toml.in").open("rb") as stream: + config = tomllib.load(stream) + assert runtime_preparer_module.plugin_settings(config)["classifier_target"] == "judge" + + +def test_offline_overrides_keep_classifier_pricing_aliases_distinct(tmp_path: Path) -> None: + output = tmp_path / "plugins.toml" + settings = runtime_preparer_module.render_config( + EXAMPLE_ROOT / "config" / "plugins.toml.in", + output, + { + "HERMES_COMMIT": "a" * 40, + "OPENINFERENCE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", + "PHOENIX_PROJECT": "offline", + "EVAL_COHORT": "offline", + }, + { + "provider_base_url": "http://127.0.0.1:8000/v1", + "strong_model": "phase2/fake-strong", + "weak_model": "phase2/fake-weak", + "judge_model": "phase2/fake-judge", + }, + ) + with output.open("rb") as stream: + config = tomllib.load(stream) + entries = config["components"][0]["config"]["sources"][0]["catalog"]["entries"] + assert settings["judge_model"] == "phase2/fake-judge" + assert {entry["model_id"] for entry in entries} == { + "phase2/fake-strong", + "phase2/fake-weak", + "phase2/fake-judge", + } + + +def test_hermetic_runtime_requires_portable_ca_bundle(tmp_path: Path) -> None: + marker = make_payload(tmp_path) + (tmp_path / agent_module._HERMETIC_CA_BUNDLE_RELATIVE).unlink() + with pytest.raises(FileNotFoundError, match="hermetic runtime is incomplete"): + agent_module._load_hermetic_runtime( + tmp_path, + expected_digest=str(marker["content_sha256"]), + hermes_commit=str(marker["hermes_commit"]), + relay_wheel_sha256=str(marker["relay_wheel_sha256"]), + relay_architecture=str(marker["relay_architecture"]), + ) + + +def test_payload_tree_digest_ignores_its_marker(tmp_path: Path) -> None: + content = tmp_path / "bin" / "python" + content.parent.mkdir(parents=True) + content.write_text("payload", encoding="utf-8") + first = builder_module.sha256_tree(tmp_path) + (tmp_path / "payload.json").write_text("first", encoding="utf-8") + assert builder_module.sha256_tree(tmp_path) == first + (tmp_path / "payload.json").write_text("second", encoding="utf-8") + assert builder_module.sha256_tree(tmp_path) == first + content.write_text("changed", encoding="utf-8") + assert builder_module.sha256_tree(tmp_path) != first + + +def test_admission_rejects_tampered_hermetic_runtime(tmp_path: Path) -> None: + content = tmp_path / "bin" / "python" + content.parent.mkdir(parents=True) + content.write_text("payload", encoding="utf-8") + marker = { + "schema_version": admission_module.PAYLOAD_SCHEMA, + "status": "passed", + "content_sha256": admission_module.hermetic_content_sha256(tmp_path), + } + (tmp_path / "payload.json").write_text(json.dumps(marker), encoding="utf-8") + assert admission_module.load_payload(tmp_path) == marker + content.write_text("tampered", encoding="utf-8") + with pytest.raises(ValueError, match="does not match"): + admission_module.load_payload(tmp_path) + + +def test_payload_builder_forwards_non_secret_version_pins() -> None: + source = (EXAMPLE_ROOT / "scripts" / "build_hermetic_runtime.py").read_text(encoding="utf-8") + assert 'f"UV_VERSION={UV_VERSION}"' in source + assert 'f"PYTHON_VERSION={PYTHON_VERSION}"' in source + assert 'f"RELAY_WHEEL_NAME={relay_wheel.name}"' in source + + +def test_completed_result_is_invalidated_by_plan_input_change(tmp_path: Path) -> None: + plan = { + "inputs": {"concurrency": 4, "hermetic_runtime_sha256": "a" * 64}, + "tasks": [{"name": "task-one", "task_sha256": "b" * 64}], + } + binding = admission_module.task_bindings(plan)["task-one"] + results = tmp_path / "task-results" + results.mkdir() + (results / "task-one.json").write_text( + json.dumps( + { + "schema_version": admission_module.RESULT_SCHEMA, + "status": "passed", + "binding_sha256": binding, + } + ), + encoding="utf-8", + ) + assert admission_module.completed_names(tmp_path, plan) == {"task-one"} + plan["inputs"]["concurrency"] = 5 + assert admission_module.completed_names(tmp_path, plan) == set() + + +def test_job_result_import_keeps_newest_attempt(tmp_path: Path) -> None: + plan = { + "inputs": {"concurrency": 4}, + "tasks": [{"name": "task-one", "task_sha256": "b" * 64}], + } + for job_name, message in (("job-001", "old failure"), ("job-002", "new failure")): + trial = tmp_path / "jobs" / job_name / "trial-one" + trial.mkdir(parents=True) + (trial / "result.json").write_text( + json.dumps( + { + "task_name": "task-one", + "exception_info": { + "exception_type": "RuntimeError", + "exception_message": message, + }, + "environment_setup": None, + "agent_setup": None, + "agent_execution": None, + "verifier": None, + } + ), + encoding="utf-8", + ) + (tmp_path / "task-results").mkdir() + admission_module.parse_job_results(tmp_path, plan) + result = json.loads((tmp_path / "task-results" / "task-one.json").read_text()) + assert result["exception_message"] == "new failure" + + +def test_clock_preflight_rejects_remote_time_drift() -> None: + evidence = admission_module.evaluate_clock_preflight( + host_epoch=1_000.0, + docker_epoch=1_001.0, + reference_epoch=4_611.0, + ) + assert evidence["status"] == "failed" + assert evidence["host_reference_offset_seconds"] == 3_611.0 + assert evidence["docker_host_offset_seconds"] == 1.0 + + +def test_clock_preflight_accepts_small_offsets() -> None: + evidence = admission_module.evaluate_clock_preflight( + host_epoch=1_000.0, + docker_epoch=1_001.0, + reference_epoch=1_002.0, + ) + assert evidence["status"] == "passed" + + +def test_plugin_compatibility_uses_oldest_supported_base_without_secrets( + tmp_path: Path, +) -> None: + plan = { + "inputs": { + "relay_architecture": "aarch64", + "switchyard_bundle": str(tmp_path / "switchyard"), + } + } + command = admission_module.plugin_compatibility_command(plan) + assert "python:3.11-bullseye" in command + assert "linux/arm64" in command + assert "nemo_relay_register_plugin" in command[-1] + rendered = " ".join(command) + assert "SWITCHYARD_PROVIDER_AUTHORIZATION" not in rendered + assert "Bearer " not in rendered + + +def test_harbor_command_uses_install_only_without_provider_secret( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + output = tmp_path / "output" + output.mkdir() + payload = tmp_path / "payload" + payload.mkdir() + harbor = tmp_path / "harbor" + harbor.write_text("stub", encoding="utf-8") + args = argparse.Namespace( + harbor=harbor, + dataset=tmp_path / "dataset", + hermetic_runtime=payload, + output=output, + concurrency=6, + force_build=True, + preserve_containers=False, + ) + plan = { + "inputs": { + "hermes_commit": "b" * 40, + "relay_config": str(tmp_path / "plugins.toml"), + "switchyard_bundle": str(tmp_path / "switchyard"), + "relay_wheel": str(tmp_path / "relay.whl"), + "relay_wheel_sha256": "c" * 64, + "relay_architecture": "aarch64", + "hermetic_runtime_root": str(payload), + "hermetic_runtime_sha256": "d" * 64, + } + } + captured: list[str] = [] + + class Completed: + returncode = 0 + + def fake_run(command: list[str], **_: object) -> Completed: + captured.extend(command) + return Completed() + + monkeypatch.setattr(admission_module.subprocess, "run", fake_run) + assert admission_module.run_harbor(args, plan, ["one", "two"]) == 0 + assert "--install-only" in captured + assert "--disable-verification" in captured + assert "--force-build" in captured + assert "--no-delete" not in captured + assert captured.count("--include-task-name") == 2 + rendered = " ".join(captured) + assert "SWITCHYARD_PROVIDER_AUTHORIZATION" not in rendered + assert "provider-authorization" not in rendered + + +def test_setup_failure_classifies_transient_downloads_for_retry(tmp_path: Path) -> None: + root = tmp_path / "admission" + (root / "task-results").mkdir(parents=True) + result = root / "jobs" / "job" / "trial" / "result.json" + result.parent.mkdir(parents=True) + result.write_text( + json.dumps( + { + "task_name": "one", + "exception_info": { + "exception_type": "DockerBuildError", + "exception_message": "TLS handshake timeout contacting registry-1.docker.io", + }, + "environment_setup": None, + "agent_setup": None, + "agent_execution": None, + "verifier": None, + } + ), + encoding="utf-8", + ) + plan = { + "inputs": {"concurrency": 2}, + "tasks": [{"name": "one", "task_sha256": "a" * 64}], + } + admission_module.parse_job_results(root, plan) + parsed = json.loads(admission_module.result_path(root, "one").read_text()) + assert parsed["status"] == "failed" + assert parsed["failure_class"] == "infrastructure" diff --git a/examples/harbor-hermes-switchyard/tests/test_validation_contract.py b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py new file mode 100644 index 000000000..40a0fa884 --- /dev/null +++ b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py @@ -0,0 +1,227 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +EXAMPLE_ROOT = Path(__file__).resolve().parents[1] + + +def load_validator(): + path = EXAMPLE_ROOT / "scripts" / "validate_run.py" + spec = importlib.util.spec_from_file_location("phase1_validator", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_harbor_job_summary_is_not_a_trial_result() -> None: + module = load_validator() + assert module.is_trial_result({"n_total_trials": 1, "stats": {}}) is False + assert module.is_trial_result({"task_name": "task", "verifier_result": {}}) is True + + +def test_harbor_018_numeric_reward_is_normalized() -> None: + module = load_validator() + assert module.read_benchmark_passed({"verifier_result": {"rewards": {"reward": 0.0}}}) is False + assert module.read_benchmark_passed({"verifier_result": {"rewards": {"reward": 1.0}}}) is True + + +def test_verifier_backed_agent_timeout_is_a_completed_nonpass() -> None: + module = load_validator() + direct_result = { + "status": "failed", + "error": {"phase": "agent", "type": "CancelledError"}, + } + harbor_result = { + "exception_info": {"exception_type": "AgentTimeoutError"}, + "verifier_result": {"rewards": {"reward": 0.0}}, + } + assert module.is_verifier_backed_agent_timeout_nonpass(direct_result, harbor_result) is True + + harbor_result["exception_info"]["exception_type"] = "RuntimeError" + assert module.is_verifier_backed_agent_timeout_nonpass(direct_result, harbor_result) is False + + harbor_result["exception_info"]["exception_type"] = "AgentTimeoutError" + harbor_result["verifier_result"]["rewards"]["reward"] = 1.0 + assert module.is_verifier_backed_agent_timeout_nonpass(direct_result, harbor_result) is False + + +def test_harbor_timeout_multipliers_are_validated(tmp_path: Path) -> None: + module = load_validator() + config = { + "agent_timeout_multiplier": 3.0, + "agent_setup_timeout_multiplier": 6.0, + "environment_build_timeout_multiplier": 6.0, + } + (tmp_path / "config.json").write_text(json.dumps(config), encoding="utf-8") + observed, errors = module.validate_harbor_job_config(tmp_path) + assert observed == config + assert errors == [] + + config["agent_timeout_multiplier"] = 1.0 + (tmp_path / "config.json").write_text(json.dumps(config), encoding="utf-8") + _, errors = module.validate_harbor_job_config(tmp_path) + assert errors == ["Harbor job config agent_timeout_multiplier=1.0 does not match required 3.0"] + + +def test_atof_reader_extracts_switchyard_selected_targets(tmp_path: Path) -> None: + module = load_validator() + path = tmp_path / "trajectory.atof.jsonl" + events = [ + {"name": "switchyard.routing.requested", "data": {"algorithm": "llm_task_classifier"}}, + { + "name": "switchyard.routing.decision", + "data": {"selected_target": "weak", "routing_tier": "weak"}, + }, + { + "name": "switchyard.routing.decision", + "data": {"selected_target": "strong", "routing_tier": "strong"}, + }, + ] + path.write_text("\n".join(json.dumps(event) for event in events) + "\n") + count, marks, models, targets = module.read_atof(path) + assert count == 3 + assert marks == ["switchyard.routing.decision", "switchyard.routing.requested"] + assert models == [] + assert targets == ["strong", "weak"] + + +def test_atof_inspection_extracts_provider_models_and_cache_usage(tmp_path: Path) -> None: + module = load_validator() + path = tmp_path / "trajectory.atof.jsonl" + events = [ + { + "name": "switchyard.routing.decision", + "data": {"selected_target": "weak"}, + }, + { + "name": "openai.chat_completions", + "category_profile": {"annotated_response": {"model": "aws/anthropic/bedrock-claude-sonnet-4-6"}}, + "data": {"model": "aws/anthropic/bedrock-claude-sonnet-4-6"}, + }, + { + "name": "llm.chunk", + "data": {"usage": {"cache_read_tokens": 120, "cache_write_tokens": 30}}, + }, + ] + path.write_text("\n".join(json.dumps(event) for event in events) + "\n") + evidence = module.inspect_atof(path) + assert evidence == { + "count": 3, + "marks": ["switchyard.routing.decision"], + "models": ["aws/anthropic/bedrock-claude-sonnet-4-6"], + "targets": ["weak"], + "decision_count": 1, + "cache_read_tokens": 120, + "cache_write_tokens": 30, + } + + +def test_secret_scan_finds_raw_key_within_artifact(tmp_path: Path) -> None: + module = load_validator() + artifact = tmp_path / "artifact.log" + artifact.write_text("raw-provider-key") + assert module.scan_secrets([artifact], [b"Bearer raw-provider-key", b"raw-provider-key"]) == [ + "artifact.log:secret[1]" + ] + + +def test_receipt_provenance_validation_covers_every_staged_digest() -> None: + module = load_validator() + provenance = { + "nemo_relay": {"version": "0.7.1", "wheel_sha256": "relay"}, + "hermes": {"commit": "hermes"}, + "switchyard": { + "commit": "switchyard", + "manifest_sha256": "manifest", + "library_sha256": "library", + }, + "relay_config_sha256": "config", + } + receipt = { + "activation_mode": "relay_standard_dynamic", + "dynamic_plugin_ids": ["nvidia.switchyard"], + "relay_config_sha256": "config", + "dependencies": { + "nemo_relay": {"version": "0.7.1", "wheel_sha256": "relay"}, + "hermes": {"commit": "hermes"}, + "switchyard": { + "commit": "switchyard", + "manifest_sha256": "manifest", + "library_sha256": "library", + }, + }, + "routing_contract": { + "relay_outer_lifecycle": True, + "execution_intercept_owner": "nvidia.switchyard", + "provider_http_client_owner": "switchyard-llm-client", + "separate_switchyard_service": False, + }, + "cleanup": {"plugin_host_closed": True, "exporters_flushed": True}, + } + assert module.validate_receipt_provenance(receipt, provenance) == [] + receipt["dependencies"]["switchyard"]["library_sha256"] = "tampered" + assert module.validate_receipt_provenance(receipt, provenance) == [ + "Switchyard library digest does not match runtime provenance" + ] + + +def test_openinference_inspection_extracts_semantic_and_lineage_evidence(tmp_path: Path) -> None: + module = load_validator() + artifact = tmp_path / "openinference.jsonl" + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "openinference.project.name", "value": {"stringValue": "project"}}, + {"key": "evaluation.cohort", "value": {"stringValue": "cohort"}}, + ] + }, + "scopeSpans": [ + { + "scope": {"name": "harbor-hermes-switchyard"}, + "spans": [ + { + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "LLM"}}, + {"key": "nemo_relay.uuid", "value": {"stringValue": "uuid"}}, + {"key": "nemo_relay.scope_type", "value": {"stringValue": "llm"}}, + ] + }, + { + "parentSpanId": "parent-span", + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "CHAIN"}}, + {"key": "nemo_relay.mark.uuid", "value": {"stringValue": "mark-uuid"}}, + { + "key": "nemo_relay.mark.parent_uuid", + "value": {"stringValue": "parent-uuid"}, + }, + ], + }, + { + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "CHAIN"}}, + {"key": "nemo_relay.mark.uuid", "value": {"stringValue": "orphan-mark"}}, + ], + }, + ], + } + ], + } + ] + } + artifact.write_text(json.dumps(payload) + "\n", encoding="utf-8") + evidence = module.inspect_openinference(artifact) + assert evidence["documents"] == 1 + assert evidence["spans"] == 3 + assert evidence["span_kinds"] == ["CHAIN", "LLM"] + assert evidence["scope_names"] == ["harbor-hermes-switchyard"] + assert evidence["lineage_spans"] == 2 + assert evidence["resource_attributes"]["openinference.project.name"] == ["project"]