From a328f411a58bb15683e3971ca02cd3c332b74a3f Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 5 Aug 2026 13:32:20 -0700 Subject: [PATCH 01/14] test: add configurable coding-agent latency benchmark Signed-off-by: Yuchen Zhang --- .../maintain-coding-agent-benchmark/SKILL.md | 122 +++++++ .../agents/openai.yaml | 7 + docs/reference/performance.mdx | 88 +++++ justfile | 13 + scripts/README.md | 26 ++ scripts/benchmark-coding-agent-latency.py | 17 + .../__init__.py | 4 + .../benchmarks.py | 320 ++++++++++++++++++ scripts/benchmark_coding_agent_latency/cli.py | 125 +++++++ .../benchmark_coding_agent_latency/config.py | 291 ++++++++++++++++ .../data/agent-config.toml | 5 + .../data/default.toml | 23 ++ .../data/fake-codex.cmd | 12 + .../data/fake-codex.sh | 12 + .../data/plugins-file.toml | 20 ++ .../data/plugins-minimal.toml | 5 + .../data/plugins-otlp.toml | 18 + .../data/relay-config.toml | 2 + .../fixtures.py | 94 +++++ .../processes.py | 206 +++++++++++ .../protocol.py | 229 +++++++++++++ .../reporting.py | 76 +++++ .../benchmark_coding_agent_latency/servers.py | 153 +++++++++ .../test_benchmark_coding_agent_latency.py | 93 +++++ 24 files changed, 1961 insertions(+) create mode 100644 .agents/skills/maintain-coding-agent-benchmark/SKILL.md create mode 100644 .agents/skills/maintain-coding-agent-benchmark/agents/openai.yaml create mode 100644 scripts/benchmark-coding-agent-latency.py create mode 100644 scripts/benchmark_coding_agent_latency/__init__.py create mode 100644 scripts/benchmark_coding_agent_latency/benchmarks.py create mode 100644 scripts/benchmark_coding_agent_latency/cli.py create mode 100644 scripts/benchmark_coding_agent_latency/config.py create mode 100644 scripts/benchmark_coding_agent_latency/data/agent-config.toml create mode 100644 scripts/benchmark_coding_agent_latency/data/default.toml create mode 100644 scripts/benchmark_coding_agent_latency/data/fake-codex.cmd create mode 100755 scripts/benchmark_coding_agent_latency/data/fake-codex.sh create mode 100644 scripts/benchmark_coding_agent_latency/data/plugins-file.toml create mode 100644 scripts/benchmark_coding_agent_latency/data/plugins-minimal.toml create mode 100644 scripts/benchmark_coding_agent_latency/data/plugins-otlp.toml create mode 100644 scripts/benchmark_coding_agent_latency/data/relay-config.toml create mode 100644 scripts/benchmark_coding_agent_latency/fixtures.py create mode 100644 scripts/benchmark_coding_agent_latency/processes.py create mode 100644 scripts/benchmark_coding_agent_latency/protocol.py create mode 100644 scripts/benchmark_coding_agent_latency/reporting.py create mode 100644 scripts/benchmark_coding_agent_latency/servers.py create mode 100644 scripts/tests/test_benchmark_coding_agent_latency.py diff --git a/.agents/skills/maintain-coding-agent-benchmark/SKILL.md b/.agents/skills/maintain-coding-agent-benchmark/SKILL.md new file mode 100644 index 000000000..717c42af2 --- /dev/null +++ b/.agents/skills/maintain-coding-agent-benchmark/SKILL.md @@ -0,0 +1,122 @@ +--- +name: maintain-coding-agent-benchmark +description: Run, configure, troubleshoot, maintain, or expand the NeMo Relay coding-agent latency benchmark fixture. Use for changes under scripts/benchmark_coding_agent_latency, new benchmark suites or matrix axes, static provider and Relay fixtures, result reporting, or coding-agent benchmark documentation. +--- + +# Maintain The Coding-Agent Latency Benchmark + +## Companion Guidance + +Use `karpathy-guidelines` for implementation and `validate-change` to select +final checks. Keep benchmark changes isolated from runtime behavior. + +## Understand The Layout + +- Use `scripts/benchmark-coding-agent-latency.py` as the stable entry point. +- Read `scripts/benchmark_coding_agent_latency/data/default.toml` before a run. + A custom TOML file overlays these defaults, then CLI arguments take final + precedence. +- Change config parsing and validation in `config.py`. +- Keep OpenAI and Anthropic payload shapes in `protocol.py`. +- Keep loopback provider and OTLP behavior in `servers.py`. +- Keep temporary Relay and coding-agent process lifecycle in `processes.py`. +- Add measurement logic to `benchmarks.py`, orchestration to `cli.py`, and + terminal presentation to `reporting.py`. +- Put multi-line configs, scripts, and other fixed fixture text under `data/`. + Load or render those assets through `fixtures.py`; do not embed them in + executable modules. + +## Run The Fixture + +List every config override without building Relay: + +```bash +uv run python scripts/benchmark-coding-agent-latency.py --help +``` + +Run a small functional check before a statistically meaningful run: + +```bash +just benchmark-coding-agent-latency \ + --tests gateway \ + --providers openai \ + --modes buffered \ + --payload-sizes 4096 \ + --concurrency 1 \ + --samples 5 \ + --warmup 1 \ + --response-bytes 1024 +``` + +Treat a small run only as a correctness check. Run the default matrix with +`just benchmark-coding-agent-latency` when collecting performance data. The +default file-exporter matrix can write tens of gigabytes temporarily, so check +free disk space first. + +Use a partial TOML config for repeatable experiments: + +```toml +tests = ["gateway"] +providers = ["openai"] +modes = ["streaming"] +samples = 50 +payload_sizes = [4096, 65536] +concurrency = [1, 4] +``` + +```bash +just benchmark-coding-agent-latency --config /path/to/benchmark.toml +``` + +Find the JSON report at +`target/benchmark-results/coding-agent-latency.json` unless `output_dir` was +overridden. Compare added milliseconds and paired confidence intervals; do not +draw performance conclusions from a smoke run. + +## Maintain Measurement Integrity + +- Keep providers on loopback and deterministic. Do not add model-service or + Internet latency to the core fixture. +- Compare variants within the same measurement cycle and retain the rotated or + randomized execution order to reduce ordering bias. +- Warm persistent connections before recording gateway samples. +- Keep streaming time-to-first-content separate from total stream time. +- Preserve exporter-delivery checks when gateway or hook traffic is measured. +- Record all resolved matrix values in the JSON result so another engineer can + reproduce the run. +- Keep temporary state isolated from the developer's home and Relay config. + +## Expand The Fixture + +To add a test suite: + +1. Add its name to `AVAILABLE_TESTS` in `config.py`. +2. Implement the measurement in `benchmarks.py`. +3. Dispatch it conditionally in `cli.py` and report it conditionally in + `reporting.py`. +4. Add config and selection tests in + `scripts/tests/test_benchmark_coding_agent_latency.py`. +5. Update `scripts/README.md` and `docs/reference/performance.mdx`. + +To add a provider, mode, or matrix axis, update config validation, the protocol +fixture, the loopback server, orchestration, result parameters, and tests +together. Add static fixture files when the change introduces fixed text. + +## Validate Changes + +Format and test the focused surface first: + +```bash +uv run ruff format scripts/benchmark-coding-agent-latency.py \ + scripts/benchmark_coding_agent_latency \ + scripts/tests/test_benchmark_coding_agent_latency.py +uv run ruff check scripts/benchmark-coding-agent-latency.py \ + scripts/benchmark_coding_agent_latency \ + scripts/tests/test_benchmark_coding_agent_latency.py +uv run python -m unittest scripts.tests.test_benchmark_coding_agent_latency +``` + +Run the small functional command above after lifecycle, protocol, server, +fixture, or orchestration changes. It requires permission to bind loopback +ports in restricted environments. Run `just docs` when the performance page +changes and `uv run pre-commit run --all-files` before review. diff --git a/.agents/skills/maintain-coding-agent-benchmark/agents/openai.yaml b/.agents/skills/maintain-coding-agent-benchmark/agents/openai.yaml new file mode 100644 index 000000000..27a7f71f4 --- /dev/null +++ b/.agents/skills/maintain-coding-agent-benchmark/agents/openai.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +interface: + display_name: "Maintain Coding-Agent Benchmark" + short_description: "Run and extend the Relay latency fixture" + default_prompt: "Use $maintain-coding-agent-benchmark to run or extend the coding-agent latency benchmark." diff --git a/docs/reference/performance.mdx b/docs/reference/performance.mdx index 2ca497cb1..3cc2e8993 100644 --- a/docs/reference/performance.mdx +++ b/docs/reference/performance.mdx @@ -29,6 +29,94 @@ Use these practices when applying the concept in application or integration code - Use execution intercepts when you need to wrap real execution and sanitize guardrails when you only need to change emitted observability payloads. - Use binding-native typed wrappers and codecs when provider payload conversion would otherwise be repeated at many call sites. +## Coding-Agent Latency Benchmark + +Use the opt-in coding-agent benchmark to measure the local latency that Relay +adds around Codex and Claude Code traffic. The harness uses deterministic +loopback OpenAI and Anthropic providers so network and model-service latency do +not hide Relay's contribution. + +```bash +just benchmark-coding-agent-latency +``` + +The command builds the release CLI. Its default configuration measures these +paths: + +- Direct requests to the mock provider. +- Relay with no exporter, which isolates the managed gateway pipeline. +- Relay with a local ATOF file sink. +- Relay with a local OpenTelemetry HTTP receiver. +- Full `nemo-relay hook-forward` subprocesses for Codex and Claude Code. +- Cold Relay process startup through gateway readiness. + +Gateway scenarios cover OpenAI Responses and Anthropic Messages, buffered and +streaming responses, request payloads from 4 KiB through 4 MiB, and concurrency +levels 1, 2, 4, 8, and 16. +For buffered calls, inspect total latency. For streaming calls, inspect both +time to the first content delta and total stream time. The report includes +p50, p95, and p99 paired latency differences and a bootstrap 95% confidence +interval for the median. + +Configure a run with a partial TOML file. Unspecified values inherit from +`scripts/benchmark_coding_agent_latency/data/default.toml`: + +```toml +tests = ["gateway"] +providers = ["openai"] +modes = ["buffered", "streaming"] +samples = 20 +warmup = 1 +payload_sizes = [4096] +concurrency = [1, 4] +``` + +Pass the file to the benchmark: + +```bash +just benchmark-coding-agent-latency --config /path/to/benchmark.toml +``` + +Command-line values take precedence over the TOML file. Use comma-separated +values for list overrides. This example runs only the OpenAI streaming gateway +suite, regardless of the values in the file: + +```bash +just benchmark-coding-agent-latency \ + --config /path/to/benchmark.toml \ + --tests gateway \ + --providers openai \ + --modes streaming \ + --payload-sizes 4096,65536 \ + --concurrency 1,4 +``` + +The selectable suites are `gateway`, `hooks`, and `startup`. Run +`uv run python scripts/benchmark-coding-agent-latency.py --help` to see every +matrix and sample-count override. Results contain only the selected suite +sections. + +The command prints a readable summary and writes structured JSON to +`target/benchmark-results/coding-agent-latency.json`. Set the repository-wide +`output_dir` variable to choose another result location: + +```bash +just output_dir=/tmp/relay-benchmarks benchmark-coding-agent-latency +``` + +The local-file scenarios can write tens of gigabytes of temporary ATOF data +with the default matrix. The harness verifies exporter delivery for gateway and +hook runs and removes its temporary workspace after the run, but you should +confirm that the system has sufficient free disk space before starting it. + +Treat results as environment-specific. Record the commit, release build, +hardware, operating system, workload sizes, and sample counts when sharing a +number. Prefer added milliseconds over percentages: a small absolute increase +can look disproportionately large when the direct loopback baseline is much +faster than a real model call. Use real Codex or Claude Code runs as an +end-to-end validation, not as the primary gateway measurement, because host +startup and scheduling add unrelated variance. + ## Related Topics Use these links to continue into adjacent concepts and workflows. diff --git a/justfile b/justfile index 4d331e8c3..4d3b13f90 100644 --- a/justfile +++ b/justfile @@ -1112,6 +1112,19 @@ test-claude-plugin-e2e: test-hermes-mcp-e2e: ./scripts/test-hermes-mcp-e2e.sh +# Opt-in: builds the release CLI and runs configurable local latency suites. +benchmark-coding-agent-latency *benchmark_args: + #!/usr/bin/env bash + set -euo pipefail + result_dir={{ quote(output_dir) }} + result_dir="${result_dir:-target/benchmark-results}" + benchmark_args=({{ benchmark_args }}) + cargo build --release -p nemo-relay-cli + uv run python scripts/benchmark-coding-agent-latency.py \ + --relay-bin target/release/nemo-relay \ + --output "$result_dir/coding-agent-latency.json" \ + "${benchmark_args[@]}" + # --set [output_dir=] [ci=true|false] test-rust: #!/usr/bin/env bash diff --git a/scripts/README.md b/scripts/README.md index c8fce82ab..3aa43886c 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -26,6 +26,32 @@ These checks exercise installed coding-agent clients and are intentionally outsi - `just test-claude-plugin-e2e` - `just test-hermes-mcp-e2e` +## Opt-In Performance Benchmark + +Run `just benchmark-coding-agent-latency` to build the release CLI and compare +direct provider requests with Relay's minimal, local-file, and local-OTLP +configurations. The benchmark also measures full hook subprocess and cold +gateway startup time. It writes structured results under +`target/benchmark-results/` by default and is intentionally outside regular CI. + +The defaults live in +`scripts/benchmark_coding_agent_latency/data/default.toml`. Supply a partial +TOML file with `--config`, or override individual values on the command line. +For example, this runs only a small OpenAI gateway matrix: + +```bash +just benchmark-coding-agent-latency \ + --tests gateway \ + --providers openai \ + --payload-sizes 4096 \ + --concurrency 1 \ + --samples 10 +``` + +Run `uv run python scripts/benchmark-coding-agent-latency.py --help` to list +all overrides. The three selectable suites are `gateway`, `hooks`, and +`startup`. + ## Internal Layout - `docs/`: Fern reference-generation, migration cleanup, and `docs-website` branch sync helpers. Generated API reference output under `docs/reference/api/*-library-reference/` is ignored and recreated by `just docs`. diff --git a/scripts/benchmark-coding-agent-latency.py b/scripts/benchmark-coding-agent-latency.py new file mode 100644 index 000000000..63ec6cf8f --- /dev/null +++ b/scripts/benchmark-coding-agent-latency.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the configurable coding-agent latency benchmark.""" + +from __future__ import annotations + +import sys + +from benchmark_coding_agent_latency.cli import main + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("benchmark interrupted", file=sys.stderr) + raise SystemExit(130) from None diff --git a/scripts/benchmark_coding_agent_latency/__init__.py b/scripts/benchmark_coding_agent_latency/__init__.py new file mode 100644 index 000000000..6a4347ddd --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Coding-agent latency benchmark fixture.""" diff --git a/scripts/benchmark_coding_agent_latency/benchmarks.py b/scripts/benchmark_coding_agent_latency/benchmarks.py new file mode 100644 index 000000000..5e3e47d69 --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/benchmarks.py @@ -0,0 +1,320 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Gateway, hook, and process-startup benchmark suites.""" + +from __future__ import annotations + +import concurrent.futures +import contextlib +import http.client +import json +import math +import random +import subprocess +import threading +import time +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +from .fixtures import isolated_environment +from .processes import RelayProcess, TransparentRelayProcess +from .protocol import make_request, request_headers, request_path +from .servers import connection_for + +VARIANTS = ("direct", "relay-minimal", "relay-file", "relay-otlp") +RELAY_VARIANTS = ("relay-minimal", "relay-file", "relay-otlp") + + +def percentile(values: Sequence[int | float], fraction: float) -> float: + """Return a linearly interpolated percentile for a non-empty sample.""" + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1 - weight) + ordered[upper] * weight + + +def summarize_ns(values: list[int]) -> dict[str, Any]: + milliseconds = [value / 1_000_000 for value in values] + return { + "samples": len(values), + "p50_ms": round(percentile(milliseconds, 0.50), 6), + "p95_ms": round(percentile(milliseconds, 0.95), 6), + "p99_ms": round(percentile(milliseconds, 0.99), 6), + "min_ms": round(min(milliseconds), 6), + "max_ms": round(max(milliseconds), 6), + } + + +def median_confidence_interval_ns(values: list[int], *, seed: int, resamples: int = 1_000) -> list[float]: + """Return a deterministic bootstrap 95% confidence interval for the median.""" + randomizer = random.Random(seed) + medians = [] + for _ in range(resamples): + sample = [values[randomizer.randrange(len(values))] for _ in values] + medians.append(percentile(sample, 0.50) / 1_000_000) + return [ + round(percentile(medians, 0.025), 6), + round(percentile(medians, 0.975), 6), + ] + + +def perform_request( + connection: http.client.HTTPConnection, + provider: str, + body: bytes, + streaming: bool, +) -> dict[str, int]: + started = time.perf_counter_ns() + connection.request("POST", request_path(provider), body=body, headers=request_headers(provider)) + response = connection.getresponse() + if response.status != 200: + details = response.read().decode(errors="replace") + raise RuntimeError(f"benchmark request failed with HTTP {response.status}: {details}") + if not streaming: + response.read() + return {"total_ns": time.perf_counter_ns() - started} + first_content_ns = 0 + while True: + line = response.readline() + if not line: + break + if not line.startswith(b"data:"): + continue + payload = line[5:].strip() + if not payload or payload == b"[DONE]": + continue + event = json.loads(payload) + if event.get("type") in {"response.output_text.delta", "content_block_delta"}: + first_content_ns = first_content_ns or time.perf_counter_ns() - started + if first_content_ns == 0: + raise RuntimeError("stream ended before a content delta was received") + return { + "first_content_ns": first_content_ns, + "total_ns": time.perf_counter_ns() - started, + } + + +def benchmark_scenario( + urls: dict[str, str], + *, + provider: str, + model: str, + request_fill: str, + streaming: bool, + payload_bytes: int, + samples: int, + warmup: int, + concurrency: int, +) -> dict[str, Any]: + body = make_request( + provider, + streaming, + payload_bytes, + model=model, + request_fill=request_fill, + ) + observations: list[dict[str, dict[str, int]]] = [] + observation_lock = threading.Lock() + barrier = threading.Barrier(concurrency) + + def worker(worker_id: int, indices: list[int]) -> None: + connections = {name: connection_for(url) for name, url in urls.items()} + try: + for _ in range(warmup): + for name in VARIANTS: + perform_request(connections[name], provider, body, streaming) + barrier.wait() + local = [] + for index in indices: + order = list(VARIANTS) + shift = (index + worker_id) % len(order) + order = order[shift:] + order[:shift] + local.append({name: perform_request(connections[name], provider, body, streaming) for name in order}) + with observation_lock: + observations.extend(local) + finally: + for connection in connections.values(): + connection.close() + + assignments = [list(range(worker_id, samples, concurrency)) for worker_id in range(concurrency)] + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = [executor.submit(worker, worker_id, indices) for worker_id, indices in enumerate(assignments)] + for future in futures: + future.result() + + metrics = ("total_ns",) if not streaming else ("first_content_ns", "total_ns") + absolute = { + name: { + metric.removesuffix("_ns"): summarize_ns([cycle[name][metric] for cycle in observations]) + for metric in metrics + } + for name in VARIANTS + } + comparison_pairs = { + "relay-minimal_vs_direct": ("relay-minimal", "direct"), + "relay-file_vs_direct": ("relay-file", "direct"), + "relay-otlp_vs_direct": ("relay-otlp", "direct"), + "file_exporter_vs_minimal": ("relay-file", "relay-minimal"), + "otlp_exporter_vs_minimal": ("relay-otlp", "relay-minimal"), + } + comparisons = {} + for comparison, (left, right) in comparison_pairs.items(): + comparisons[comparison] = {} + for metric_index, metric in enumerate(metrics): + deltas = [cycle[left][metric] - cycle[right][metric] for cycle in observations] + summary = summarize_ns(deltas) + summary["median_ci95_ms"] = median_confidence_interval_ns( + deltas, + seed=payload_bytes + concurrency * 101 + metric_index * 10_007, + ) + comparisons[comparison][metric.removesuffix("_ns")] = summary + return { + "provider": provider, + "mode": "streaming" if streaming else "buffered", + "payload_bytes": payload_bytes, + "serialized_request_bytes": len(body), + "concurrency": concurrency, + "absolute": absolute, + "comparisons": comparisons, + } + + +def run_subprocess_timed(command: list[str], *, root: Path, input_bytes: bytes | None = None) -> int: + started = time.perf_counter_ns() + result = subprocess.run( + command, + cwd=root, + env=isolated_environment(root), + input=input_bytes, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + elapsed = time.perf_counter_ns() - started + if result.returncode != 0: + details = result.stderr.decode(errors="replace").strip() + raise RuntimeError(f"command exited with status {result.returncode}: {command}\n{details}") + return elapsed + + +def benchmark_hooks( + binary: Path, + root: Path, + provider_url: str, + configs: dict[str, Path], + *, + samples: int, + warmup: int, +) -> dict[str, Any]: + measurements = { + "process_baseline": [], + "codex_minimal": [], + "codex_file": [], + "codex_otlp": [], + "claude_minimal": [], + "claude_file": [], + "claude_otlp": [], + } + + with contextlib.ExitStack() as stack: + relay_urls = { + variant: stack.enter_context( + TransparentRelayProcess( + binary, + root, + provider_url, + configs[f"relay-{variant}"], + f"transparent-{variant}", + ) + ).url + for variant in ("minimal", "file", "otlp") + } + + def hook(agent: str, variant: str, index: int) -> int: + event_name = "sessionStart" if agent == "codex" else "SessionStart" + payload = json.dumps( + { + "session_id": f"benchmark-{agent}-{variant}-{index}", + "hook_event_name": event_name, + } + ).encode() + return run_subprocess_timed( + [ + str(binary), + "hook-forward", + agent, + "--gateway-url", + relay_urls[variant], + "--transparent-run", + "--fail-closed", + ], + root=root, + input_bytes=payload, + ) + + for index in range(-warmup, samples): + cycle = {"process_baseline": lambda: run_subprocess_timed([str(binary), "--version"], root=root)} + for agent in ("codex", "claude"): + for variant in ("minimal", "file", "otlp"): + cycle[f"{agent}_{variant}"] = lambda agent=agent, variant=variant: hook(agent, variant, index) + names = list(cycle) + random.Random(index).shuffle(names) + values = {name: cycle[name]() for name in names} + if index >= 0: + for name, value in values.items(): + measurements[name].append(value) + + baseline = measurements["process_baseline"] + result: dict[str, Any] = { + "absolute": {name: summarize_ns(values) for name, values in measurements.items()}, + "comparisons": {}, + } + for agent in ("codex", "claude"): + for variant in ("minimal", "file", "otlp"): + name = f"{agent}_{variant}" + deltas = [left - right for left, right in zip(measurements[name], baseline)] + summary = summarize_ns(deltas) + summary["median_ci95_ms"] = median_confidence_interval_ns(deltas, seed=len(name) * 1_009) + result["comparisons"][f"{name}_vs_process_baseline"] = summary + return result + + +def benchmark_startup( + binary: Path, + root: Path, + provider_url: str, + configs: dict[str, Path], + *, + samples: int, + warmup: int, +) -> dict[str, Any]: + measurements = {"process_baseline": [], **{variant: [] for variant in RELAY_VARIANTS}} + for index in range(-warmup, samples): + baseline = run_subprocess_timed([str(binary), "--version"], root=root) + cycle = {} + for variant in RELAY_VARIANTS: + process = RelayProcess(binary, root, provider_url, configs[variant], f"startup-{variant}-{index}") + process.start() + cycle[variant] = process.startup_ns + process.stop() + if index >= 0: + measurements["process_baseline"].append(baseline) + for variant, value in cycle.items(): + measurements[variant].append(value) + result: dict[str, Any] = { + "absolute": {name: summarize_ns(values) for name, values in measurements.items()}, + "comparisons": {}, + } + baseline = measurements["process_baseline"] + for variant in RELAY_VARIANTS: + deltas = [left - right for left, right in zip(measurements[variant], baseline)] + summary = summarize_ns(deltas) + summary["median_ci95_ms"] = median_confidence_interval_ns(deltas, seed=len(variant) * 2_003) + result["comparisons"][f"{variant}_readiness_vs_process_baseline"] = summary + return result diff --git a/scripts/benchmark_coding_agent_latency/cli.py b/scripts/benchmark_coding_agent_latency/cli.py new file mode 100644 index 000000000..5e4ba04bb --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/cli.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Command-line orchestration for the coding-agent latency benchmark.""" + +from __future__ import annotations + +import contextlib +import json +import tempfile +from pathlib import Path +from typing import Any + +from .benchmarks import benchmark_hooks, benchmark_scenario, benchmark_startup +from .config import BenchmarkConfig, parse_args +from .fixtures import write_plugin_configs, write_relay_config +from .processes import RelayProcess +from .reporting import environment_record, print_results +from .servers import OtlpHandler, ProviderHandler, local_server + + +def _benchmark_gateway( + binary: Path, + root: Path, + provider_url: str, + configs: dict[str, Path], + config: BenchmarkConfig, +) -> list[dict[str, Any]]: + with contextlib.ExitStack() as stack: + relays = { + variant: stack.enter_context(RelayProcess(binary, root, provider_url, configs[variant], variant)) + for variant in ("relay-minimal", "relay-file", "relay-otlp") + } + urls = {"direct": provider_url} | {variant: relay.url for variant, relay in relays.items()} + scenarios = [] + for provider in config.providers: + model = config.openai_model if provider == "openai" else config.anthropic_model + for mode in config.modes: + streaming = mode == "streaming" + for payload_bytes in config.payload_sizes: + for concurrency in config.concurrency: + print( + f"Benchmarking {provider} {mode}, payload={payload_bytes}, concurrency={concurrency}...", + flush=True, + ) + scenarios.append( + benchmark_scenario( + urls, + provider=provider, + model=model, + request_fill=config.request_fill, + streaming=streaming, + payload_bytes=payload_bytes, + samples=config.samples, + warmup=config.warmup, + concurrency=concurrency, + ) + ) + return scenarios + + +def run_benchmarks(binary: Path, config: BenchmarkConfig) -> dict[str, Any]: + """Run selected suites and return the versioned result document.""" + OtlpHandler.reset() + results: dict[str, Any] = { + "schema_version": 2, + "environment": environment_record(binary), + "parameters": config.parameters(), + } + with tempfile.TemporaryDirectory(prefix="nemo-relay-latency-") as temporary: + root = Path(temporary) + write_relay_config(root) + with ( + local_server( + ProviderHandler, + response_bytes=config.response_bytes, + stream_chunks=config.stream_chunks, + response_fill=config.response_fill, + openai_model=config.openai_model, + anthropic_model=config.anthropic_model, + ) as provider_url, + local_server(OtlpHandler) as otlp_url, + ): + configs = write_plugin_configs(root, otlp_url) + if "gateway" in config.tests: + results["gateway"] = _benchmark_gateway(binary, root, provider_url, configs, config) + if "hooks" in config.tests: + results["hooks"] = benchmark_hooks( + binary, + root, + provider_url, + configs, + samples=config.hook_samples, + warmup=config.warmup, + ) + if "startup" in config.tests: + results["startup"] = benchmark_startup( + binary, + root, + provider_url, + configs, + samples=config.startup_samples, + warmup=config.warmup, + ) + + if {"gateway", "hooks"}.intersection(config.tests): + atof_path = root / "atof" / "events.jsonl" + if not atof_path.is_file() or atof_path.stat().st_size == 0: + raise RuntimeError("local ATOF exporter did not write benchmark events") + if OtlpHandler.request_count == 0: + raise RuntimeError("local OTLP receiver did not receive benchmark exports") + results["exporter_delivery"] = { + "atof_bytes": atof_path.stat().st_size, + "otlp_requests": OtlpHandler.request_count, + } + return results + + +def main(argv: list[str] | None = None) -> None: + options = parse_args(argv) + results = run_benchmarks(options.relay_bin, options.config) + options.output.parent.mkdir(parents=True, exist_ok=True) + options.output.write_text(json.dumps(results, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print_results(results) + print(f"\nJSON results: {options.output}") diff --git a/scripts/benchmark_coding_agent_latency/config.py b/scripts/benchmark_coding_agent_latency/config.py new file mode 100644 index 000000000..ed643f9c6 --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/config.py @@ -0,0 +1,291 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Configuration loading and command-line overrides for the benchmark.""" + +from __future__ import annotations + +import argparse +import tomllib +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any + +PACKAGE_ROOT = Path(__file__).resolve().parent +DATA_ROOT = PACKAGE_ROOT / "data" +DEFAULT_CONFIG_PATH = DATA_ROOT / "default.toml" + +AVAILABLE_TESTS = ("gateway", "hooks", "startup") +AVAILABLE_PROVIDERS = ("openai", "anthropic") +AVAILABLE_MODES = ("buffered", "streaming") +CONFIG_KEYS = { + "tests", + "providers", + "modes", + "samples", + "hook_samples", + "startup_samples", + "warmup", + "payload_sizes", + "concurrency", + "response_bytes", + "stream_chunks", + "models", + "content", +} +TABLE_KEYS = { + "models": {"openai", "anthropic"}, + "content": {"request_fill", "response_fill"}, +} + + +@dataclass(frozen=True) +class BenchmarkConfig: + """Validated benchmark matrix and sample settings.""" + + tests: tuple[str, ...] + providers: tuple[str, ...] + modes: tuple[str, ...] + samples: int + hook_samples: int + startup_samples: int + warmup: int + payload_sizes: tuple[int, ...] + concurrency: tuple[int, ...] + response_bytes: int + stream_chunks: int + openai_model: str + anthropic_model: str + request_fill: str + response_fill: str + + def parameters(self) -> dict[str, Any]: + """Return the configuration embedded in the JSON result.""" + return { + "tests": self.tests, + "providers": self.providers, + "modes": self.modes, + "samples": self.samples, + "hook_samples": self.hook_samples, + "startup_samples": self.startup_samples, + "warmup": self.warmup, + "payload_sizes": self.payload_sizes, + "concurrency": self.concurrency, + "response_bytes": self.response_bytes, + "stream_chunks": self.stream_chunks, + "models": { + "openai": self.openai_model, + "anthropic": self.anthropic_model, + }, + "content": { + "request_fill": self.request_fill, + "response_fill": self.response_fill, + }, + } + + +@dataclass(frozen=True) +class CliOptions: + """File paths and resolved benchmark configuration.""" + + relay_bin: Path + output: Path + config: BenchmarkConfig + + +def _read_toml(path: Path) -> dict[str, Any]: + with path.open("rb") as config_file: + return tomllib.load(config_file) + + +def _merge_config(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + unknown = set(override) - CONFIG_KEYS + if unknown: + raise ValueError(f"unknown config key(s): {', '.join(sorted(unknown))}") + merged = dict(base) + for key, value in override.items(): + if key in {"models", "content"}: + if not isinstance(value, dict): + raise ValueError(f"[{key}] must be a TOML table") + unknown_nested = set(value) - TABLE_KEYS[key] + if unknown_nested: + raise ValueError(f"unknown [{key}] key(s): {', '.join(sorted(unknown_nested))}") + merged[key] = dict(merged.get(key, {})) | value + else: + merged[key] = value + return merged + + +def _string_tuple(value: Any, name: str, available: tuple[str, ...]) -> tuple[str, ...]: + if not isinstance(value, list) or not value or not all(isinstance(item, str) for item in value): + raise ValueError(f"{name} must be a non-empty list of strings") + values = tuple(value) + invalid = sorted(set(values) - set(available)) + if invalid: + raise ValueError(f"unknown {name}: {', '.join(invalid)}; choose from {', '.join(available)}") + if len(set(values)) != len(values): + raise ValueError(f"{name} must not contain duplicates") + return values + + +def _positive_int(value: Any, name: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + return value + + +def _positive_int_tuple(value: Any, name: str) -> tuple[int, ...]: + if not isinstance(value, list) or not value: + raise ValueError(f"{name} must be a non-empty list of positive integers") + values = tuple(_positive_int(item, name) for item in value) + if len(set(values)) != len(values): + raise ValueError(f"{name} must not contain duplicates") + return values + + +def _nonempty_string(value: Any, name: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{name} must be a non-empty string") + return value + + +def _config_from_mapping(value: dict[str, Any]) -> BenchmarkConfig: + unknown = set(value) - CONFIG_KEYS + if unknown: + raise ValueError(f"unknown config key(s): {', '.join(sorted(unknown))}") + models = value.get("models") + content = value.get("content") + if not isinstance(models, dict) or not isinstance(content, dict): + raise ValueError("config must contain [models] and [content] tables") + warmup = value.get("warmup") + if not isinstance(warmup, int) or isinstance(warmup, bool) or warmup < 0: + raise ValueError("warmup must be a non-negative integer") + request_fill = _nonempty_string(content.get("request_fill"), "content.request_fill") + response_fill = _nonempty_string(content.get("response_fill"), "content.response_fill") + if len(request_fill) != 1 or len(response_fill) != 1 or not request_fill.isascii() or not response_fill.isascii(): + raise ValueError("content fill values must each contain exactly one ASCII character") + config = BenchmarkConfig( + tests=_string_tuple(value.get("tests"), "tests", AVAILABLE_TESTS), + providers=_string_tuple(value.get("providers"), "providers", AVAILABLE_PROVIDERS), + modes=_string_tuple(value.get("modes"), "modes", AVAILABLE_MODES), + samples=_positive_int(value.get("samples"), "samples"), + hook_samples=_positive_int(value.get("hook_samples"), "hook_samples"), + startup_samples=_positive_int(value.get("startup_samples"), "startup_samples"), + warmup=warmup, + payload_sizes=_positive_int_tuple(value.get("payload_sizes"), "payload_sizes"), + concurrency=_positive_int_tuple(value.get("concurrency"), "concurrency"), + response_bytes=_positive_int(value.get("response_bytes"), "response_bytes"), + stream_chunks=_positive_int(value.get("stream_chunks"), "stream_chunks"), + openai_model=_nonempty_string(models.get("openai"), "models.openai"), + anthropic_model=_nonempty_string(models.get("anthropic"), "models.anthropic"), + request_fill=request_fill, + response_fill=response_fill, + ) + if "gateway" in config.tests and max(config.concurrency) > config.samples: + raise ValueError("samples must be greater than or equal to every gateway concurrency value") + return config + + +def load_config(path: Path) -> BenchmarkConfig: + """Load the defaults and overlay a possibly partial user config.""" + defaults = _read_toml(DEFAULT_CONFIG_PATH) + if path.resolve() != DEFAULT_CONFIG_PATH.resolve(): + defaults = _merge_config(defaults, _read_toml(path)) + return _config_from_mapping(defaults) + + +def _csv_strings(value: str) -> tuple[str, ...]: + values = tuple(item.strip() for item in value.split(",") if item.strip()) + if not values: + raise argparse.ArgumentTypeError("expected a comma-separated list") + return values + + +def _csv_ints(value: str) -> tuple[int, ...]: + try: + values = tuple(int(item.strip()) for item in value.split(",") if item.strip()) + except ValueError as error: + raise argparse.ArgumentTypeError("expected comma-separated positive integers") from error + if not values or any(item <= 0 for item in values): + raise argparse.ArgumentTypeError("expected comma-separated positive integers") + return values + + +def _arg_positive_int(value: str) -> int: + try: + return _positive_int(int(value), "value") + except ValueError as error: + raise argparse.ArgumentTypeError("expected a positive integer") from error + + +def _arg_nonnegative_int(value: str) -> int: + try: + number = int(value) + except ValueError as error: + raise argparse.ArgumentTypeError("expected a non-negative integer") from error + if number < 0: + raise argparse.ArgumentTypeError("expected a non-negative integer") + return number + + +def parse_args(argv: list[str] | None = None) -> CliOptions: + """Parse CLI options, applying them after values from the config file.""" + parser = argparse.ArgumentParser(description="Measure local coding-agent gateway latency.") + parser.add_argument("--relay-bin", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG_PATH, help="TOML overrides for the benchmark") + parser.add_argument("--tests", type=_csv_strings, help=f"override test suites: {','.join(AVAILABLE_TESTS)}") + parser.add_argument("--providers", type=_csv_strings, help=f"override providers: {','.join(AVAILABLE_PROVIDERS)}") + parser.add_argument("--modes", type=_csv_strings, help=f"override response modes: {','.join(AVAILABLE_MODES)}") + parser.add_argument("--samples", type=_arg_positive_int) + parser.add_argument("--hook-samples", type=_arg_positive_int) + parser.add_argument("--startup-samples", type=_arg_positive_int) + parser.add_argument("--warmup", type=_arg_nonnegative_int) + parser.add_argument("--payload-sizes", type=_csv_ints, help="override comma-separated request payload sizes") + parser.add_argument("--concurrency", type=_csv_ints, help="override comma-separated in-flight request counts") + parser.add_argument("--response-bytes", type=_arg_positive_int) + parser.add_argument("--stream-chunks", type=_arg_positive_int) + args = parser.parse_args(argv) + + try: + config = load_config(args.config) + overrides = { + name: getattr(args, name) + for name in ( + "tests", + "providers", + "modes", + "samples", + "hook_samples", + "startup_samples", + "warmup", + "payload_sizes", + "concurrency", + "response_bytes", + "stream_chunks", + ) + if getattr(args, name) is not None + } + config = replace(config, **overrides) + # Reuse the same validation for values supplied by argparse. + config = _config_from_mapping( + { + **config.parameters(), + "tests": list(config.tests), + "providers": list(config.providers), + "modes": list(config.modes), + "payload_sizes": list(config.payload_sizes), + "concurrency": list(config.concurrency), + } + ) + except (OSError, tomllib.TOMLDecodeError, ValueError) as error: + parser.error(f"invalid benchmark config: {error}") + + relay_bin = args.relay_bin.resolve() + if not relay_bin.is_file(): + parser.error(f"Relay binary does not exist: {relay_bin}") + return CliOptions( + relay_bin=relay_bin, + output=args.output.resolve(), + config=config, + ) diff --git a/scripts/benchmark_coding_agent_latency/data/agent-config.toml b/scripts/benchmark_coding_agent_latency/data/agent-config.toml new file mode 100644 index 000000000..13d6e5224 --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/data/agent-config.toml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[agents.codex] +command = "__CODEX_COMMAND__" diff --git a/scripts/benchmark_coding_agent_latency/data/default.toml b/scripts/benchmark_coding_agent_latency/data/default.toml new file mode 100644 index 000000000..f20a4639c --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/data/default.toml @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# A custom config file may contain only the values it needs to override. +tests = ["gateway", "hooks", "startup"] +providers = ["openai", "anthropic"] +modes = ["buffered", "streaming"] +samples = 100 +hook_samples = 30 +startup_samples = 20 +warmup = 3 +payload_sizes = [4096, 65536, 262144, 1048576, 4194304] +concurrency = [1, 2, 4, 8, 16] +response_bytes = 16384 +stream_chunks = 32 + +[models] +openai = "gpt-5-codex" +anthropic = "claude-sonnet-4-5" + +[content] +request_fill = "p" +response_fill = "r" diff --git a/scripts/benchmark_coding_agent_latency/data/fake-codex.cmd b/scripts/benchmark_coding_agent_latency/data/fake-codex.cmd new file mode 100644 index 000000000..052446d4b --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/data/fake-codex.cmd @@ -0,0 +1,12 @@ +@REM SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +@REM SPDX-License-Identifier: Apache-2.0 +@echo off +if "%1"=="--version" ( + echo codex-cli 0.143.0 + exit /b 0 +) +> "%BENCHMARK_GATEWAY_FILE%" echo %NEMO_RELAY_GATEWAY_URL% +:wait +if exist "%BENCHMARK_STOP_FILE%" exit /b 0 +ping 127.0.0.1 -n 2 >nul +goto wait diff --git a/scripts/benchmark_coding_agent_latency/data/fake-codex.sh b/scripts/benchmark_coding_agent_latency/data/fake-codex.sh new file mode 100755 index 000000000..67bd9a530 --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/data/fake-codex.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +if [ "${1:-}" = "--version" ]; then + printf 'codex-cli 0.143.0\n' + exit 0 +fi +printf '%s' "$NEMO_RELAY_GATEWAY_URL" > "$BENCHMARK_GATEWAY_FILE" +while [ ! -f "$BENCHMARK_STOP_FILE" ]; do + sleep 0.1 +done diff --git a/scripts/benchmark_coding_agent_latency/data/plugins-file.toml b/scripts/benchmark_coding_agent_latency/data/plugins-file.toml new file mode 100644 index 000000000..533a65759 --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/data/plugins-file.toml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = 1 + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 3 + +[components.config.atof] +enabled = true + +[[components.config.atof.sinks]] +type = "file" +output_directory = "__ATOF_OUTPUT_DIRECTORY__" +filename = "events.jsonl" +mode = "append" diff --git a/scripts/benchmark_coding_agent_latency/data/plugins-minimal.toml b/scripts/benchmark_coding_agent_latency/data/plugins-minimal.toml new file mode 100644 index 000000000..f02170300 --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/data/plugins-minimal.toml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = 1 +components = [] diff --git a/scripts/benchmark_coding_agent_latency/data/plugins-otlp.toml b/scripts/benchmark_coding_agent_latency/data/plugins-otlp.toml new file mode 100644 index 000000000..818d51940 --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/data/plugins-otlp.toml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = 1 + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 3 + +[components.config.opentelemetry] +enabled = true + +[[components.config.opentelemetry.endpoints]] +type = "openinference" +endpoint = "__OTLP_ENDPOINT__" diff --git a/scripts/benchmark_coding_agent_latency/data/relay-config.toml b/scripts/benchmark_coding_agent_latency/data/relay-config.toml new file mode 100644 index 000000000..d51c4fe1e --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/data/relay-config.toml @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/benchmark_coding_agent_latency/fixtures.py b/scripts/benchmark_coding_agent_latency/fixtures.py new file mode 100644 index 000000000..ad18c405e --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/fixtures.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Materialize static benchmark data in an isolated workspace.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +from .config import DATA_ROOT + + +def _read_data(name: str) -> str: + return (DATA_ROOT / name).read_text(encoding="utf-8") + + +def _render_data(name: str, replacements: dict[str, str]) -> str: + rendered = _read_data(name) + for marker, value in replacements.items(): + if marker not in rendered: + raise RuntimeError(f"static fixture {name} is missing marker {marker}") + rendered = rendered.replace(marker, value) + return rendered + + +def toml_string(value: str | Path) -> str: + """Encode a string using TOML-compatible JSON quoting.""" + return json.dumps(str(value)) + + +def write_relay_config(root: Path) -> Path: + path = root / "config.toml" + path.write_text(_read_data("relay-config.toml"), encoding="utf-8") + return path + + +def write_plugin_configs(root: Path, otlp_url: str) -> dict[str, Path]: + """Write the three Relay plugin configurations used for paired runs.""" + paths = { + "relay-minimal": root / "plugins-minimal.toml", + "relay-file": root / "plugins-file.toml", + "relay-otlp": root / "plugins-otlp.toml", + } + paths["relay-minimal"].write_text(_read_data("plugins-minimal.toml"), encoding="utf-8") + + atof_dir = root / "atof" + atof_dir.mkdir() + paths["relay-file"].write_text( + _render_data("plugins-file.toml", {'"__ATOF_OUTPUT_DIRECTORY__"': toml_string(atof_dir)}), + encoding="utf-8", + ) + paths["relay-otlp"].write_text( + _render_data("plugins-otlp.toml", {'"__OTLP_ENDPOINT__"': toml_string(f"{otlp_url}/v1/traces")}), + encoding="utf-8", + ) + return paths + + +def write_fake_codex(root: Path) -> Path: + """Copy the platform-specific static fake Codex client into the workspace.""" + source_name = "fake-codex.cmd" if os.name == "nt" else "fake-codex.sh" + target_name = "benchmark-codex.cmd" if os.name == "nt" else "benchmark-codex" + path = root / target_name + path.write_text(_read_data(source_name), encoding="utf-8") + if os.name != "nt": + path.chmod(0o755) + return path + + +def write_agent_config(root: Path, name: str, fake_codex: Path) -> Path: + path = root / f"{name}-config.toml" + path.write_text( + _render_data("agent-config.toml", {'"__CODEX_COMMAND__"': toml_string(fake_codex)}), + encoding="utf-8", + ) + return path + + +def isolated_environment(root: Path) -> dict[str, str]: + """Return an environment that cannot discover the developer's Relay state.""" + environment = os.environ.copy() + environment.update( + { + "HOME": str(root / "home"), + "XDG_CONFIG_HOME": str(root / "xdg-config"), + "XDG_DATA_HOME": str(root / "xdg-data"), + "NO_COLOR": "1", + } + ) + for directory in ("home", "xdg-config", "xdg-data"): + (root / directory).mkdir(exist_ok=True) + return environment diff --git a/scripts/benchmark_coding_agent_latency/processes.py b/scripts/benchmark_coding_agent_latency/processes.py new file mode 100644 index 000000000..c49356b84 --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/processes.py @@ -0,0 +1,206 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Relay subprocess lifecycle helpers for latency measurements.""" + +from __future__ import annotations + +import socket +import subprocess +import time +import uuid +from pathlib import Path +from typing import IO + +from .fixtures import isolated_environment, write_agent_config, write_fake_codex +from .servers import connection_for + + +class RelayProcess: + """Run a normal Relay gateway and record readiness latency.""" + + def __init__( + self, + binary: Path, + root: Path, + provider_url: str, + plugin_config: Path, + name: str, + ) -> None: + self.binary = binary + self.root = root + self.provider_url = provider_url + self.plugin_config = plugin_config + self.name = name + self.process: subprocess.Popen[bytes] | None = None + self.url = "" + self.startup_ns = 0 + self.log_handle: IO[bytes] | None = None + + def start(self) -> None: + log_path = self.root / f"{self.name}.log" + self.log_handle = log_path.open("ab") + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + self.url = f"http://127.0.0.1:{port}" + command = [ + str(self.binary), + "--bind", + f"127.0.0.1:{port}", + "--openai-base-url", + f"{self.provider_url}/v1", + "--anthropic-base-url", + self.provider_url, + "--config", + str(self.root / "config.toml"), + "--plugin-config-path", + str(self.plugin_config), + ] + started = time.perf_counter_ns() + self.process = subprocess.Popen( + command, + cwd=self.root, + env=isolated_environment(self.root), + stdout=subprocess.DEVNULL, + stderr=self.log_handle, + ) + deadline = time.monotonic() + 15 + while not self._healthy(): + if self.process.poll() is not None: + self._raise_start_error(log_path) + if time.monotonic() >= deadline: + self.stop() + raise RuntimeError(f"timed out waiting for {self.name} readiness") + time.sleep(0.0005) + self.startup_ns = time.perf_counter_ns() - started + + def _healthy(self) -> bool: + connection = connection_for(self.url) + connection.timeout = 0.1 + try: + connection.request("GET", "/healthz") + response = connection.getresponse() + response.read() + return response.status == 200 + except OSError: + return False + finally: + connection.close() + + def stop(self) -> None: + if self.process is not None and self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + if self.log_handle is not None: + self.log_handle.close() + self.log_handle = None + + def _raise_start_error(self, log_path: Path) -> None: + if self.log_handle is not None: + self.log_handle.close() + self.log_handle = None + details = log_path.read_text(encoding="utf-8", errors="replace") + raise RuntimeError(f"{self.name} exited during startup:\n{details}") + + def __enter__(self) -> RelayProcess: + self.start() + return self + + def __exit__(self, *_: object) -> None: + self.stop() + + +class TransparentRelayProcess: + """Run Relay's transparent coding-agent path for hook benchmarks.""" + + def __init__( + self, + binary: Path, + root: Path, + provider_url: str, + plugin_config: Path, + name: str, + ) -> None: + self.binary = binary + self.root = root + self.provider_url = provider_url + self.plugin_config = plugin_config + self.name = name + self.process: subprocess.Popen[bytes] | None = None + self.url = "" + self.log_handle: IO[bytes] | None = None + self.stop_file = root / f"{name}-{uuid.uuid4().hex}.stop" + + def start(self) -> None: + gateway_file = self.root / f"{self.name}-{uuid.uuid4().hex}.gateway" + config = write_agent_config(self.root, self.name, write_fake_codex(self.root)) + log_path = self.root / f"{self.name}.log" + self.log_handle = log_path.open("ab") + environment = isolated_environment(self.root) + environment["BENCHMARK_GATEWAY_FILE"] = str(gateway_file) + environment["BENCHMARK_STOP_FILE"] = str(self.stop_file) + command = [ + str(self.binary), + "run", + "--agent", + "codex", + "--config", + str(config), + "--openai-base-url", + f"{self.provider_url}/v1", + "--anthropic-base-url", + self.provider_url, + "--plugin-config-path", + str(self.plugin_config), + ] + self.process = subprocess.Popen( + command, + cwd=self.root, + env=environment, + stdout=subprocess.DEVNULL, + stderr=self.log_handle, + ) + deadline = time.monotonic() + 15 + while not gateway_file.exists(): + if self.process.poll() is not None: + self._raise_start_error(log_path) + if time.monotonic() >= deadline: + self.stop() + raise RuntimeError(f"timed out waiting for {self.name} transparent gateway") + time.sleep(0.001) + self.url = gateway_file.read_text(encoding="utf-8").strip() + + def stop(self) -> None: + self.stop_file.touch() + if self.process is not None and self.process.poll() is None: + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + if self.log_handle is not None: + self.log_handle.close() + self.log_handle = None + + def _raise_start_error(self, log_path: Path) -> None: + if self.log_handle is not None: + self.log_handle.close() + self.log_handle = None + details = log_path.read_text(encoding="utf-8", errors="replace") + raise RuntimeError(f"{self.name} exited during startup:\n{details}") + + def __enter__(self) -> TransparentRelayProcess: + self.start() + return self + + def __exit__(self, *_: object) -> None: + self.stop() diff --git a/scripts/benchmark_coding_agent_latency/protocol.py b/scripts/benchmark_coding_agent_latency/protocol.py new file mode 100644 index 000000000..6a08ca47e --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/protocol.py @@ -0,0 +1,229 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Deterministic OpenAI and Anthropic request/response fixtures.""" + +from __future__ import annotations + +import json +import math +import time +import uuid +from typing import Any + + +def split_text(text: str, chunks: int) -> list[str]: + chunk_size = max(1, math.ceil(len(text) / chunks)) + return [text[index : index + chunk_size] for index in range(0, len(text), chunk_size)] + + +def response_document(model: str, text: str) -> dict[str, Any]: + response_id = f"resp_{uuid.uuid4().hex}" + item_id = f"msg_{uuid.uuid4().hex}" + created = int(time.time()) + item = { + "id": item_id, + "type": "message", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": text, + "annotations": [], + "logprobs": [], + } + ], + } + return { + "id": response_id, + "object": "response", + "created_at": created, + "completed_at": created, + "status": "completed", + "background": False, + "error": None, + "incomplete_details": None, + "instructions": None, + "max_output_tokens": None, + "max_tool_calls": None, + "model": model, + "output": [item], + "parallel_tool_calls": True, + "previous_response_id": None, + "prompt_cache_key": None, + "reasoning": {"effort": "medium", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": False, + "temperature": None, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": None, + "truncation": "disabled", + "usage": { + "input_tokens": 1, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 1, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 2, + }, + "user": None, + "metadata": {}, + } + + +def openai_events(model: str, text: str, chunks: int) -> list[dict[str, Any]]: + response = response_document(model, text) + response_id = response["id"] + item = response["output"][0] + item_id = item["id"] + in_progress = {**response, "completed_at": None, "status": "in_progress", "output": []} + events: list[dict[str, Any]] = [ + {"type": "response.created", "response": in_progress}, + { + "type": "response.output_item.added", + "response_id": response_id, + "output_index": 0, + "item": {**item, "status": "in_progress", "content": []}, + }, + { + "type": "response.content_part.added", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": 0, + "part": {"type": "output_text", "text": "", "annotations": [], "logprobs": []}, + }, + ] + for delta in split_text(text, chunks): + events.append( + { + "type": "response.output_text.delta", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": 0, + "delta": delta, + "logprobs": [], + } + ) + events.extend( + [ + { + "type": "response.output_text.done", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": 0, + "text": text, + "logprobs": [], + }, + { + "type": "response.content_part.done", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": 0, + "part": item["content"][0], + }, + { + "type": "response.output_item.done", + "response_id": response_id, + "output_index": 0, + "item": item, + }, + {"type": "response.completed", "response": response}, + ] + ) + return events + + +def anthropic_events(model: str, text: str, chunks: int) -> list[dict[str, Any]]: + message_id = f"msg_{uuid.uuid4().hex}" + events: list[dict[str, Any]] = [ + { + "type": "message_start", + "message": { + "id": message_id, + "type": "message", + "role": "assistant", + "content": [], + "model": model, + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + ] + for delta in split_text(text, chunks): + events.append( + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": delta}, + } + ) + events.extend( + [ + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 1}, + }, + {"type": "message_stop"}, + ] + ) + return events + + +def make_request( + provider: str, + streaming: bool, + payload_bytes: int, + *, + model: str, + request_fill: str, +) -> bytes: + payload = request_fill * payload_bytes + if provider == "openai": + value = { + "model": model, + "input": [ + { + "role": "user", + "content": [{"type": "input_text", "text": payload}], + } + ], + "stream": streaming, + } + else: + value = { + "model": model, + "max_tokens": 1024, + "messages": [{"role": "user", "content": payload}], + "stream": streaming, + } + return json.dumps(value, separators=(",", ":")).encode() + + +def request_path(provider: str) -> str: + return "/v1/responses" if provider == "openai" else "/v1/messages" + + +def request_headers(provider: str) -> dict[str, str]: + headers = {"Content-Type": "application/json"} + if provider == "openai": + headers["Authorization"] = "Bearer relay-benchmark" + else: + headers["x-api-key"] = "relay-benchmark" + headers["anthropic-version"] = "2023-06-01" + return headers diff --git a/scripts/benchmark_coding_agent_latency/reporting.py b/scripts/benchmark_coding_agent_latency/reporting.py new file mode 100644 index 000000000..49e902e8f --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/reporting.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Benchmark environment metadata and terminal reporting.""" + +from __future__ import annotations + +import datetime as dt +import platform +import subprocess +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[2] + + +def _git_output(*args: str) -> str: + result = subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=False) + return result.stdout.strip() if result.returncode == 0 else "unknown" + + +def environment_record(binary: Path) -> dict[str, Any]: + version = subprocess.run([str(binary), "--version"], capture_output=True, text=True, check=True).stdout.strip() + return { + "generated_at": dt.datetime.now(dt.UTC).isoformat(), + "git_commit": _git_output("rev-parse", "HEAD"), + "git_dirty": bool(_git_output("status", "--porcelain")), + "relay_version": version, + "platform": platform.platform(), + "machine": platform.machine(), + "processor": platform.processor(), + "python": platform.python_version(), + } + + +def print_results(results: dict[str, Any]) -> None: + """Print only the result sections produced by selected test suites.""" + if "gateway" in results: + print("\nGateway incremental latency (milliseconds; negative values are measurement noise)") + headers = ("provider", "mode", "bytes", "c", "comparison", "metric", "p50", "p95", "p99") + print(" ".join(f"{header:>12}" for header in headers)) + for scenario in results["gateway"]: + for comparison in ( + "relay-minimal_vs_direct", + "relay-file_vs_direct", + "relay-otlp_vs_direct", + ): + for metric, summary in scenario["comparisons"][comparison].items(): + values = ( + scenario["provider"], + scenario["mode"], + str(scenario["payload_bytes"]), + str(scenario["concurrency"]), + comparison.replace("relay-", "").replace("_vs_direct", ""), + metric, + f"{summary['p50_ms']:.3f}", + f"{summary['p95_ms']:.3f}", + f"{summary['p99_ms']:.3f}", + ) + print(" ".join(f"{value:>12}" for value in values)) + + if "hooks" in results: + print("\nHook subprocess wall time (p50 milliseconds)") + for name, summary in results["hooks"]["absolute"].items(): + print(f" {name:<24} {summary['p50_ms']:>9.3f}") + + if "startup" in results: + print("\nCold process/readiness time (p50 milliseconds)") + for name, summary in results["startup"]["absolute"].items(): + print(f" {name:<24} {summary['p50_ms']:>9.3f}") + + if "exporter_delivery" in results: + delivery = results["exporter_delivery"] + print("\nExporter delivery verification") + print(f" ATOF bytes written {delivery['atof_bytes']:>12}") + print(f" OTLP requests received {delivery['otlp_requests']:>12}") diff --git a/scripts/benchmark_coding_agent_latency/servers.py b/scripts/benchmark_coding_agent_latency/servers.py new file mode 100644 index 000000000..a7a88fb4e --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/servers.py @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Loopback provider and OTLP servers used by the benchmark.""" + +from __future__ import annotations + +import contextlib +import http.client +import json +import socket +import threading +import uuid +from collections.abc import Iterable, Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, ClassVar +from urllib.parse import urlparse + +from .protocol import anthropic_events, openai_events, response_document + + +class QuietThreadingServer(ThreadingHTTPServer): + daemon_threads = True + allow_reuse_address = True + + +class ProviderHandler(BaseHTTPRequestHandler): + """Serve deterministic OpenAI Responses and Anthropic Messages payloads.""" + + protocol_version = "HTTP/1.1" + + def setup(self) -> None: + super().setup() + self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + del format, args + + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("content-length", "0")) + request = json.loads(self.rfile.read(length) or b"{}") + path = urlparse(self.path).path + text = getattr(self.server, "response_fill") * getattr(self.server, "response_bytes") + chunks = getattr(self.server, "stream_chunks") + if path.endswith("/responses"): + self._openai(request, text, chunks) + elif path.endswith("/messages"): + self._anthropic(request, text, chunks) + else: + self.send_error(404) + + def _openai(self, request: dict[str, Any], text: str, chunks: int) -> None: + model = request.get("model", getattr(self.server, "openai_model")) + if request.get("stream", False): + frames = [ + f"data: {json.dumps(event, separators=(',', ':'))}\n\n".encode() + for event in openai_events(model, text, chunks) + ] + frames.append(b"data: [DONE]\n\n") + self._send_stream(frames) + return + self._send_json(response_document(model, text)) + + def _anthropic(self, request: dict[str, Any], text: str, chunks: int) -> None: + model = request.get("model", getattr(self.server, "anthropic_model")) + if request.get("stream", False): + frames = [] + for event in anthropic_events(model, text, chunks): + name = event["type"] + frames.append(f"event: {name}\ndata: {json.dumps(event, separators=(',', ':'))}\n\n".encode()) + self._send_stream(frames) + return + self._send_json( + { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": text}], + "model": model, + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ) + + def _send_json(self, value: dict[str, Any]) -> None: + body = json.dumps(value, separators=(",", ":")).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _send_stream(self, frames: Iterable[bytes]) -> None: + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + for frame in frames: + self.wfile.write(f"{len(frame):X}\r\n".encode()) + self.wfile.write(frame) + self.wfile.write(b"\r\n") + self.wfile.flush() + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + + +class OtlpHandler(BaseHTTPRequestHandler): + """Accept OTLP requests and track whether the exporter delivered data.""" + + protocol_version = "HTTP/1.1" + request_count: ClassVar[int] = 0 + request_count_lock: ClassVar[threading.Lock] = threading.Lock() + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + del format, args + + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("content-length", "0")) + self.rfile.read(length) + with self.request_count_lock: + type(self).request_count += 1 + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + @classmethod + def reset(cls) -> None: + with cls.request_count_lock: + cls.request_count = 0 + + +@contextlib.contextmanager +def local_server(handler: type[BaseHTTPRequestHandler], **attributes: Any) -> Iterator[str]: + server = QuietThreadingServer(("127.0.0.1", 0), handler) + for name, value in attributes.items(): + setattr(server, name, value) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + server.server_close() + thread.join() + + +def connection_for(url: str) -> http.client.HTTPConnection: + parsed = urlparse(url) + if parsed.hostname is None: + raise ValueError(f"URL does not contain a host: {url}") + return http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=30) diff --git a/scripts/tests/test_benchmark_coding_agent_latency.py b/scripts/tests/test_benchmark_coding_agent_latency.py new file mode 100644 index 000000000..1f71c5785 --- /dev/null +++ b/scripts/tests/test_benchmark_coding_agent_latency.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the coding-agent latency benchmark fixture.""" + +import tempfile +import unittest +from pathlib import Path + +from scripts.benchmark_coding_agent_latency.config import DEFAULT_CONFIG_PATH, load_config, parse_args +from scripts.benchmark_coding_agent_latency.fixtures import write_agent_config, write_fake_codex, write_plugin_configs + + +class BenchmarkConfigTests(unittest.TestCase): + def test_default_config_defines_every_suite_and_matrix_axis(self) -> None: + config = load_config(DEFAULT_CONFIG_PATH) + + self.assertEqual(config.tests, ("gateway", "hooks", "startup")) + self.assertEqual(config.providers, ("openai", "anthropic")) + self.assertEqual(config.modes, ("buffered", "streaming")) + self.assertTrue(config.payload_sizes) + self.assertTrue(config.concurrency) + + def test_partial_config_and_cli_arguments_override_defaults(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + custom_config = root / "quick.toml" + custom_config.write_text('tests = ["startup"]\nsamples = 7\n', encoding="utf-8") + relay_bin = root / "nemo-relay" + relay_bin.touch() + + options = parse_args( + [ + "--relay-bin", + str(relay_bin), + "--output", + str(root / "results.json"), + "--config", + str(custom_config), + "--tests", + "gateway,hooks", + "--samples", + "3", + "--concurrency", + "1", + "--providers", + "openai", + ] + ) + + self.assertEqual(options.config.tests, ("gateway", "hooks")) + self.assertEqual(options.config.samples, 3) + self.assertEqual(options.config.concurrency, (1,)) + self.assertEqual(options.config.providers, ("openai",)) + self.assertEqual(options.config.modes, ("buffered", "streaming")) + + def test_rejects_unknown_config_keys(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + config_path = Path(temporary) / "invalid.toml" + config_path.write_text("sample = 1\n", encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "unknown config key"): + load_config(config_path) + + def test_rejects_gateway_concurrency_greater_than_samples(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + config_path = Path(temporary) / "invalid.toml" + config_path.write_text( + 'tests = ["gateway"]\nsamples = 2\nconcurrency = [4]\n', + encoding="utf-8", + ) + + with self.assertRaisesRegex(ValueError, "samples must be greater"): + load_config(config_path) + + +class StaticFixtureTests(unittest.TestCase): + def test_materializes_templates_without_embedded_markers(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + configs = write_plugin_configs(root, "http://127.0.0.1:4318") + fake_codex = write_fake_codex(root) + agent_config = write_agent_config(root, "test", fake_codex) + + rendered = "\n".join(path.read_text(encoding="utf-8") for path in (*configs.values(), agent_config)) + + self.assertNotIn("__ATOF_OUTPUT_DIRECTORY__", rendered) + self.assertNotIn("__OTLP_ENDPOINT__", rendered) + self.assertNotIn("__CODEX_COMMAND__", rendered) + + +if __name__ == "__main__": + unittest.main() From 7c7485e41a2dca63dd2410992e4bc739c5babaa4 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 5 Aug 2026 13:40:38 -0700 Subject: [PATCH 02/14] refactor: consolidate coding-agent benchmark fixture Signed-off-by: Yuchen Zhang --- .../maintain-coding-agent-benchmark/SKILL.md | 21 +-- docs/reference/performance.mdx | 2 +- justfile | 2 +- scripts/README.md | 6 +- .../benchmark_coding_agent_latency/README.md | 137 ++++++++++++++++++ .../__main__.py} | 2 +- .../benchmark_coding_agent_latency/config.py | 5 +- .../tests/__init__.py | 4 + .../tests/test_config.py} | 4 +- 9 files changed, 165 insertions(+), 18 deletions(-) create mode 100644 scripts/benchmark_coding_agent_latency/README.md rename scripts/{benchmark-coding-agent-latency.py => benchmark_coding_agent_latency/__main__.py} (89%) create mode 100644 scripts/benchmark_coding_agent_latency/tests/__init__.py rename scripts/{tests/test_benchmark_coding_agent_latency.py => benchmark_coding_agent_latency/tests/test_config.py} (94%) diff --git a/.agents/skills/maintain-coding-agent-benchmark/SKILL.md b/.agents/skills/maintain-coding-agent-benchmark/SKILL.md index 717c42af2..a3a28fd17 100644 --- a/.agents/skills/maintain-coding-agent-benchmark/SKILL.md +++ b/.agents/skills/maintain-coding-agent-benchmark/SKILL.md @@ -12,7 +12,11 @@ final checks. Keep benchmark changes isolated from runtime behavior. ## Understand The Layout -- Use `scripts/benchmark-coding-agent-latency.py` as the stable entry point. +- Use `python -m scripts.benchmark_coding_agent_latency` as the direct entry + point and `just benchmark-coding-agent-latency` as the normal wrapper. +- Keep the human run guide in + `scripts/benchmark_coding_agent_latency/README.md` current with CLI and + configuration changes. - Read `scripts/benchmark_coding_agent_latency/data/default.toml` before a run. A custom TOML file overlays these defaults, then CLI arguments take final precedence. @@ -31,7 +35,7 @@ final checks. Keep benchmark changes isolated from runtime behavior. List every config override without building Relay: ```bash -uv run python scripts/benchmark-coding-agent-latency.py --help +uv run python -m scripts.benchmark_coding_agent_latency --help ``` Run a small functional check before a statistically meaningful run: @@ -95,7 +99,7 @@ To add a test suite: 3. Dispatch it conditionally in `cli.py` and report it conditionally in `reporting.py`. 4. Add config and selection tests in - `scripts/tests/test_benchmark_coding_agent_latency.py`. + `scripts/benchmark_coding_agent_latency/tests/test_config.py`. 5. Update `scripts/README.md` and `docs/reference/performance.mdx`. To add a provider, mode, or matrix axis, update config validation, the protocol @@ -107,13 +111,10 @@ together. Add static fixture files when the change introduces fixed text. Format and test the focused surface first: ```bash -uv run ruff format scripts/benchmark-coding-agent-latency.py \ - scripts/benchmark_coding_agent_latency \ - scripts/tests/test_benchmark_coding_agent_latency.py -uv run ruff check scripts/benchmark-coding-agent-latency.py \ - scripts/benchmark_coding_agent_latency \ - scripts/tests/test_benchmark_coding_agent_latency.py -uv run python -m unittest scripts.tests.test_benchmark_coding_agent_latency +uv run ruff format scripts/benchmark_coding_agent_latency +uv run ruff check scripts/benchmark_coding_agent_latency +uv run python -m unittest \ + scripts.benchmark_coding_agent_latency.tests.test_config ``` Run the small functional command above after lifecycle, protocol, server, diff --git a/docs/reference/performance.mdx b/docs/reference/performance.mdx index 3cc2e8993..4809a8229 100644 --- a/docs/reference/performance.mdx +++ b/docs/reference/performance.mdx @@ -92,7 +92,7 @@ just benchmark-coding-agent-latency \ ``` The selectable suites are `gateway`, `hooks`, and `startup`. Run -`uv run python scripts/benchmark-coding-agent-latency.py --help` to see every +`uv run python -m scripts.benchmark_coding_agent_latency --help` to see every matrix and sample-count override. Results contain only the selected suite sections. diff --git a/justfile b/justfile index 4d3b13f90..6e6198ce4 100644 --- a/justfile +++ b/justfile @@ -1120,7 +1120,7 @@ benchmark-coding-agent-latency *benchmark_args: result_dir="${result_dir:-target/benchmark-results}" benchmark_args=({{ benchmark_args }}) cargo build --release -p nemo-relay-cli - uv run python scripts/benchmark-coding-agent-latency.py \ + uv run python -m scripts.benchmark_coding_agent_latency \ --relay-bin target/release/nemo-relay \ --output "$result_dir/coding-agent-latency.json" \ "${benchmark_args[@]}" diff --git a/scripts/README.md b/scripts/README.md index 3aa43886c..ae181c3ec 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -48,9 +48,11 @@ just benchmark-coding-agent-latency \ --samples 10 ``` -Run `uv run python scripts/benchmark-coding-agent-latency.py --help` to list +Run `uv run python -m scripts.benchmark_coding_agent_latency --help` to list all overrides. The three selectable suites are `gateway`, `hooks`, and -`startup`. +`startup`. See +[`benchmark_coding_agent_latency/README.md`](benchmark_coding_agent_latency/README.md) +for the complete human-facing run guide. ## Internal Layout diff --git a/scripts/benchmark_coding_agent_latency/README.md b/scripts/benchmark_coding_agent_latency/README.md new file mode 100644 index 000000000..4ab27b1cd --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/README.md @@ -0,0 +1,137 @@ + + +# Coding-Agent Latency Benchmark + +Use this opt-in benchmark to measure the local latency that NeMo Relay adds +around OpenAI Responses, Anthropic Messages, Codex hooks, Claude Code hooks, +and Relay process startup. The fixture runs deterministic providers on +loopback, so network and model-service latency do not hide Relay overhead. + +Run all commands from the repository root. The default matrix is intentionally +large and its file-exporter scenarios can temporarily write tens of gigabytes +of ATOF data. Start with the smoke test unless you are collecting reportable +performance results. + +## Prerequisites + +Install the repository development prerequisites, including Rust, Python 3.11 +or newer, `uv`, and `just`. The `just` recipe builds the release-mode Relay CLI +before running the benchmark. + +## Run a Smoke Test + +Use a small matrix to verify the fixture and exporter paths: + +```bash +just benchmark-coding-agent-latency \ + --tests gateway \ + --providers openai \ + --modes buffered \ + --payload-sizes 4096 \ + --concurrency 1 \ + --samples 5 \ + --warmup 1 \ + --response-bytes 1024 +``` + +Do not use a smoke-test result for performance conclusions. Its sample count +is only large enough to catch functional failures. + +## Run the Default Matrix + +After you check available disk space, run the default matrix with the following +command: + +```bash +just benchmark-coding-agent-latency +``` + +The default configuration runs three suites: + +| Suite | What It Measures | +| --- | --- | +| `gateway` | Direct loopback calls compared with minimal, ATOF file, and OTLP Relay gateways | +| `hooks` | Codex and Claude Code `hook-forward` subprocess wall time | +| `startup` | Cold Relay process startup through gateway readiness | + +The gateway suite covers OpenAI and Anthropic, buffered and streaming +responses, multiple request sizes, and multiple concurrency levels. + +## Configure a Run + +The benchmark resolves settings in this order: + +1. Defaults from `data/default.toml`. +2. Values from the file passed with `--config`. +3. CLI arguments, which take final precedence. + +A custom TOML file can contain only the settings that differ from the +defaults. For example: + +```toml +tests = ["gateway"] +providers = ["openai"] +modes = ["streaming"] +samples = 50 +warmup = 3 +payload_sizes = [4096, 65536] +concurrency = [1, 4] +``` + +Run the custom configuration with the following command: + +```bash +just benchmark-coding-agent-latency --config /path/to/benchmark.toml +``` + +Override any list from the command line with comma-separated values: + +```bash +just benchmark-coding-agent-latency \ + --config /path/to/benchmark.toml \ + --tests gateway,startup \ + --providers openai,anthropic \ + --modes buffered \ + --concurrency 1,4,8 +``` + +List every supported override without running the benchmark: + +```bash +uv run python -m scripts.benchmark_coding_agent_latency --help +``` + +## Read the Results + +The command prints a terminal summary and writes +`target/benchmark-results/coding-agent-latency.json`. Use the following command +to choose another directory: + +```bash +just output_dir=/tmp/relay-benchmarks benchmark-coding-agent-latency +``` + +The JSON report records the resolved matrix, environment, absolute latency, +paired latency differences, and exporter-delivery counts. Gateway results +include total latency; streaming results also include time to first content. +Summaries include p50, p95, p99, and a bootstrap 95% confidence interval for +the median difference. + +When comparing variants, prefer added milliseconds over percentages. Record +the commit, release build, hardware, operating system, matrix, and sample count +with any shared result. Small loopback baselines can make harmless absolute +differences look large as percentages. + +## Troubleshoot + +- A loopback bind error means the environment must allow local HTTP listeners. +- An exporter-delivery error means the ATOF file or OTLP receiver observed no + benchmark events. Rerun a small gateway suite to isolate the exporter path; + Relay startup failures include their captured log output. +- A validation error names the invalid TOML or CLI value. Gateway samples must + be at least as large as every requested concurrency value. +- An interrupted run removes its temporary workspace, but a default run still + needs enough free disk space while it is active. diff --git a/scripts/benchmark-coding-agent-latency.py b/scripts/benchmark_coding_agent_latency/__main__.py similarity index 89% rename from scripts/benchmark-coding-agent-latency.py rename to scripts/benchmark_coding_agent_latency/__main__.py index 63ec6cf8f..df484528f 100644 --- a/scripts/benchmark-coding-agent-latency.py +++ b/scripts/benchmark_coding_agent_latency/__main__.py @@ -7,7 +7,7 @@ import sys -from benchmark_coding_agent_latency.cli import main +from .cli import main if __name__ == "__main__": try: diff --git a/scripts/benchmark_coding_agent_latency/config.py b/scripts/benchmark_coding_agent_latency/config.py index ed643f9c6..4fc1e5bd3 100644 --- a/scripts/benchmark_coding_agent_latency/config.py +++ b/scripts/benchmark_coding_agent_latency/config.py @@ -230,7 +230,10 @@ def _arg_nonnegative_int(value: str) -> int: def parse_args(argv: list[str] | None = None) -> CliOptions: """Parse CLI options, applying them after values from the config file.""" - parser = argparse.ArgumentParser(description="Measure local coding-agent gateway latency.") + parser = argparse.ArgumentParser( + prog="python -m scripts.benchmark_coding_agent_latency", + description="Measure local coding-agent gateway latency.", + ) parser.add_argument("--relay-bin", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG_PATH, help="TOML overrides for the benchmark") diff --git a/scripts/benchmark_coding_agent_latency/tests/__init__.py b/scripts/benchmark_coding_agent_latency/tests/__init__.py new file mode 100644 index 000000000..9b652833e --- /dev/null +++ b/scripts/benchmark_coding_agent_latency/tests/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the coding-agent latency benchmark.""" diff --git a/scripts/tests/test_benchmark_coding_agent_latency.py b/scripts/benchmark_coding_agent_latency/tests/test_config.py similarity index 94% rename from scripts/tests/test_benchmark_coding_agent_latency.py rename to scripts/benchmark_coding_agent_latency/tests/test_config.py index 1f71c5785..ea7e4c2e8 100644 --- a/scripts/tests/test_benchmark_coding_agent_latency.py +++ b/scripts/benchmark_coding_agent_latency/tests/test_config.py @@ -7,8 +7,8 @@ import unittest from pathlib import Path -from scripts.benchmark_coding_agent_latency.config import DEFAULT_CONFIG_PATH, load_config, parse_args -from scripts.benchmark_coding_agent_latency.fixtures import write_agent_config, write_fake_codex, write_plugin_configs +from ..config import DEFAULT_CONFIG_PATH, load_config, parse_args +from ..fixtures import write_agent_config, write_fake_codex, write_plugin_configs class BenchmarkConfigTests(unittest.TestCase): From e17462619cedb04a475cce710e4755c189abdba2 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 5 Aug 2026 13:54:53 -0700 Subject: [PATCH 03/14] refactor: separate benchmark configuration assets Signed-off-by: Yuchen Zhang --- .../maintain-coding-agent-benchmark/SKILL.md | 8 ++++---- docs/reference/performance.mdx | 2 +- scripts/README.md | 2 +- .../benchmark_coding_agent_latency/README.md | 2 +- .../benchmark_coding_agent_latency/config.py | 3 ++- .../{data => config}/agent-config.toml | 0 .../{data => config}/default.toml | 0 .../{data => config}/plugins-file.toml | 0 .../{data => config}/plugins-minimal.toml | 0 .../{data => config}/plugins-otlp.toml | 0 .../{data => config}/relay-config.toml | 0 .../fixtures.py | 20 +++++++++++-------- 12 files changed, 21 insertions(+), 16 deletions(-) rename scripts/benchmark_coding_agent_latency/{data => config}/agent-config.toml (100%) rename scripts/benchmark_coding_agent_latency/{data => config}/default.toml (100%) rename scripts/benchmark_coding_agent_latency/{data => config}/plugins-file.toml (100%) rename scripts/benchmark_coding_agent_latency/{data => config}/plugins-minimal.toml (100%) rename scripts/benchmark_coding_agent_latency/{data => config}/plugins-otlp.toml (100%) rename scripts/benchmark_coding_agent_latency/{data => config}/relay-config.toml (100%) diff --git a/.agents/skills/maintain-coding-agent-benchmark/SKILL.md b/.agents/skills/maintain-coding-agent-benchmark/SKILL.md index a3a28fd17..dea7921d5 100644 --- a/.agents/skills/maintain-coding-agent-benchmark/SKILL.md +++ b/.agents/skills/maintain-coding-agent-benchmark/SKILL.md @@ -17,7 +17,7 @@ final checks. Keep benchmark changes isolated from runtime behavior. - Keep the human run guide in `scripts/benchmark_coding_agent_latency/README.md` current with CLI and configuration changes. -- Read `scripts/benchmark_coding_agent_latency/data/default.toml` before a run. +- Read `scripts/benchmark_coding_agent_latency/config/default.toml` before a run. A custom TOML file overlays these defaults, then CLI arguments take final precedence. - Change config parsing and validation in `config.py`. @@ -26,9 +26,9 @@ final checks. Keep benchmark changes isolated from runtime behavior. - Keep temporary Relay and coding-agent process lifecycle in `processes.py`. - Add measurement logic to `benchmarks.py`, orchestration to `cli.py`, and terminal presentation to `reporting.py`. -- Put multi-line configs, scripts, and other fixed fixture text under `data/`. - Load or render those assets through `fixtures.py`; do not embed them in - executable modules. +- Put TOML assets under `config/` and platform scripts under `data/`. Load or + render those assets through `fixtures.py`; do not embed them in executable + modules. ## Run The Fixture diff --git a/docs/reference/performance.mdx b/docs/reference/performance.mdx index 4809a8229..d3fab03f2 100644 --- a/docs/reference/performance.mdx +++ b/docs/reference/performance.mdx @@ -59,7 +59,7 @@ p50, p95, and p99 paired latency differences and a bootstrap 95% confidence interval for the median. Configure a run with a partial TOML file. Unspecified values inherit from -`scripts/benchmark_coding_agent_latency/data/default.toml`: +`scripts/benchmark_coding_agent_latency/config/default.toml`: ```toml tests = ["gateway"] diff --git a/scripts/README.md b/scripts/README.md index ae181c3ec..3e4528e98 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -35,7 +35,7 @@ gateway startup time. It writes structured results under `target/benchmark-results/` by default and is intentionally outside regular CI. The defaults live in -`scripts/benchmark_coding_agent_latency/data/default.toml`. Supply a partial +`scripts/benchmark_coding_agent_latency/config/default.toml`. Supply a partial TOML file with `--config`, or override individual values on the command line. For example, this runs only a small OpenAI gateway matrix: diff --git a/scripts/benchmark_coding_agent_latency/README.md b/scripts/benchmark_coding_agent_latency/README.md index 4ab27b1cd..445822651 100644 --- a/scripts/benchmark_coding_agent_latency/README.md +++ b/scripts/benchmark_coding_agent_latency/README.md @@ -64,7 +64,7 @@ responses, multiple request sizes, and multiple concurrency levels. The benchmark resolves settings in this order: -1. Defaults from `data/default.toml`. +1. Defaults from `config/default.toml`. 2. Values from the file passed with `--config`. 3. CLI arguments, which take final precedence. diff --git a/scripts/benchmark_coding_agent_latency/config.py b/scripts/benchmark_coding_agent_latency/config.py index 4fc1e5bd3..3f6bd119b 100644 --- a/scripts/benchmark_coding_agent_latency/config.py +++ b/scripts/benchmark_coding_agent_latency/config.py @@ -13,7 +13,8 @@ PACKAGE_ROOT = Path(__file__).resolve().parent DATA_ROOT = PACKAGE_ROOT / "data" -DEFAULT_CONFIG_PATH = DATA_ROOT / "default.toml" +CONFIG_ROOT = PACKAGE_ROOT / "config" +DEFAULT_CONFIG_PATH = CONFIG_ROOT / "default.toml" AVAILABLE_TESTS = ("gateway", "hooks", "startup") AVAILABLE_PROVIDERS = ("openai", "anthropic") diff --git a/scripts/benchmark_coding_agent_latency/data/agent-config.toml b/scripts/benchmark_coding_agent_latency/config/agent-config.toml similarity index 100% rename from scripts/benchmark_coding_agent_latency/data/agent-config.toml rename to scripts/benchmark_coding_agent_latency/config/agent-config.toml diff --git a/scripts/benchmark_coding_agent_latency/data/default.toml b/scripts/benchmark_coding_agent_latency/config/default.toml similarity index 100% rename from scripts/benchmark_coding_agent_latency/data/default.toml rename to scripts/benchmark_coding_agent_latency/config/default.toml diff --git a/scripts/benchmark_coding_agent_latency/data/plugins-file.toml b/scripts/benchmark_coding_agent_latency/config/plugins-file.toml similarity index 100% rename from scripts/benchmark_coding_agent_latency/data/plugins-file.toml rename to scripts/benchmark_coding_agent_latency/config/plugins-file.toml diff --git a/scripts/benchmark_coding_agent_latency/data/plugins-minimal.toml b/scripts/benchmark_coding_agent_latency/config/plugins-minimal.toml similarity index 100% rename from scripts/benchmark_coding_agent_latency/data/plugins-minimal.toml rename to scripts/benchmark_coding_agent_latency/config/plugins-minimal.toml diff --git a/scripts/benchmark_coding_agent_latency/data/plugins-otlp.toml b/scripts/benchmark_coding_agent_latency/config/plugins-otlp.toml similarity index 100% rename from scripts/benchmark_coding_agent_latency/data/plugins-otlp.toml rename to scripts/benchmark_coding_agent_latency/config/plugins-otlp.toml diff --git a/scripts/benchmark_coding_agent_latency/data/relay-config.toml b/scripts/benchmark_coding_agent_latency/config/relay-config.toml similarity index 100% rename from scripts/benchmark_coding_agent_latency/data/relay-config.toml rename to scripts/benchmark_coding_agent_latency/config/relay-config.toml diff --git a/scripts/benchmark_coding_agent_latency/fixtures.py b/scripts/benchmark_coding_agent_latency/fixtures.py index ad18c405e..627fff75d 100644 --- a/scripts/benchmark_coding_agent_latency/fixtures.py +++ b/scripts/benchmark_coding_agent_latency/fixtures.py @@ -9,15 +9,19 @@ import os from pathlib import Path -from .config import DATA_ROOT +from .config import CONFIG_ROOT, DATA_ROOT def _read_data(name: str) -> str: return (DATA_ROOT / name).read_text(encoding="utf-8") -def _render_data(name: str, replacements: dict[str, str]) -> str: - rendered = _read_data(name) +def _read_config(name: str) -> str: + return (CONFIG_ROOT / name).read_text(encoding="utf-8") + + +def _render_config(name: str, replacements: dict[str, str]) -> str: + rendered = _read_config(name) for marker, value in replacements.items(): if marker not in rendered: raise RuntimeError(f"static fixture {name} is missing marker {marker}") @@ -32,7 +36,7 @@ def toml_string(value: str | Path) -> str: def write_relay_config(root: Path) -> Path: path = root / "config.toml" - path.write_text(_read_data("relay-config.toml"), encoding="utf-8") + path.write_text(_read_config("relay-config.toml"), encoding="utf-8") return path @@ -43,16 +47,16 @@ def write_plugin_configs(root: Path, otlp_url: str) -> dict[str, Path]: "relay-file": root / "plugins-file.toml", "relay-otlp": root / "plugins-otlp.toml", } - paths["relay-minimal"].write_text(_read_data("plugins-minimal.toml"), encoding="utf-8") + paths["relay-minimal"].write_text(_read_config("plugins-minimal.toml"), encoding="utf-8") atof_dir = root / "atof" atof_dir.mkdir() paths["relay-file"].write_text( - _render_data("plugins-file.toml", {'"__ATOF_OUTPUT_DIRECTORY__"': toml_string(atof_dir)}), + _render_config("plugins-file.toml", {'"__ATOF_OUTPUT_DIRECTORY__"': toml_string(atof_dir)}), encoding="utf-8", ) paths["relay-otlp"].write_text( - _render_data("plugins-otlp.toml", {'"__OTLP_ENDPOINT__"': toml_string(f"{otlp_url}/v1/traces")}), + _render_config("plugins-otlp.toml", {'"__OTLP_ENDPOINT__"': toml_string(f"{otlp_url}/v1/traces")}), encoding="utf-8", ) return paths @@ -72,7 +76,7 @@ def write_fake_codex(root: Path) -> Path: def write_agent_config(root: Path, name: str, fake_codex: Path) -> Path: path = root / f"{name}-config.toml" path.write_text( - _render_data("agent-config.toml", {'"__CODEX_COMMAND__"': toml_string(fake_codex)}), + _render_config("agent-config.toml", {'"__CODEX_COMMAND__"': toml_string(fake_codex)}), encoding="utf-8", ) return path From 47ebfa7e1f1474ea0abed21173a189e4139de03e Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 5 Aug 2026 15:43:16 -0700 Subject: [PATCH 04/14] test: expand NeMo Relay latency benchmark Signed-off-by: Yuchen Zhang --- .../maintain-coding-agent-benchmark/SKILL.md | 139 ++++- docs/reference/performance.mdx | 53 +- justfile | 11 +- scripts/README.md | 29 +- .../benchmark_coding_agent_latency/README.md | 137 ----- .../benchmark_coding_agent_latency/config.py | 295 ----------- .../tests/test_config.py | 93 ---- scripts/latency_benchmark/README.md | 290 +++++++++++ .../config/agent-config.toml | 0 .../config/default.toml | 1 + .../config/plugins-file.toml | 0 .../config/plugins-minimal.toml | 0 .../config/plugins-otlp.toml | 0 .../config/plugins-pii-redaction.toml | 19 + .../config/relay-config.toml | 0 .../data/mock-codex.cmd} | 0 .../data/mock-codex.sh} | 0 .../src}/__init__.py | 0 .../src}/__main__.py | 0 .../src}/benchmarks.py | 65 +-- .../src}/cli.py | 7 +- scripts/latency_benchmark/src/config.py | 483 ++++++++++++++++++ .../src}/fixtures.py | 23 +- scripts/latency_benchmark/src/html_report.py | 44 ++ .../src}/processes.py | 5 +- .../src}/protocol.py | 0 .../latency_benchmark/src/report/report.js | 470 +++++++++++++++++ .../latency_benchmark/src/report/styles.css | 434 ++++++++++++++++ .../src/report/template.html | 213 ++++++++ .../src}/reporting.py | 12 +- .../src}/servers.py | 0 .../tests/__init__.py | 0 .../latency_benchmark/tests/test_config.py | 214 ++++++++ .../tests/test_html_report.py | 126 +++++ .../latency_benchmark/tests/test_processes.py | 55 ++ 35 files changed, 2582 insertions(+), 636 deletions(-) delete mode 100644 scripts/benchmark_coding_agent_latency/README.md delete mode 100644 scripts/benchmark_coding_agent_latency/config.py delete mode 100644 scripts/benchmark_coding_agent_latency/tests/test_config.py create mode 100644 scripts/latency_benchmark/README.md rename scripts/{benchmark_coding_agent_latency => latency_benchmark}/config/agent-config.toml (100%) rename scripts/{benchmark_coding_agent_latency => latency_benchmark}/config/default.toml (97%) rename scripts/{benchmark_coding_agent_latency => latency_benchmark}/config/plugins-file.toml (100%) rename scripts/{benchmark_coding_agent_latency => latency_benchmark}/config/plugins-minimal.toml (100%) rename scripts/{benchmark_coding_agent_latency => latency_benchmark}/config/plugins-otlp.toml (100%) create mode 100644 scripts/latency_benchmark/config/plugins-pii-redaction.toml rename scripts/{benchmark_coding_agent_latency => latency_benchmark}/config/relay-config.toml (100%) rename scripts/{benchmark_coding_agent_latency/data/fake-codex.cmd => latency_benchmark/data/mock-codex.cmd} (100%) rename scripts/{benchmark_coding_agent_latency/data/fake-codex.sh => latency_benchmark/data/mock-codex.sh} (100%) rename scripts/{benchmark_coding_agent_latency => latency_benchmark/src}/__init__.py (100%) rename scripts/{benchmark_coding_agent_latency => latency_benchmark/src}/__main__.py (100%) rename scripts/{benchmark_coding_agent_latency => latency_benchmark/src}/benchmarks.py (86%) rename scripts/{benchmark_coding_agent_latency => latency_benchmark/src}/cli.py (95%) create mode 100644 scripts/latency_benchmark/src/config.py rename scripts/{benchmark_coding_agent_latency => latency_benchmark/src}/fixtures.py (79%) create mode 100644 scripts/latency_benchmark/src/html_report.py rename scripts/{benchmark_coding_agent_latency => latency_benchmark/src}/processes.py (98%) rename scripts/{benchmark_coding_agent_latency => latency_benchmark/src}/protocol.py (100%) create mode 100644 scripts/latency_benchmark/src/report/report.js create mode 100644 scripts/latency_benchmark/src/report/styles.css create mode 100644 scripts/latency_benchmark/src/report/template.html rename scripts/{benchmark_coding_agent_latency => latency_benchmark/src}/reporting.py (90%) rename scripts/{benchmark_coding_agent_latency => latency_benchmark/src}/servers.py (100%) rename scripts/{benchmark_coding_agent_latency => latency_benchmark}/tests/__init__.py (100%) create mode 100644 scripts/latency_benchmark/tests/test_config.py create mode 100644 scripts/latency_benchmark/tests/test_html_report.py create mode 100644 scripts/latency_benchmark/tests/test_processes.py diff --git a/.agents/skills/maintain-coding-agent-benchmark/SKILL.md b/.agents/skills/maintain-coding-agent-benchmark/SKILL.md index dea7921d5..b268a56ff 100644 --- a/.agents/skills/maintain-coding-agent-benchmark/SKILL.md +++ b/.agents/skills/maintain-coding-agent-benchmark/SKILL.md @@ -1,6 +1,6 @@ --- name: maintain-coding-agent-benchmark -description: Run, configure, troubleshoot, maintain, or expand the NeMo Relay coding-agent latency benchmark fixture. Use for changes under scripts/benchmark_coding_agent_latency, new benchmark suites or matrix axes, static provider and Relay fixtures, result reporting, or coding-agent benchmark documentation. +description: Run, configure, troubleshoot, maintain, or expand the NeMo Relay coding-agent latency benchmark fixture. Use for changes under scripts/latency_benchmark, custom middleware benchmark variants, new benchmark suites or matrix axes, static provider and Relay fixtures, result reporting, or coding-agent benchmark documentation. --- # Maintain The Coding-Agent Latency Benchmark @@ -12,36 +12,46 @@ final checks. Keep benchmark changes isolated from runtime behavior. ## Understand The Layout -- Use `python -m scripts.benchmark_coding_agent_latency` as the direct entry - point and `just benchmark-coding-agent-latency` as the normal wrapper. +- Use `python -m scripts.latency_benchmark.src` as the direct entry + point and `just latency-benchmark` as the normal wrapper. - Keep the human run guide in - `scripts/benchmark_coding_agent_latency/README.md` current with CLI and + `scripts/latency_benchmark/README.md` current with CLI and configuration changes. -- Read `scripts/benchmark_coding_agent_latency/config/default.toml` before a run. +- Read `scripts/latency_benchmark/config/default.toml` before a run. A custom TOML file overlays these defaults, then CLI arguments take final precedence. -- Change config parsing and validation in `config.py`. -- Keep OpenAI and Anthropic payload shapes in `protocol.py`. -- Keep loopback provider and OTLP behavior in `servers.py`. -- Keep temporary Relay and coding-agent process lifecycle in `processes.py`. -- Add measurement logic to `benchmarks.py`, orchestration to `cli.py`, and - terminal presentation to `reporting.py`. +- Keep all runtime Python modules under `src/`. Change config parsing and + validation in `src/config.py`. +- Keep OpenAI and Anthropic payload shapes in `src/protocol.py`. +- Keep loopback provider and OTLP behavior in `src/servers.py`. +- Keep temporary Relay and coding-agent process lifecycle in `src/processes.py`. +- Add measurement logic to `src/benchmarks.py`, orchestration to `src/cli.py`, + and terminal presentation to `src/reporting.py`. +- Keep HTML report assembly in `src/html_report.py` and its static template, + CSS, and JavaScript under `src/report/`. Keep the report self-contained and + offline. - Put TOML assets under `config/` and platform scripts under `data/`. Load or - render those assets through `fixtures.py`; do not embed them in executable + render those assets through `src/fixtures.py`; do not embed them in executable modules. +- Use `config/plugins-pii-redaction.toml` as the self-contained real-middleware + smoke fixture. It uses the built-in email detector and requires no external + service. +- Treat `data/mock-codex.*` as the transparent-mode lifecycle stub, not as the + Codex hook implementation. Hook measurements call `hook-forward` directly for + both Codex and Claude Code, so they do not require a mock Claude executable. ## Run The Fixture List every config override without building Relay: ```bash -uv run python -m scripts.benchmark_coding_agent_latency --help +just latency-benchmark --help ``` Run a small functional check before a statistically meaningful run: ```bash -just benchmark-coding-agent-latency \ +just latency-benchmark \ --tests gateway \ --providers openai \ --modes buffered \ @@ -53,9 +63,19 @@ just benchmark-coding-agent-latency \ ``` Treat a small run only as a correctness check. Run the default matrix with -`just benchmark-coding-agent-latency` when collecting performance data. The +`just latency-benchmark` when collecting performance data. The default file-exporter matrix can write tens of gigabytes temporarily, so check -free disk space first. +free disk space first. With the current defaults, 11,860 file-exporter gateway +requests contain about 12.3 GiB of request content and are expected to produce +about 25 GiB of ATOF JSON. Reserve at least 30 GiB; treat these values as an +estimate because event serialization can change. + +The benchmark writes this large ATOF output and other ephemeral Relay files to +the operating system's temporary directory in a folder named +`nemo-relay-latency-*` by default. This directory is typically under `/tmp`; +macOS can use a private per-user temporary directory instead. A normal run +removes the directory and its large output. The JSON result and HTML report are +smaller persistent artifacts in the configured result directory. Use a partial TOML config for repeatable experiments: @@ -69,14 +89,68 @@ concurrency = [1, 4] ``` ```bash -just benchmark-coding-agent-latency --config /path/to/benchmark.toml +just latency-benchmark --config /path/to/benchmark.toml ``` -Find the JSON report at -`target/benchmark-results/coding-agent-latency.json` unless `output_dir` was -overridden. Compare added milliseconds and paired confidence intervals; do not +Keep the minimal, file, and OTLP variants enabled on every run. Add opt-in +middleware variants through `[[middleware]]` benchmark config tables: + +```toml +[[middleware]] +name = "pii-redaction" +plugin_config = "./plugins-pii-redaction.toml" +``` + +Run the bundled PII-redaction middleware through a small gateway matrix: + +```bash +just latency-benchmark \ + --tests gateway \ + --providers openai \ + --modes buffered \ + --payload-sizes 4096 \ + --concurrency 1 \ + --samples 5 \ + --warmup 1 \ + --response-bytes 1024 \ + --middleware pii-redaction=scripts/latency_benchmark/config/plugins-pii-redaction.toml +``` + +Treat this command as a middleware lifecycle and reporting check. The static +payload contains no email address, so it does not verify redaction correctness. + +Resolve TOML plugin paths relative to the benchmark config. Use repeatable +`--middleware NAME=PATH` options for one-off CLI variants; CLI middleware +entries replace those from the TOML file. Compare each custom gateway variant +with direct calls and minimal Relay, and include it in selected hook and startup +suites. + +Find the JSON result and self-contained HTML report at +`target/benchmark-results/nemo-relay-latency-report.json` and +`target/benchmark-results/nemo-relay-latency-report.html` unless `output_dir` +was overridden. Use `--report` only when the HTML path must differ from the +JSON path. Compare added milliseconds and paired confidence intervals; do not draw performance conclusions from a smoke run. +## Interpret Suites And Metrics + +- Treat gateway `total` as request start through buffered-body completion or + streaming end-of-stream. Treat streaming `first_content` as request start + through the first content-delta event. +- Use gateway Relay-versus-direct paired deltas for total Relay overhead. Use + file-versus-minimal and OTLP-versus-minimal deltas to isolate exporter + overhead. +- Treat hook absolute values as complete `hook-forward` subprocess wall time. + Hook paired deltas subtract a `nemo-relay --version` process measurement from + the same cycle. +- Treat startup absolute values as process launch through healthy gateway + readiness. Startup paired deltas subtract the same process baseline. +- Read p50 as the median and p95/p99 as tail percentiles. Min and max are + observed extremes. `median_ci95_ms` is a bootstrap uncertainty interval for + the median paired delta, not an interval containing 95% of observations. +- Treat exporter-delivery bytes and request counts as correctness checks, not + latency metrics. + ## Maintain Measurement Integrity - Keep providers on loopback and deterministic. Do not add model-service or @@ -86,6 +160,8 @@ draw performance conclusions from a smoke run. - Warm persistent connections before recording gateway samples. - Keep streaming time-to-first-content separate from total stream time. - Preserve exporter-delivery checks when gateway or hook traffic is measured. +- Keep the three default variants when adding custom middleware. Validate + custom names and plugin paths before starting subprocesses. - Record all resolved matrix values in the JSON result so another engineer can reproduce the run. - Keep temporary state isolated from the developer's home and Relay config. @@ -94,27 +170,34 @@ draw performance conclusions from a smoke run. To add a test suite: -1. Add its name to `AVAILABLE_TESTS` in `config.py`. -2. Implement the measurement in `benchmarks.py`. -3. Dispatch it conditionally in `cli.py` and report it conditionally in - `reporting.py`. +1. Add its name to `AVAILABLE_TESTS` in `src/config.py`. +2. Implement the measurement in `src/benchmarks.py`. +3. Dispatch it conditionally in `src/cli.py` and report it conditionally in + `src/reporting.py` and the HTML report. 4. Add config and selection tests in - `scripts/benchmark_coding_agent_latency/tests/test_config.py`. + `scripts/latency_benchmark/tests/test_config.py`. 5. Update `scripts/README.md` and `docs/reference/performance.mdx`. To add a provider, mode, or matrix axis, update config validation, the protocol fixture, the loopback server, orchestration, result parameters, and tests together. Add static fixture files when the change introduces fixed text. +To change middleware variant behavior, update config parsing, fixture config +selection, all selected suite loops, terminal output, HTML series discovery, +tests, and the human README together. Do not hard-code a custom middleware name +in reporting. + ## Validate Changes Format and test the focused surface first: ```bash -uv run ruff format scripts/benchmark_coding_agent_latency -uv run ruff check scripts/benchmark_coding_agent_latency +uv run ruff format scripts/latency_benchmark +uv run ruff check scripts/latency_benchmark uv run python -m unittest \ - scripts.benchmark_coding_agent_latency.tests.test_config + scripts.latency_benchmark.tests.test_config \ + scripts.latency_benchmark.tests.test_html_report \ + scripts.latency_benchmark.tests.test_processes ``` Run the small functional command above after lifecycle, protocol, server, diff --git a/docs/reference/performance.mdx b/docs/reference/performance.mdx index d3fab03f2..c9a852eea 100644 --- a/docs/reference/performance.mdx +++ b/docs/reference/performance.mdx @@ -37,11 +37,11 @@ loopback OpenAI and Anthropic providers so network and model-service latency do not hide Relay's contribution. ```bash -just benchmark-coding-agent-latency +just latency-benchmark ``` -The command builds the release CLI. Its default configuration measures these -paths: +The command builds the release CLI. Its default configuration measures the +following paths: - Direct requests to the mock provider. - Relay with no exporter, which isolates the managed gateway pipeline. @@ -59,7 +59,7 @@ p50, p95, and p99 paired latency differences and a bootstrap 95% confidence interval for the median. Configure a run with a partial TOML file. Unspecified values inherit from -`scripts/benchmark_coding_agent_latency/config/default.toml`: +`scripts/latency_benchmark/config/default.toml`: ```toml tests = ["gateway"] @@ -74,7 +74,7 @@ concurrency = [1, 4] Pass the file to the benchmark: ```bash -just benchmark-coding-agent-latency --config /path/to/benchmark.toml +just latency-benchmark --config /path/to/benchmark.toml ``` Command-line values take precedence over the TOML file. Use comma-separated @@ -82,7 +82,7 @@ values for list overrides. This example runs only the OpenAI streaming gateway suite, regardless of the values in the file: ```bash -just benchmark-coding-agent-latency \ +just latency-benchmark \ --config /path/to/benchmark.toml \ --tests gateway \ --providers openai \ @@ -92,22 +92,43 @@ just benchmark-coding-agent-latency \ ``` The selectable suites are `gateway`, `hooks`, and `startup`. Run -`uv run python -m scripts.benchmark_coding_agent_latency --help` to see every -matrix and sample-count override. Results contain only the selected suite -sections. +`just latency-benchmark --help` to see every matrix and +sample-count override without building Relay. Results contain only the +selected suite sections. + +The minimal, ATOF file-exporter, and OTLP variants run by default. Add an +opt-in middleware variant with a `[[middleware]]` table in the benchmark TOML +file: + +```toml +[[middleware]] +name = "pii-redaction" +plugin_config = "./plugins-pii-redaction.toml" +``` + +The plugin configuration path is relative to the benchmark TOML file. For a +one-off run, pass `--middleware NAME=PATH`; repeat the option for multiple +variants. Each custom gateway variant is compared with direct calls and the +minimal Relay variant, and each custom variant is included in the selected hook +and startup suites. The command prints a readable summary and writes structured JSON to -`target/benchmark-results/coding-agent-latency.json`. Set the repository-wide -`output_dir` variable to choose another result location: +`target/benchmark-results/nemo-relay-latency-report.json`. It also writes a +self-contained report with graphs and metric explanations to +`target/benchmark-results/nemo-relay-latency-report.html`. Set the +repository-wide `output_dir` variable to choose another result location: ```bash -just output_dir=/tmp/relay-benchmarks benchmark-coding-agent-latency +just output_dir=/tmp/relay-benchmarks latency-benchmark ``` -The local-file scenarios can write tens of gigabytes of temporary ATOF data -with the default matrix. The harness verifies exporter delivery for gateway and -hook runs and removes its temporary workspace after the run, but you should -confirm that the system has sufficient free disk space before starting it. +With the current default matrix, the local-file scenarios are expected to write +about 25 GiB of temporary ATOF data. Reserve at least 30 GiB of free space. By +default, this large output goes to the operating system's temporary directory +in a folder named `nemo-relay-latency-*`, typically under `/tmp`. The harness +verifies exporter delivery for gateway and hook runs and removes the temporary +workspace after a normal run. The JSON and HTML report files remain in the +result directory. Treat results as environment-specific. Record the commit, release build, hardware, operating system, workload sizes, and sample counts when sharing a diff --git a/justfile b/justfile index 6e6198ce4..15f0bc01d 100644 --- a/justfile +++ b/justfile @@ -1113,16 +1113,21 @@ test-hermes-mcp-e2e: ./scripts/test-hermes-mcp-e2e.sh # Opt-in: builds the release CLI and runs configurable local latency suites. -benchmark-coding-agent-latency *benchmark_args: +latency-benchmark *benchmark_args: #!/usr/bin/env bash set -euo pipefail result_dir={{ quote(output_dir) }} result_dir="${result_dir:-target/benchmark-results}" benchmark_args=({{ benchmark_args }}) + for argument in "${benchmark_args[@]}"; do + if [[ "$argument" == "-h" || "$argument" == "--help" ]]; then + exec uv run python -m scripts.latency_benchmark.src "${benchmark_args[@]}" + fi + done cargo build --release -p nemo-relay-cli - uv run python -m scripts.benchmark_coding_agent_latency \ + uv run python -m scripts.latency_benchmark.src \ --relay-bin target/release/nemo-relay \ - --output "$result_dir/coding-agent-latency.json" \ + --output "$result_dir/nemo-relay-latency-report.json" \ "${benchmark_args[@]}" # --set [output_dir=] [ci=true|false] diff --git a/scripts/README.md b/scripts/README.md index 3e4528e98..dbecbf070 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -28,19 +28,22 @@ These checks exercise installed coding-agent clients and are intentionally outsi ## Opt-In Performance Benchmark -Run `just benchmark-coding-agent-latency` to build the release CLI and compare +Run `just latency-benchmark` to build the release CLI and compare direct provider requests with Relay's minimal, local-file, and local-OTLP configurations. The benchmark also measures full hook subprocess and cold gateway startup time. It writes structured results under -`target/benchmark-results/` by default and is intentionally outside regular CI. +`target/benchmark-results/nemo-relay-latency-report.json` and +`target/benchmark-results/nemo-relay-latency-report.html` by default and is +intentionally outside regular CI. Large ATOF output goes to a +`nemo-relay-latency-*` directory in the operating system's temporary location +and is removed after a normal run. -The defaults live in -`scripts/benchmark_coding_agent_latency/config/default.toml`. Supply a partial -TOML file with `--config`, or override individual values on the command line. -For example, this runs only a small OpenAI gateway matrix: +The defaults live in `scripts/latency_benchmark/config/default.toml`. Supply a +partial TOML file with `--config`, or override individual values on the command +line. For example, this runs only a small OpenAI gateway matrix: ```bash -just benchmark-coding-agent-latency \ +just latency-benchmark \ --tests gateway \ --providers openai \ --payload-sizes 4096 \ @@ -48,11 +51,13 @@ just benchmark-coding-agent-latency \ --samples 10 ``` -Run `uv run python -m scripts.benchmark_coding_agent_latency --help` to list -all overrides. The three selectable suites are `gateway`, `hooks`, and -`startup`. See -[`benchmark_coding_agent_latency/README.md`](benchmark_coding_agent_latency/README.md) -for the complete human-facing run guide. +Run `just latency-benchmark --help` to list all overrides without +building Relay. The three selectable suites are `gateway`, `hooks`, and +`startup`. Each run writes machine-readable JSON and a self-contained HTML +report with graphs. Add repeatable `--middleware NAME=PATH` options to measure +custom Relay plugin configurations alongside the three default variants. Refer +to [`latency_benchmark/README.md`](latency_benchmark/README.md) for the complete +human-facing run guide. ## Internal Layout diff --git a/scripts/benchmark_coding_agent_latency/README.md b/scripts/benchmark_coding_agent_latency/README.md deleted file mode 100644 index 445822651..000000000 --- a/scripts/benchmark_coding_agent_latency/README.md +++ /dev/null @@ -1,137 +0,0 @@ - - -# Coding-Agent Latency Benchmark - -Use this opt-in benchmark to measure the local latency that NeMo Relay adds -around OpenAI Responses, Anthropic Messages, Codex hooks, Claude Code hooks, -and Relay process startup. The fixture runs deterministic providers on -loopback, so network and model-service latency do not hide Relay overhead. - -Run all commands from the repository root. The default matrix is intentionally -large and its file-exporter scenarios can temporarily write tens of gigabytes -of ATOF data. Start with the smoke test unless you are collecting reportable -performance results. - -## Prerequisites - -Install the repository development prerequisites, including Rust, Python 3.11 -or newer, `uv`, and `just`. The `just` recipe builds the release-mode Relay CLI -before running the benchmark. - -## Run a Smoke Test - -Use a small matrix to verify the fixture and exporter paths: - -```bash -just benchmark-coding-agent-latency \ - --tests gateway \ - --providers openai \ - --modes buffered \ - --payload-sizes 4096 \ - --concurrency 1 \ - --samples 5 \ - --warmup 1 \ - --response-bytes 1024 -``` - -Do not use a smoke-test result for performance conclusions. Its sample count -is only large enough to catch functional failures. - -## Run the Default Matrix - -After you check available disk space, run the default matrix with the following -command: - -```bash -just benchmark-coding-agent-latency -``` - -The default configuration runs three suites: - -| Suite | What It Measures | -| --- | --- | -| `gateway` | Direct loopback calls compared with minimal, ATOF file, and OTLP Relay gateways | -| `hooks` | Codex and Claude Code `hook-forward` subprocess wall time | -| `startup` | Cold Relay process startup through gateway readiness | - -The gateway suite covers OpenAI and Anthropic, buffered and streaming -responses, multiple request sizes, and multiple concurrency levels. - -## Configure a Run - -The benchmark resolves settings in this order: - -1. Defaults from `config/default.toml`. -2. Values from the file passed with `--config`. -3. CLI arguments, which take final precedence. - -A custom TOML file can contain only the settings that differ from the -defaults. For example: - -```toml -tests = ["gateway"] -providers = ["openai"] -modes = ["streaming"] -samples = 50 -warmup = 3 -payload_sizes = [4096, 65536] -concurrency = [1, 4] -``` - -Run the custom configuration with the following command: - -```bash -just benchmark-coding-agent-latency --config /path/to/benchmark.toml -``` - -Override any list from the command line with comma-separated values: - -```bash -just benchmark-coding-agent-latency \ - --config /path/to/benchmark.toml \ - --tests gateway,startup \ - --providers openai,anthropic \ - --modes buffered \ - --concurrency 1,4,8 -``` - -List every supported override without running the benchmark: - -```bash -uv run python -m scripts.benchmark_coding_agent_latency --help -``` - -## Read the Results - -The command prints a terminal summary and writes -`target/benchmark-results/coding-agent-latency.json`. Use the following command -to choose another directory: - -```bash -just output_dir=/tmp/relay-benchmarks benchmark-coding-agent-latency -``` - -The JSON report records the resolved matrix, environment, absolute latency, -paired latency differences, and exporter-delivery counts. Gateway results -include total latency; streaming results also include time to first content. -Summaries include p50, p95, p99, and a bootstrap 95% confidence interval for -the median difference. - -When comparing variants, prefer added milliseconds over percentages. Record -the commit, release build, hardware, operating system, matrix, and sample count -with any shared result. Small loopback baselines can make harmless absolute -differences look large as percentages. - -## Troubleshoot - -- A loopback bind error means the environment must allow local HTTP listeners. -- An exporter-delivery error means the ATOF file or OTLP receiver observed no - benchmark events. Rerun a small gateway suite to isolate the exporter path; - Relay startup failures include their captured log output. -- A validation error names the invalid TOML or CLI value. Gateway samples must - be at least as large as every requested concurrency value. -- An interrupted run removes its temporary workspace, but a default run still - needs enough free disk space while it is active. diff --git a/scripts/benchmark_coding_agent_latency/config.py b/scripts/benchmark_coding_agent_latency/config.py deleted file mode 100644 index 3f6bd119b..000000000 --- a/scripts/benchmark_coding_agent_latency/config.py +++ /dev/null @@ -1,295 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Configuration loading and command-line overrides for the benchmark.""" - -from __future__ import annotations - -import argparse -import tomllib -from dataclasses import dataclass, replace -from pathlib import Path -from typing import Any - -PACKAGE_ROOT = Path(__file__).resolve().parent -DATA_ROOT = PACKAGE_ROOT / "data" -CONFIG_ROOT = PACKAGE_ROOT / "config" -DEFAULT_CONFIG_PATH = CONFIG_ROOT / "default.toml" - -AVAILABLE_TESTS = ("gateway", "hooks", "startup") -AVAILABLE_PROVIDERS = ("openai", "anthropic") -AVAILABLE_MODES = ("buffered", "streaming") -CONFIG_KEYS = { - "tests", - "providers", - "modes", - "samples", - "hook_samples", - "startup_samples", - "warmup", - "payload_sizes", - "concurrency", - "response_bytes", - "stream_chunks", - "models", - "content", -} -TABLE_KEYS = { - "models": {"openai", "anthropic"}, - "content": {"request_fill", "response_fill"}, -} - - -@dataclass(frozen=True) -class BenchmarkConfig: - """Validated benchmark matrix and sample settings.""" - - tests: tuple[str, ...] - providers: tuple[str, ...] - modes: tuple[str, ...] - samples: int - hook_samples: int - startup_samples: int - warmup: int - payload_sizes: tuple[int, ...] - concurrency: tuple[int, ...] - response_bytes: int - stream_chunks: int - openai_model: str - anthropic_model: str - request_fill: str - response_fill: str - - def parameters(self) -> dict[str, Any]: - """Return the configuration embedded in the JSON result.""" - return { - "tests": self.tests, - "providers": self.providers, - "modes": self.modes, - "samples": self.samples, - "hook_samples": self.hook_samples, - "startup_samples": self.startup_samples, - "warmup": self.warmup, - "payload_sizes": self.payload_sizes, - "concurrency": self.concurrency, - "response_bytes": self.response_bytes, - "stream_chunks": self.stream_chunks, - "models": { - "openai": self.openai_model, - "anthropic": self.anthropic_model, - }, - "content": { - "request_fill": self.request_fill, - "response_fill": self.response_fill, - }, - } - - -@dataclass(frozen=True) -class CliOptions: - """File paths and resolved benchmark configuration.""" - - relay_bin: Path - output: Path - config: BenchmarkConfig - - -def _read_toml(path: Path) -> dict[str, Any]: - with path.open("rb") as config_file: - return tomllib.load(config_file) - - -def _merge_config(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: - unknown = set(override) - CONFIG_KEYS - if unknown: - raise ValueError(f"unknown config key(s): {', '.join(sorted(unknown))}") - merged = dict(base) - for key, value in override.items(): - if key in {"models", "content"}: - if not isinstance(value, dict): - raise ValueError(f"[{key}] must be a TOML table") - unknown_nested = set(value) - TABLE_KEYS[key] - if unknown_nested: - raise ValueError(f"unknown [{key}] key(s): {', '.join(sorted(unknown_nested))}") - merged[key] = dict(merged.get(key, {})) | value - else: - merged[key] = value - return merged - - -def _string_tuple(value: Any, name: str, available: tuple[str, ...]) -> tuple[str, ...]: - if not isinstance(value, list) or not value or not all(isinstance(item, str) for item in value): - raise ValueError(f"{name} must be a non-empty list of strings") - values = tuple(value) - invalid = sorted(set(values) - set(available)) - if invalid: - raise ValueError(f"unknown {name}: {', '.join(invalid)}; choose from {', '.join(available)}") - if len(set(values)) != len(values): - raise ValueError(f"{name} must not contain duplicates") - return values - - -def _positive_int(value: Any, name: str) -> int: - if not isinstance(value, int) or isinstance(value, bool) or value <= 0: - raise ValueError(f"{name} must be a positive integer") - return value - - -def _positive_int_tuple(value: Any, name: str) -> tuple[int, ...]: - if not isinstance(value, list) or not value: - raise ValueError(f"{name} must be a non-empty list of positive integers") - values = tuple(_positive_int(item, name) for item in value) - if len(set(values)) != len(values): - raise ValueError(f"{name} must not contain duplicates") - return values - - -def _nonempty_string(value: Any, name: str) -> str: - if not isinstance(value, str) or not value: - raise ValueError(f"{name} must be a non-empty string") - return value - - -def _config_from_mapping(value: dict[str, Any]) -> BenchmarkConfig: - unknown = set(value) - CONFIG_KEYS - if unknown: - raise ValueError(f"unknown config key(s): {', '.join(sorted(unknown))}") - models = value.get("models") - content = value.get("content") - if not isinstance(models, dict) or not isinstance(content, dict): - raise ValueError("config must contain [models] and [content] tables") - warmup = value.get("warmup") - if not isinstance(warmup, int) or isinstance(warmup, bool) or warmup < 0: - raise ValueError("warmup must be a non-negative integer") - request_fill = _nonempty_string(content.get("request_fill"), "content.request_fill") - response_fill = _nonempty_string(content.get("response_fill"), "content.response_fill") - if len(request_fill) != 1 or len(response_fill) != 1 or not request_fill.isascii() or not response_fill.isascii(): - raise ValueError("content fill values must each contain exactly one ASCII character") - config = BenchmarkConfig( - tests=_string_tuple(value.get("tests"), "tests", AVAILABLE_TESTS), - providers=_string_tuple(value.get("providers"), "providers", AVAILABLE_PROVIDERS), - modes=_string_tuple(value.get("modes"), "modes", AVAILABLE_MODES), - samples=_positive_int(value.get("samples"), "samples"), - hook_samples=_positive_int(value.get("hook_samples"), "hook_samples"), - startup_samples=_positive_int(value.get("startup_samples"), "startup_samples"), - warmup=warmup, - payload_sizes=_positive_int_tuple(value.get("payload_sizes"), "payload_sizes"), - concurrency=_positive_int_tuple(value.get("concurrency"), "concurrency"), - response_bytes=_positive_int(value.get("response_bytes"), "response_bytes"), - stream_chunks=_positive_int(value.get("stream_chunks"), "stream_chunks"), - openai_model=_nonempty_string(models.get("openai"), "models.openai"), - anthropic_model=_nonempty_string(models.get("anthropic"), "models.anthropic"), - request_fill=request_fill, - response_fill=response_fill, - ) - if "gateway" in config.tests and max(config.concurrency) > config.samples: - raise ValueError("samples must be greater than or equal to every gateway concurrency value") - return config - - -def load_config(path: Path) -> BenchmarkConfig: - """Load the defaults and overlay a possibly partial user config.""" - defaults = _read_toml(DEFAULT_CONFIG_PATH) - if path.resolve() != DEFAULT_CONFIG_PATH.resolve(): - defaults = _merge_config(defaults, _read_toml(path)) - return _config_from_mapping(defaults) - - -def _csv_strings(value: str) -> tuple[str, ...]: - values = tuple(item.strip() for item in value.split(",") if item.strip()) - if not values: - raise argparse.ArgumentTypeError("expected a comma-separated list") - return values - - -def _csv_ints(value: str) -> tuple[int, ...]: - try: - values = tuple(int(item.strip()) for item in value.split(",") if item.strip()) - except ValueError as error: - raise argparse.ArgumentTypeError("expected comma-separated positive integers") from error - if not values or any(item <= 0 for item in values): - raise argparse.ArgumentTypeError("expected comma-separated positive integers") - return values - - -def _arg_positive_int(value: str) -> int: - try: - return _positive_int(int(value), "value") - except ValueError as error: - raise argparse.ArgumentTypeError("expected a positive integer") from error - - -def _arg_nonnegative_int(value: str) -> int: - try: - number = int(value) - except ValueError as error: - raise argparse.ArgumentTypeError("expected a non-negative integer") from error - if number < 0: - raise argparse.ArgumentTypeError("expected a non-negative integer") - return number - - -def parse_args(argv: list[str] | None = None) -> CliOptions: - """Parse CLI options, applying them after values from the config file.""" - parser = argparse.ArgumentParser( - prog="python -m scripts.benchmark_coding_agent_latency", - description="Measure local coding-agent gateway latency.", - ) - parser.add_argument("--relay-bin", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG_PATH, help="TOML overrides for the benchmark") - parser.add_argument("--tests", type=_csv_strings, help=f"override test suites: {','.join(AVAILABLE_TESTS)}") - parser.add_argument("--providers", type=_csv_strings, help=f"override providers: {','.join(AVAILABLE_PROVIDERS)}") - parser.add_argument("--modes", type=_csv_strings, help=f"override response modes: {','.join(AVAILABLE_MODES)}") - parser.add_argument("--samples", type=_arg_positive_int) - parser.add_argument("--hook-samples", type=_arg_positive_int) - parser.add_argument("--startup-samples", type=_arg_positive_int) - parser.add_argument("--warmup", type=_arg_nonnegative_int) - parser.add_argument("--payload-sizes", type=_csv_ints, help="override comma-separated request payload sizes") - parser.add_argument("--concurrency", type=_csv_ints, help="override comma-separated in-flight request counts") - parser.add_argument("--response-bytes", type=_arg_positive_int) - parser.add_argument("--stream-chunks", type=_arg_positive_int) - args = parser.parse_args(argv) - - try: - config = load_config(args.config) - overrides = { - name: getattr(args, name) - for name in ( - "tests", - "providers", - "modes", - "samples", - "hook_samples", - "startup_samples", - "warmup", - "payload_sizes", - "concurrency", - "response_bytes", - "stream_chunks", - ) - if getattr(args, name) is not None - } - config = replace(config, **overrides) - # Reuse the same validation for values supplied by argparse. - config = _config_from_mapping( - { - **config.parameters(), - "tests": list(config.tests), - "providers": list(config.providers), - "modes": list(config.modes), - "payload_sizes": list(config.payload_sizes), - "concurrency": list(config.concurrency), - } - ) - except (OSError, tomllib.TOMLDecodeError, ValueError) as error: - parser.error(f"invalid benchmark config: {error}") - - relay_bin = args.relay_bin.resolve() - if not relay_bin.is_file(): - parser.error(f"Relay binary does not exist: {relay_bin}") - return CliOptions( - relay_bin=relay_bin, - output=args.output.resolve(), - config=config, - ) diff --git a/scripts/benchmark_coding_agent_latency/tests/test_config.py b/scripts/benchmark_coding_agent_latency/tests/test_config.py deleted file mode 100644 index ea7e4c2e8..000000000 --- a/scripts/benchmark_coding_agent_latency/tests/test_config.py +++ /dev/null @@ -1,93 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for the coding-agent latency benchmark fixture.""" - -import tempfile -import unittest -from pathlib import Path - -from ..config import DEFAULT_CONFIG_PATH, load_config, parse_args -from ..fixtures import write_agent_config, write_fake_codex, write_plugin_configs - - -class BenchmarkConfigTests(unittest.TestCase): - def test_default_config_defines_every_suite_and_matrix_axis(self) -> None: - config = load_config(DEFAULT_CONFIG_PATH) - - self.assertEqual(config.tests, ("gateway", "hooks", "startup")) - self.assertEqual(config.providers, ("openai", "anthropic")) - self.assertEqual(config.modes, ("buffered", "streaming")) - self.assertTrue(config.payload_sizes) - self.assertTrue(config.concurrency) - - def test_partial_config_and_cli_arguments_override_defaults(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - custom_config = root / "quick.toml" - custom_config.write_text('tests = ["startup"]\nsamples = 7\n', encoding="utf-8") - relay_bin = root / "nemo-relay" - relay_bin.touch() - - options = parse_args( - [ - "--relay-bin", - str(relay_bin), - "--output", - str(root / "results.json"), - "--config", - str(custom_config), - "--tests", - "gateway,hooks", - "--samples", - "3", - "--concurrency", - "1", - "--providers", - "openai", - ] - ) - - self.assertEqual(options.config.tests, ("gateway", "hooks")) - self.assertEqual(options.config.samples, 3) - self.assertEqual(options.config.concurrency, (1,)) - self.assertEqual(options.config.providers, ("openai",)) - self.assertEqual(options.config.modes, ("buffered", "streaming")) - - def test_rejects_unknown_config_keys(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - config_path = Path(temporary) / "invalid.toml" - config_path.write_text("sample = 1\n", encoding="utf-8") - - with self.assertRaisesRegex(ValueError, "unknown config key"): - load_config(config_path) - - def test_rejects_gateway_concurrency_greater_than_samples(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - config_path = Path(temporary) / "invalid.toml" - config_path.write_text( - 'tests = ["gateway"]\nsamples = 2\nconcurrency = [4]\n', - encoding="utf-8", - ) - - with self.assertRaisesRegex(ValueError, "samples must be greater"): - load_config(config_path) - - -class StaticFixtureTests(unittest.TestCase): - def test_materializes_templates_without_embedded_markers(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - configs = write_plugin_configs(root, "http://127.0.0.1:4318") - fake_codex = write_fake_codex(root) - agent_config = write_agent_config(root, "test", fake_codex) - - rendered = "\n".join(path.read_text(encoding="utf-8") for path in (*configs.values(), agent_config)) - - self.assertNotIn("__ATOF_OUTPUT_DIRECTORY__", rendered) - self.assertNotIn("__OTLP_ENDPOINT__", rendered) - self.assertNotIn("__CODEX_COMMAND__", rendered) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/latency_benchmark/README.md b/scripts/latency_benchmark/README.md new file mode 100644 index 000000000..d90da021b --- /dev/null +++ b/scripts/latency_benchmark/README.md @@ -0,0 +1,290 @@ + + +# Coding-Agent Latency Benchmark + +Use this opt-in benchmark to measure the local latency that NeMo Relay adds +around OpenAI Responses, Anthropic Messages, Codex hooks, Claude Code hooks, +and Relay process startup. The fixture runs deterministic providers on +loopback, so network and model-service latency do not hide Relay overhead. + +Run all commands from the repository root. Start with the smoke test unless you +are collecting reportable performance results. + +The default matrix is intentionally large. With the current defaults, its +ATOF file-exporter path sends 11,860 measured and warmup gateway requests. The +request content totals about 12.3 GiB before ATOF serialization. A representative +4 MiB request produced about 8.1 MiB of ATOF JSON in a local fixture run, so a +full default run is expected to write about 25 GiB temporarily. The exact size +depends on the event shape. Reserve at least 30 GiB of free space before a +default run. + +By default, the benchmark writes this large ATOF output and other ephemeral +Relay files to the operating system's temporary directory in a folder named +`nemo-relay-latency-*`. This directory is typically under `/tmp`; macOS can use +a private per-user temporary directory instead. The benchmark removes the +temporary directory after a normal run. The much smaller JSON result and HTML +report are persistent and use the result directory described in +[Read the Results](#read-the-results). + +## Prerequisites + +Install the repository development prerequisites, including Rust, Python 3.11 +or newer, `uv`, and `just`. The `just` recipe builds the release-mode Relay CLI +before running the benchmark. + +## Fixture Layout + +The benchmark keeps executable source under `src/`, repeatable TOML fixtures +under `config/`, static coding-agent fixtures under `data/`, and focused unit +tests under `tests/`. Add runtime behavior to `src/` and keep fixed test input +outside executable modules. + +The `data/mock-codex.*` fixture is a small lifecycle stub used when Relay starts +a configured Codex command in transparent mode. It is not a simulated hook +client. The hooks suite invokes `nemo-relay hook-forward codex` and +`nemo-relay hook-forward claude` directly, so it does not launch either real +coding-agent executable and does not need a separate mock Claude command. + +## Run a Smoke Test + +Use a small matrix to verify the fixture and exporter paths: + +```bash +just latency-benchmark \ + --tests gateway \ + --providers openai \ + --modes buffered \ + --payload-sizes 4096 \ + --concurrency 1 \ + --samples 5 \ + --warmup 1 \ + --response-bytes 1024 +``` + +Do not use a smoke-test result for performance conclusions. Its sample count +is only large enough to catch functional failures. + +## Run the Default Matrix + +After you check available disk space, run the default matrix with the following +command: + +```bash +just latency-benchmark +``` + +The default configuration runs the following three suites: + +| Suite | What It Measures | +| --- | --- | +| `gateway` | Request latency through direct, minimal Relay, ATOF file-exporter, and OTLP Relay paths | +| `hooks` | Full Codex and Claude Code `nemo-relay hook-forward` subprocess wall time | +| `startup` | Cold Relay process launch through a healthy gateway | + +### Gateway Suite + +The gateway suite sends deterministic OpenAI Responses and Anthropic Messages +requests to a loopback provider. By default, each measurement cycle runs the +same request through four paths in a rotated order: + +- `direct` calls the mock provider without Relay. +- `relay-minimal` measures Relay without an exporter and isolates the managed + gateway pipeline. +- `relay-file` adds the local ATOF file exporter. +- `relay-otlp` adds the local OTLP HTTP exporter. + +The suite varies provider protocol, buffered or streaming response mode, +request-content size, and concurrency. `total` measures from request start +until the buffered body is read or the streaming response reaches end of +stream. Streaming scenarios also report `first_content`, which measures from +request start until the first content-delta event arrives. + +The primary paired comparisons subtract `direct` from each Relay path. The +`file_exporter_vs_minimal` and `otlp_exporter_vs_minimal` comparisons subtract +minimal Relay to isolate exporter overhead. Pairing measurements from the same +cycle reduces unrelated timing variation. + +### Hooks Suite + +The hooks suite measures the complete wall time of a new +`nemo-relay hook-forward` subprocess for Codex and Claude Code through minimal, +ATOF file-exporter, and OTLP gateways. Its paired comparisons subtract a +`nemo-relay --version` subprocess measured in the same cycle. This process +baseline estimates generic executable startup cost; it is not a no-op hook. + +### Startup Suite + +The startup suite measures a cold Relay process from launch until its +`/healthz` endpoint reports ready for minimal, ATOF file-exporter, and OTLP +configurations. Its paired comparisons subtract the same `nemo-relay --version` +process baseline to make Relay-specific readiness work easier to distinguish. + +## Configure a Run + +The benchmark resolves settings in this order: + +1. Defaults from `config/default.toml`. +2. Values from the file passed with `--config`. +3. CLI arguments, which take final precedence. + +A custom TOML file can contain only the settings that differ from the +defaults. For example: + +```toml +tests = ["gateway"] +providers = ["openai"] +modes = ["streaming"] +samples = 50 +warmup = 3 +payload_sizes = [4096, 65536] +concurrency = [1, 4] +``` + +Run the custom configuration with the following command: + +```bash +just latency-benchmark --config /path/to/benchmark.toml +``` + +Override any list from the command line with comma-separated values: + +```bash +just latency-benchmark \ + --config /path/to/benchmark.toml \ + --tests gateway,startup \ + --providers openai,anthropic \ + --modes buffered \ + --concurrency 1,4,8 +``` + +List every supported override without running the benchmark: + +```bash +just latency-benchmark --help +``` + +## Benchmark Custom Middleware + +Every run includes the `relay-minimal`, `relay-file`, and `relay-otlp` +variants. Add middleware as an opt-in variant by pointing the benchmark to a +valid Relay plugin configuration. + +The fixture includes `config/plugins-pii-redaction.toml` for the simplest +self-contained middleware test. It installs the built-in email detector and +does not require an external service. Run the following small gateway matrix: + +```bash +just latency-benchmark \ + --tests gateway \ + --providers openai \ + --modes buffered \ + --payload-sizes 4096 \ + --concurrency 1 \ + --samples 5 \ + --warmup 1 \ + --response-bytes 1024 \ + --middleware pii-redaction=scripts/latency_benchmark/config/plugins-pii-redaction.toml +``` + +This smoke test verifies that Relay loads and executes the middleware and that +the reports include the custom variant. The deterministic benchmark payload +does not contain an email address, so use this command to test latency plumbing, +not redaction correctness. Increase the sample count before drawing performance +conclusions. + +For repeatable custom runs, add one or more `[[middleware]]` tables to a +benchmark TOML file: + +```toml +[[middleware]] +name = "pii-redaction" +plugin_config = "./plugins-pii-redaction.toml" +``` + +The `plugin_config` path is relative to the benchmark TOML file. Run the +configuration with the following command: + +```bash +just latency-benchmark --config /path/to/benchmark.toml +``` + +For a one-off run, use `--middleware NAME=PATH`. Repeat the option to add more +than one variant: + +```bash +just latency-benchmark \ + --middleware pii-redaction=/path/to/plugins-pii-redaction.toml \ + --middleware guardrails=/path/to/plugins-guardrails.toml +``` + +CLI middleware options replace the `[[middleware]]` entries from the benchmark +TOML file. Names must contain lowercase letters, digits, or hyphens and cannot +be `direct`, `minimal`, `file`, or `otlp`. Each custom variant runs across the +selected gateway, hook, and startup suites. Gateway results compare it with +both direct provider calls and `relay-minimal`. + +Custom variants increase runtime in proportion to the number of variants. A +custom plugin configuration can also write additional data, so account for its +own storage behavior separately from the default ATOF estimate. + +## Read the Results + +The command prints a terminal summary and writes both of the following +persistent files: + +- `target/benchmark-results/nemo-relay-latency-report.json` contains the + complete, machine-readable result. +- `target/benchmark-results/nemo-relay-latency-report.html` is a self-contained + report with interactive gateway graphs, hook and startup graphs, numeric + tables, metric explanations, and the resolved run environment. + +Open the HTML file directly in a browser. It embeds its styles, scripts, and +result data, so it does not require an Internet connection or a web server. +Use the following command to choose another output directory: + +```bash +just output_dir=/tmp/relay-benchmarks latency-benchmark +``` + +Use `--report` to choose a different HTML path without changing the JSON path: + +```bash +just latency-benchmark --report /tmp/nemo-relay-latency-report.html +``` + +The reports use the following statistics: + +| Metric | Meaning | +| --- | --- | +| Absolute latency | Complete elapsed wall time for one measured path | +| Paired delta | Left path minus its baseline in the same cycle; positive is slower and small negative values can be measurement noise | +| p50 | Median observation | +| p95 and p99 | Tail percentiles that show slower observations | +| Min and max | Fastest and slowest observed values; these are sensitive to outliers | +| Median 95% CI | Deterministic bootstrap uncertainty interval around the median paired delta, not a range containing 95% of observations | + +The exporter-delivery byte and request counts are correctness checks. They +confirm that the fixture observed ATOF and OTLP delivery; they are not latency +metrics. + +When comparing variants, prefer added milliseconds over percentages. Record +the commit, release build, hardware, operating system, matrix, and sample count +with any shared result. Small loopback baselines can make harmless absolute +differences look large as percentages. + +## Troubleshoot + +- A loopback bind error means the environment must allow local HTTP listeners. +- An exporter-delivery error means the ATOF file or OTLP receiver observed no + benchmark events. Rerun a small gateway suite to isolate the exporter path; + Relay startup failures include their captured log output. +- A validation error names the invalid TOML or CLI value. Gateway samples must + be at least as large as every requested concurrency value. +- A normal run removes the large temporary ATOF output and its + `nemo-relay-latency-*` workspace, but a default run still needs at least 30 + GiB of free disk space while it is active. After an abrupt process + termination, remove any stale benchmark directory from the operating + system's temporary location. diff --git a/scripts/benchmark_coding_agent_latency/config/agent-config.toml b/scripts/latency_benchmark/config/agent-config.toml similarity index 100% rename from scripts/benchmark_coding_agent_latency/config/agent-config.toml rename to scripts/latency_benchmark/config/agent-config.toml diff --git a/scripts/benchmark_coding_agent_latency/config/default.toml b/scripts/latency_benchmark/config/default.toml similarity index 97% rename from scripts/benchmark_coding_agent_latency/config/default.toml rename to scripts/latency_benchmark/config/default.toml index f20a4639c..c1c4acdeb 100644 --- a/scripts/benchmark_coding_agent_latency/config/default.toml +++ b/scripts/latency_benchmark/config/default.toml @@ -13,6 +13,7 @@ payload_sizes = [4096, 65536, 262144, 1048576, 4194304] concurrency = [1, 2, 4, 8, 16] response_bytes = 16384 stream_chunks = 32 +middleware = [] [models] openai = "gpt-5-codex" diff --git a/scripts/benchmark_coding_agent_latency/config/plugins-file.toml b/scripts/latency_benchmark/config/plugins-file.toml similarity index 100% rename from scripts/benchmark_coding_agent_latency/config/plugins-file.toml rename to scripts/latency_benchmark/config/plugins-file.toml diff --git a/scripts/benchmark_coding_agent_latency/config/plugins-minimal.toml b/scripts/latency_benchmark/config/plugins-minimal.toml similarity index 100% rename from scripts/benchmark_coding_agent_latency/config/plugins-minimal.toml rename to scripts/latency_benchmark/config/plugins-minimal.toml diff --git a/scripts/benchmark_coding_agent_latency/config/plugins-otlp.toml b/scripts/latency_benchmark/config/plugins-otlp.toml similarity index 100% rename from scripts/benchmark_coding_agent_latency/config/plugins-otlp.toml rename to scripts/latency_benchmark/config/plugins-otlp.toml diff --git a/scripts/latency_benchmark/config/plugins-pii-redaction.toml b/scripts/latency_benchmark/config/plugins-pii-redaction.toml new file mode 100644 index 000000000..b965a69ac --- /dev/null +++ b/scripts/latency_benchmark/config/plugins-pii-redaction.toml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = 1 + +[[components]] +kind = "pii_redaction" +enabled = true + +[components.config] +version = 1 + +[[components.config.profiles]] +mode = "builtin" +priority = 80 + +[components.config.profiles.builtin] +action = "redact" +detector = "email" diff --git a/scripts/benchmark_coding_agent_latency/config/relay-config.toml b/scripts/latency_benchmark/config/relay-config.toml similarity index 100% rename from scripts/benchmark_coding_agent_latency/config/relay-config.toml rename to scripts/latency_benchmark/config/relay-config.toml diff --git a/scripts/benchmark_coding_agent_latency/data/fake-codex.cmd b/scripts/latency_benchmark/data/mock-codex.cmd similarity index 100% rename from scripts/benchmark_coding_agent_latency/data/fake-codex.cmd rename to scripts/latency_benchmark/data/mock-codex.cmd diff --git a/scripts/benchmark_coding_agent_latency/data/fake-codex.sh b/scripts/latency_benchmark/data/mock-codex.sh similarity index 100% rename from scripts/benchmark_coding_agent_latency/data/fake-codex.sh rename to scripts/latency_benchmark/data/mock-codex.sh diff --git a/scripts/benchmark_coding_agent_latency/__init__.py b/scripts/latency_benchmark/src/__init__.py similarity index 100% rename from scripts/benchmark_coding_agent_latency/__init__.py rename to scripts/latency_benchmark/src/__init__.py diff --git a/scripts/benchmark_coding_agent_latency/__main__.py b/scripts/latency_benchmark/src/__main__.py similarity index 100% rename from scripts/benchmark_coding_agent_latency/__main__.py rename to scripts/latency_benchmark/src/__main__.py diff --git a/scripts/benchmark_coding_agent_latency/benchmarks.py b/scripts/latency_benchmark/src/benchmarks.py similarity index 86% rename from scripts/benchmark_coding_agent_latency/benchmarks.py rename to scripts/latency_benchmark/src/benchmarks.py index 5e3e47d69..352bb14c4 100644 --- a/scripts/benchmark_coding_agent_latency/benchmarks.py +++ b/scripts/latency_benchmark/src/benchmarks.py @@ -23,9 +23,6 @@ from .protocol import make_request, request_headers, request_path from .servers import connection_for -VARIANTS = ("direct", "relay-minimal", "relay-file", "relay-otlp") -RELAY_VARIANTS = ("relay-minimal", "relay-file", "relay-otlp") - def percentile(values: Sequence[int | float], fraction: float) -> float: """Return a linearly interpolated percentile for a non-empty sample.""" @@ -112,6 +109,7 @@ def benchmark_scenario( warmup: int, concurrency: int, ) -> dict[str, Any]: + variants = tuple(urls) body = make_request( provider, streaming, @@ -127,12 +125,12 @@ def worker(worker_id: int, indices: list[int]) -> None: connections = {name: connection_for(url) for name, url in urls.items()} try: for _ in range(warmup): - for name in VARIANTS: + for name in variants: perform_request(connections[name], provider, body, streaming) barrier.wait() local = [] for index in indices: - order = list(VARIANTS) + order = list(variants) shift = (index + worker_id) % len(order) order = order[shift:] + order[:shift] local.append({name: perform_request(connections[name], provider, body, streaming) for name in order}) @@ -154,15 +152,19 @@ def worker(worker_id: int, indices: list[int]) -> None: metric.removesuffix("_ns"): summarize_ns([cycle[name][metric] for cycle in observations]) for metric in metrics } - for name in VARIANTS - } - comparison_pairs = { - "relay-minimal_vs_direct": ("relay-minimal", "direct"), - "relay-file_vs_direct": ("relay-file", "direct"), - "relay-otlp_vs_direct": ("relay-otlp", "direct"), - "file_exporter_vs_minimal": ("relay-file", "relay-minimal"), - "otlp_exporter_vs_minimal": ("relay-otlp", "relay-minimal"), + for name in variants } + relay_variants = tuple(name for name in variants if name != "direct") + comparison_pairs = {f"{name}_vs_direct": (name, "direct") for name in relay_variants} + for name in relay_variants: + if name == "relay-minimal": + continue + label = name.removeprefix("relay-") + comparison = { + "file": "file_exporter_vs_minimal", + "otlp": "otlp_exporter_vs_minimal", + }.get(label, f"{label}_vs_minimal") + comparison_pairs[comparison] = (name, "relay-minimal") comparisons = {} for comparison, (left, right) in comparison_pairs.items(): comparisons[comparison] = {} @@ -212,15 +214,11 @@ def benchmark_hooks( samples: int, warmup: int, ) -> dict[str, Any]: - measurements = { - "process_baseline": [], - "codex_minimal": [], - "codex_file": [], - "codex_otlp": [], - "claude_minimal": [], - "claude_file": [], - "claude_otlp": [], - } + relay_variants = tuple(configs) + measurements = {"process_baseline": []} + for agent in ("codex", "claude"): + for variant in relay_variants: + measurements[f"{agent}_{variant.removeprefix('relay-')}"] = [] with contextlib.ExitStack() as stack: relay_urls = { @@ -229,18 +227,19 @@ def benchmark_hooks( binary, root, provider_url, - configs[f"relay-{variant}"], + configs[variant], f"transparent-{variant}", ) ).url - for variant in ("minimal", "file", "otlp") + for variant in relay_variants } def hook(agent: str, variant: str, index: int) -> int: + variant_name = variant.removeprefix("relay-") event_name = "sessionStart" if agent == "codex" else "SessionStart" payload = json.dumps( { - "session_id": f"benchmark-{agent}-{variant}-{index}", + "session_id": f"benchmark-{agent}-{variant_name}-{index}", "hook_event_name": event_name, } ).encode() @@ -261,8 +260,9 @@ def hook(agent: str, variant: str, index: int) -> int: for index in range(-warmup, samples): cycle = {"process_baseline": lambda: run_subprocess_timed([str(binary), "--version"], root=root)} for agent in ("codex", "claude"): - for variant in ("minimal", "file", "otlp"): - cycle[f"{agent}_{variant}"] = lambda agent=agent, variant=variant: hook(agent, variant, index) + for variant in relay_variants: + name = f"{agent}_{variant.removeprefix('relay-')}" + cycle[name] = lambda agent=agent, variant=variant: hook(agent, variant, index) names = list(cycle) random.Random(index).shuffle(names) values = {name: cycle[name]() for name in names} @@ -276,8 +276,8 @@ def hook(agent: str, variant: str, index: int) -> int: "comparisons": {}, } for agent in ("codex", "claude"): - for variant in ("minimal", "file", "otlp"): - name = f"{agent}_{variant}" + for variant in relay_variants: + name = f"{agent}_{variant.removeprefix('relay-')}" deltas = [left - right for left, right in zip(measurements[name], baseline)] summary = summarize_ns(deltas) summary["median_ci95_ms"] = median_confidence_interval_ns(deltas, seed=len(name) * 1_009) @@ -294,11 +294,12 @@ def benchmark_startup( samples: int, warmup: int, ) -> dict[str, Any]: - measurements = {"process_baseline": [], **{variant: [] for variant in RELAY_VARIANTS}} + relay_variants = tuple(configs) + measurements = {"process_baseline": [], **{variant: [] for variant in relay_variants}} for index in range(-warmup, samples): baseline = run_subprocess_timed([str(binary), "--version"], root=root) cycle = {} - for variant in RELAY_VARIANTS: + for variant in relay_variants: process = RelayProcess(binary, root, provider_url, configs[variant], f"startup-{variant}-{index}") process.start() cycle[variant] = process.startup_ns @@ -312,7 +313,7 @@ def benchmark_startup( "comparisons": {}, } baseline = measurements["process_baseline"] - for variant in RELAY_VARIANTS: + for variant in relay_variants: deltas = [left - right for left, right in zip(measurements[variant], baseline)] summary = summarize_ns(deltas) summary["median_ci95_ms"] = median_confidence_interval_ns(deltas, seed=len(variant) * 2_003) diff --git a/scripts/benchmark_coding_agent_latency/cli.py b/scripts/latency_benchmark/src/cli.py similarity index 95% rename from scripts/benchmark_coding_agent_latency/cli.py rename to scripts/latency_benchmark/src/cli.py index 5e4ba04bb..08456c864 100644 --- a/scripts/benchmark_coding_agent_latency/cli.py +++ b/scripts/latency_benchmark/src/cli.py @@ -14,6 +14,7 @@ from .benchmarks import benchmark_hooks, benchmark_scenario, benchmark_startup from .config import BenchmarkConfig, parse_args from .fixtures import write_plugin_configs, write_relay_config +from .html_report import write_html_report from .processes import RelayProcess from .reporting import environment_record, print_results from .servers import OtlpHandler, ProviderHandler, local_server @@ -29,7 +30,7 @@ def _benchmark_gateway( with contextlib.ExitStack() as stack: relays = { variant: stack.enter_context(RelayProcess(binary, root, provider_url, configs[variant], variant)) - for variant in ("relay-minimal", "relay-file", "relay-otlp") + for variant in configs } urls = {"direct": provider_url} | {variant: relay.url for variant, relay in relays.items()} scenarios = [] @@ -81,7 +82,7 @@ def run_benchmarks(binary: Path, config: BenchmarkConfig) -> dict[str, Any]: ) as provider_url, local_server(OtlpHandler) as otlp_url, ): - configs = write_plugin_configs(root, otlp_url) + configs = write_plugin_configs(root, otlp_url, config.middleware) if "gateway" in config.tests: results["gateway"] = _benchmark_gateway(binary, root, provider_url, configs, config) if "hooks" in config.tests: @@ -121,5 +122,7 @@ def main(argv: list[str] | None = None) -> None: results = run_benchmarks(options.relay_bin, options.config) options.output.parent.mkdir(parents=True, exist_ok=True) options.output.write_text(json.dumps(results, indent=2, sort_keys=True) + "\n", encoding="utf-8") + write_html_report(results, options.report) print_results(results) print(f"\nJSON results: {options.output}") + print(f"HTML report: {options.report}") diff --git a/scripts/latency_benchmark/src/config.py b/scripts/latency_benchmark/src/config.py new file mode 100644 index 000000000..f6cf1e502 --- /dev/null +++ b/scripts/latency_benchmark/src/config.py @@ -0,0 +1,483 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Configuration loading and command-line overrides for the benchmark.""" + +from __future__ import annotations + +import argparse +import re +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +BENCHMARK_ROOT = Path(__file__).resolve().parent.parent +DATA_ROOT = BENCHMARK_ROOT / "data" +CONFIG_ROOT = BENCHMARK_ROOT / "config" +DEFAULT_CONFIG_PATH = CONFIG_ROOT / "default.toml" + +AVAILABLE_TESTS = ("gateway", "hooks", "startup") +AVAILABLE_PROVIDERS = ("openai", "anthropic") +AVAILABLE_MODES = ("buffered", "streaming") +RESERVED_MIDDLEWARE_NAMES = {"direct", "minimal", "file", "otlp"} +MIDDLEWARE_NAME_PATTERN = re.compile(r"[a-z0-9][a-z0-9-]*") +CONFIG_KEYS = { + "tests", + "providers", + "modes", + "samples", + "hook_samples", + "startup_samples", + "warmup", + "payload_sizes", + "concurrency", + "response_bytes", + "stream_chunks", + "models", + "content", + "middleware", +} +TABLE_KEYS = { + "models": {"openai", "anthropic"}, + "content": {"request_fill", "response_fill"}, +} +MIDDLEWARE_KEYS = {"name", "plugin_config"} + + +@dataclass(frozen=True) +class MiddlewareVariant: + """One opt-in Relay plugin configuration benchmarked as an extra variant.""" + + name: str + plugin_config: Path + + @property + def relay_name(self) -> str: + """Return the variant key stored in benchmark results.""" + return f"relay-{self.name}" + + def parameters(self) -> dict[str, str]: + """Return the serializable configuration recorded with results.""" + return {"name": self.name, "plugin_config": str(self.plugin_config)} + + +@dataclass(frozen=True) +class BenchmarkConfig: + """Validated benchmark matrix and sample settings.""" + + tests: tuple[str, ...] + providers: tuple[str, ...] + modes: tuple[str, ...] + samples: int + hook_samples: int + startup_samples: int + warmup: int + payload_sizes: tuple[int, ...] + concurrency: tuple[int, ...] + response_bytes: int + stream_chunks: int + openai_model: str + anthropic_model: str + request_fill: str + response_fill: str + middleware: tuple[MiddlewareVariant, ...] + + def parameters(self) -> dict[str, Any]: + """Return the configuration embedded in the JSON result.""" + return { + "tests": self.tests, + "providers": self.providers, + "modes": self.modes, + "samples": self.samples, + "hook_samples": self.hook_samples, + "startup_samples": self.startup_samples, + "warmup": self.warmup, + "payload_sizes": self.payload_sizes, + "concurrency": self.concurrency, + "response_bytes": self.response_bytes, + "stream_chunks": self.stream_chunks, + "models": { + "openai": self.openai_model, + "anthropic": self.anthropic_model, + }, + "content": { + "request_fill": self.request_fill, + "response_fill": self.response_fill, + }, + "middleware": [variant.parameters() for variant in self.middleware], + } + + +@dataclass(frozen=True) +class CliOptions: + """File paths and resolved benchmark configuration.""" + + relay_bin: Path + output: Path + report: Path + config: BenchmarkConfig + + +def _read_toml(path: Path) -> dict[str, Any]: + with path.open("rb") as config_file: + return tomllib.load(config_file) + + +def _merge_config(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + unknown = set(override) - CONFIG_KEYS + if unknown: + raise ValueError(f"unknown config key(s): {', '.join(sorted(unknown))}") + merged = dict(base) + for key, value in override.items(): + if key in {"models", "content"}: + if not isinstance(value, dict): + raise ValueError(f"[{key}] must be a TOML table") + unknown_nested = set(value) - TABLE_KEYS[key] + if unknown_nested: + raise ValueError(f"unknown [{key}] key(s): {', '.join(sorted(unknown_nested))}") + merged[key] = dict(merged.get(key, {})) | value + else: + merged[key] = value + return merged + + +def _string_tuple(value: Any, name: str, available: tuple[str, ...]) -> tuple[str, ...]: + if not isinstance(value, list) or not value or not all(isinstance(item, str) for item in value): + raise ValueError(f"{name} must be a non-empty list of strings") + values = tuple(value) + invalid = sorted(set(values) - set(available)) + if invalid: + raise ValueError(f"unknown {name}: {', '.join(invalid)}; choose from {', '.join(available)}") + if len(set(values)) != len(values): + raise ValueError(f"{name} must not contain duplicates") + return values + + +def _positive_int(value: Any, name: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + return value + + +def _positive_int_tuple(value: Any, name: str) -> tuple[int, ...]: + if not isinstance(value, list) or not value: + raise ValueError(f"{name} must be a non-empty list of positive integers") + values = tuple(_positive_int(item, name) for item in value) + if len(set(values)) != len(values): + raise ValueError(f"{name} must not contain duplicates") + return values + + +def _nonempty_string(value: Any, name: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{name} must be a non-empty string") + return value + + +def _middleware_variants(value: Any, base_directory: Path) -> tuple[MiddlewareVariant, ...]: + if not isinstance(value, list): + raise ValueError("middleware must be a list of tables") + variants = [] + names = set() + for index, item in enumerate(value): + if not isinstance(item, dict): + raise ValueError(f"middleware[{index}] must be a TOML table") + item_mapping = cast(dict[str, Any], item) + unknown = set(item_mapping) - MIDDLEWARE_KEYS + if unknown: + raise ValueError(f"unknown middleware[{index}] key(s): {', '.join(sorted(unknown))}") + name = _nonempty_string(item_mapping.get("name"), f"middleware[{index}].name") + if MIDDLEWARE_NAME_PATTERN.fullmatch(name) is None: + raise ValueError(f"middleware[{index}].name must contain only lowercase letters, digits, and hyphens") + if name in RESERVED_MIDDLEWARE_NAMES: + raise ValueError(f"middleware name is reserved by a default variant: {name}") + if name in names: + raise ValueError(f"middleware names must not contain duplicates: {name}") + names.add(name) + raw_path = _nonempty_string(item_mapping.get("plugin_config"), f"middleware[{index}].plugin_config") + plugin_config = Path(raw_path) + if not plugin_config.is_absolute(): + plugin_config = base_directory / plugin_config + plugin_config = plugin_config.resolve() + if not plugin_config.is_file(): + raise ValueError(f"middleware plugin config does not exist: {plugin_config}") + variants.append(MiddlewareVariant(name=name, plugin_config=plugin_config)) + return tuple(variants) + + +def _config_from_mapping(value: dict[str, Any], *, middleware_base: Path) -> BenchmarkConfig: + unknown = set(value) - CONFIG_KEYS + if unknown: + raise ValueError(f"unknown config key(s): {', '.join(sorted(unknown))}") + models = value.get("models") + content = value.get("content") + if not isinstance(models, dict) or not isinstance(content, dict): + raise ValueError("config must contain [models] and [content] tables") + warmup = value.get("warmup") + if not isinstance(warmup, int) or isinstance(warmup, bool) or warmup < 0: + raise ValueError("warmup must be a non-negative integer") + request_fill = _nonempty_string(content.get("request_fill"), "content.request_fill") + response_fill = _nonempty_string(content.get("response_fill"), "content.response_fill") + if len(request_fill) != 1 or len(response_fill) != 1 or not request_fill.isascii() or not response_fill.isascii(): + raise ValueError("content fill values must each contain exactly one ASCII character") + config = BenchmarkConfig( + tests=_string_tuple(value.get("tests"), "tests", AVAILABLE_TESTS), + providers=_string_tuple(value.get("providers"), "providers", AVAILABLE_PROVIDERS), + modes=_string_tuple(value.get("modes"), "modes", AVAILABLE_MODES), + samples=_positive_int(value.get("samples"), "samples"), + hook_samples=_positive_int(value.get("hook_samples"), "hook_samples"), + startup_samples=_positive_int(value.get("startup_samples"), "startup_samples"), + warmup=warmup, + payload_sizes=_positive_int_tuple(value.get("payload_sizes"), "payload_sizes"), + concurrency=_positive_int_tuple(value.get("concurrency"), "concurrency"), + response_bytes=_positive_int(value.get("response_bytes"), "response_bytes"), + stream_chunks=_positive_int(value.get("stream_chunks"), "stream_chunks"), + openai_model=_nonempty_string(models.get("openai"), "models.openai"), + anthropic_model=_nonempty_string(models.get("anthropic"), "models.anthropic"), + request_fill=request_fill, + response_fill=response_fill, + middleware=_middleware_variants(value.get("middleware"), middleware_base), + ) + if "gateway" in config.tests and max(config.concurrency) > config.samples: + raise ValueError("samples must be greater than or equal to every gateway concurrency value") + return config + + +def load_config(path: Path) -> BenchmarkConfig: + """Load the defaults and overlay a possibly partial user config.""" + defaults = _read_toml(DEFAULT_CONFIG_PATH) + resolved_path = path.resolve() + middleware_base = DEFAULT_CONFIG_PATH.parent + if resolved_path != DEFAULT_CONFIG_PATH.resolve(): + override = _read_toml(resolved_path) + defaults = _merge_config(defaults, override) + if "middleware" in override: + middleware_base = resolved_path.parent + return _config_from_mapping(defaults, middleware_base=middleware_base) + + +def _csv_strings(value: str) -> tuple[str, ...]: + values = tuple(item.strip() for item in value.split(",") if item.strip()) + if not values: + raise argparse.ArgumentTypeError("expected a comma-separated list") + return values + + +def _csv_ints(value: str) -> tuple[int, ...]: + try: + values = tuple(int(item.strip()) for item in value.split(",") if item.strip()) + except ValueError as error: + raise argparse.ArgumentTypeError("expected comma-separated positive integers") from error + if not values or any(item <= 0 for item in values): + raise argparse.ArgumentTypeError("expected comma-separated positive integers") + return values + + +def _arg_positive_int(value: str) -> int: + try: + return _positive_int(int(value), "value") + except ValueError as error: + raise argparse.ArgumentTypeError("expected a positive integer") from error + + +def _arg_nonnegative_int(value: str) -> int: + try: + number = int(value) + except ValueError as error: + raise argparse.ArgumentTypeError("expected a non-negative integer") from error + if number < 0: + raise argparse.ArgumentTypeError("expected a non-negative integer") + return number + + +def _middleware_arg(value: str) -> dict[str, str]: + name, separator, plugin_config = value.partition("=") + if not separator or not name or not plugin_config: + raise argparse.ArgumentTypeError("expected NAME=PATH") + return {"name": name, "plugin_config": plugin_config} + + +def build_parser() -> argparse.ArgumentParser: + """Build the public command-line interface for the benchmark.""" + parser = argparse.ArgumentParser( + prog="just latency-benchmark", + usage="just latency-benchmark [options]", + description=( + "Measure the local latency that NeMo Relay adds to coding-agent gateway, hook, and startup paths." + ), + epilog=( + "Configuration precedence: built-in defaults, then --config, then CLI overrides.\n" + "Example smoke test: just latency-benchmark --tests gateway " + "--providers openai --modes buffered --payload-sizes 4096 " + "--concurrency 1 --samples 5 --warmup 1" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + outputs = parser.add_argument_group("output and configuration") + outputs.add_argument( + "--relay-bin", + type=Path, + required=True, + metavar="PATH", + help="Relay CLI executable to benchmark (the just recipe supplies the release binary).", + ) + outputs.add_argument( + "--output", + type=Path, + required=True, + metavar="PATH", + help="write machine-readable JSON results to PATH (the just recipe supplies a default).", + ) + outputs.add_argument( + "--report", + type=Path, + metavar="PATH", + help="write the self-contained HTML report to PATH (default: the JSON path with an .html suffix).", + ) + outputs.add_argument( + "--config", + type=Path, + default=DEFAULT_CONFIG_PATH, + metavar="PATH", + help="overlay benchmark settings from a TOML file (default: bundled config/default.toml).", + ) + + matrix = parser.add_argument_group("suite and matrix selection") + matrix.add_argument( + "--tests", + type=_csv_strings, + metavar="LIST", + help=f"run comma-separated suites; choices: {','.join(AVAILABLE_TESTS)}.", + ) + matrix.add_argument( + "--providers", + type=_csv_strings, + metavar="LIST", + help=f"benchmark comma-separated mock provider protocols; choices: {','.join(AVAILABLE_PROVIDERS)}.", + ) + matrix.add_argument( + "--modes", + type=_csv_strings, + metavar="LIST", + help=f"benchmark comma-separated response modes; choices: {','.join(AVAILABLE_MODES)}.", + ) + matrix.add_argument( + "--payload-sizes", + type=_csv_ints, + metavar="BYTES", + help="benchmark comma-separated request-content sizes in bytes.", + ) + matrix.add_argument( + "--concurrency", + type=_csv_ints, + metavar="COUNTS", + help="benchmark comma-separated in-flight request counts; each value must not exceed --samples.", + ) + matrix.add_argument( + "--middleware", + action="append", + type=_middleware_arg, + metavar="NAME=PATH", + help=( + "add a named Relay middleware plugin config as an extra variant; repeat for multiple variants " + "and use this option to replace middleware entries from --config." + ), + ) + + sampling = parser.add_argument_group("sampling") + sampling.add_argument( + "--samples", + type=_arg_positive_int, + metavar="COUNT", + help="record COUNT gateway measurement cycles per scenario.", + ) + sampling.add_argument( + "--hook-samples", + type=_arg_positive_int, + metavar="COUNT", + help="record COUNT subprocess measurements for every hook path.", + ) + sampling.add_argument( + "--startup-samples", + type=_arg_positive_int, + metavar="COUNT", + help="record COUNT cold-start measurements for every Relay variant.", + ) + sampling.add_argument( + "--warmup", + type=_arg_nonnegative_int, + metavar="COUNT", + help="run COUNT unrecorded warmup cycles before each measured workload.", + ) + + response = parser.add_argument_group("mock provider response") + response.add_argument( + "--response-bytes", + type=_arg_positive_int, + metavar="BYTES", + help="return approximately BYTES of deterministic content from the mock provider.", + ) + response.add_argument( + "--stream-chunks", + type=_arg_positive_int, + metavar="COUNT", + help="split streaming mock responses into COUNT content-delta events.", + ) + return parser + + +def parse_args(argv: list[str] | None = None) -> CliOptions: + """Parse CLI options, applying them after values from the config file.""" + parser = build_parser() + args = parser.parse_args(argv) + + try: + config = load_config(args.config) + values = config.parameters() + values.update( + { + name: getattr(args, name) + for name in ( + "tests", + "providers", + "modes", + "samples", + "hook_samples", + "startup_samples", + "warmup", + "payload_sizes", + "concurrency", + "response_bytes", + "stream_chunks", + "middleware", + ) + if getattr(args, name) is not None + } + ) + config = _config_from_mapping( + { + **values, + "tests": list(values["tests"]), + "providers": list(values["providers"]), + "modes": list(values["modes"]), + "payload_sizes": list(values["payload_sizes"]), + "concurrency": list(values["concurrency"]), + }, + middleware_base=Path.cwd(), + ) + except (OSError, tomllib.TOMLDecodeError, ValueError) as error: + parser.error(f"invalid benchmark config: {error}") + + relay_bin = args.relay_bin.resolve() + if not relay_bin.is_file(): + parser.error(f"Relay binary does not exist: {relay_bin}") + output = args.output.resolve() + report = args.report.resolve() if args.report is not None else output.with_suffix(".html") + return CliOptions( + relay_bin=relay_bin, + output=output, + report=report, + config=config, + ) diff --git a/scripts/benchmark_coding_agent_latency/fixtures.py b/scripts/latency_benchmark/src/fixtures.py similarity index 79% rename from scripts/benchmark_coding_agent_latency/fixtures.py rename to scripts/latency_benchmark/src/fixtures.py index 627fff75d..fd1aedaea 100644 --- a/scripts/benchmark_coding_agent_latency/fixtures.py +++ b/scripts/latency_benchmark/src/fixtures.py @@ -9,7 +9,7 @@ import os from pathlib import Path -from .config import CONFIG_ROOT, DATA_ROOT +from .config import CONFIG_ROOT, DATA_ROOT, MiddlewareVariant def _read_data(name: str) -> str: @@ -40,8 +40,12 @@ def write_relay_config(root: Path) -> Path: return path -def write_plugin_configs(root: Path, otlp_url: str) -> dict[str, Path]: - """Write the three Relay plugin configurations used for paired runs.""" +def write_plugin_configs( + root: Path, + otlp_url: str, + middleware: tuple[MiddlewareVariant, ...] = (), +) -> dict[str, Path]: + """Write default plugin configs and add opt-in middleware variants.""" paths = { "relay-minimal": root / "plugins-minimal.toml", "relay-file": root / "plugins-file.toml", @@ -59,13 +63,14 @@ def write_plugin_configs(root: Path, otlp_url: str) -> dict[str, Path]: _render_config("plugins-otlp.toml", {'"__OTLP_ENDPOINT__"': toml_string(f"{otlp_url}/v1/traces")}), encoding="utf-8", ) + paths.update((variant.relay_name, variant.plugin_config) for variant in middleware) return paths -def write_fake_codex(root: Path) -> Path: - """Copy the platform-specific static fake Codex client into the workspace.""" - source_name = "fake-codex.cmd" if os.name == "nt" else "fake-codex.sh" - target_name = "benchmark-codex.cmd" if os.name == "nt" else "benchmark-codex" +def write_mock_codex(root: Path) -> Path: + """Copy the platform-specific static mock Codex client into the workspace.""" + source_name = "mock-codex.cmd" if os.name == "nt" else "mock-codex.sh" + target_name = "mock-codex.cmd" if os.name == "nt" else "mock-codex" path = root / target_name path.write_text(_read_data(source_name), encoding="utf-8") if os.name != "nt": @@ -73,10 +78,10 @@ def write_fake_codex(root: Path) -> Path: return path -def write_agent_config(root: Path, name: str, fake_codex: Path) -> Path: +def write_agent_config(root: Path, name: str, mock_codex: Path) -> Path: path = root / f"{name}-config.toml" path.write_text( - _render_config("agent-config.toml", {'"__CODEX_COMMAND__"': toml_string(fake_codex)}), + _render_config("agent-config.toml", {'"__CODEX_COMMAND__"': toml_string(mock_codex)}), encoding="utf-8", ) return path diff --git a/scripts/latency_benchmark/src/html_report.py b/scripts/latency_benchmark/src/html_report.py new file mode 100644 index 000000000..4da1c82d0 --- /dev/null +++ b/scripts/latency_benchmark/src/html_report.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Render a self-contained HTML report for benchmark results.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +REPORT_ROOT = Path(__file__).resolve().parent / "report" +TEMPLATE_PATH = REPORT_ROOT / "template.html" +STYLES_PATH = REPORT_ROOT / "styles.css" +SCRIPT_PATH = REPORT_ROOT / "report.js" + + +def _embedded_json(results: dict[str, Any]) -> str: + """Serialize JSON so it cannot terminate its script element.""" + return ( + json.dumps(results, separators=(",", ":"), sort_keys=True) + .replace("&", "\\u0026") + .replace("<", "\\u003c") + .replace(">", "\\u003e") + ) + + +def render_html_report(results: dict[str, Any]) -> str: + """Return a portable HTML report with embedded styles, data, and scripts.""" + template = TEMPLATE_PATH.read_text(encoding="utf-8") + replacements = { + "__BENCHMARK_STYLES__": STYLES_PATH.read_text(encoding="utf-8"), + "__BENCHMARK_DATA__": _embedded_json(results), + "__BENCHMARK_SCRIPT__": SCRIPT_PATH.read_text(encoding="utf-8"), + } + for marker, value in replacements.items(): + template = template.replace(marker, value) + return template + + +def write_html_report(results: dict[str, Any], path: Path) -> None: + """Write a portable HTML report to ``path``.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(render_html_report(results), encoding="utf-8") diff --git a/scripts/benchmark_coding_agent_latency/processes.py b/scripts/latency_benchmark/src/processes.py similarity index 98% rename from scripts/benchmark_coding_agent_latency/processes.py rename to scripts/latency_benchmark/src/processes.py index c49356b84..6bbbb3672 100644 --- a/scripts/benchmark_coding_agent_latency/processes.py +++ b/scripts/latency_benchmark/src/processes.py @@ -12,7 +12,7 @@ from pathlib import Path from typing import IO -from .fixtures import isolated_environment, write_agent_config, write_fake_codex +from .fixtures import isolated_environment, write_agent_config, write_mock_codex from .servers import connection_for @@ -138,7 +138,7 @@ def __init__( def start(self) -> None: gateway_file = self.root / f"{self.name}-{uuid.uuid4().hex}.gateway" - config = write_agent_config(self.root, self.name, write_fake_codex(self.root)) + config = write_agent_config(self.root, self.name, write_mock_codex(self.root)) log_path = self.root / f"{self.name}.log" self.log_handle = log_path.open("ab") environment = isolated_environment(self.root) @@ -162,6 +162,7 @@ def start(self) -> None: command, cwd=self.root, env=environment, + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=self.log_handle, ) diff --git a/scripts/benchmark_coding_agent_latency/protocol.py b/scripts/latency_benchmark/src/protocol.py similarity index 100% rename from scripts/benchmark_coding_agent_latency/protocol.py rename to scripts/latency_benchmark/src/protocol.py diff --git a/scripts/latency_benchmark/src/report/report.js b/scripts/latency_benchmark/src/report/report.js new file mode 100644 index 000000000..82bc4ec9e --- /dev/null +++ b/scripts/latency_benchmark/src/report/report.js @@ -0,0 +1,470 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +"use strict"; + +const benchmark = JSON.parse(document.getElementById("benchmark-data").textContent); +const svgNamespace = "http://www.w3.org/2000/svg"; +const colors = ["#5f8f00", "#006e9c", "#b45f06", "#7042a6", "#b3261e", "#007c70", "#725f00", "#4554a3"]; + +function byId(id) { + return document.getElementById(id); +} + +function addElement(parent, tag, attributes = {}, text = null) { + const element = document.createElement(tag); + for (const [name, value] of Object.entries(attributes)) { + element.setAttribute(name, String(value)); + } + if (text !== null) { + element.textContent = text; + } + parent.appendChild(element); + return element; +} + +function addSvg(parent, tag, attributes = {}, text = null) { + const element = document.createElementNS(svgNamespace, tag); + for (const [name, value] of Object.entries(attributes)) { + element.setAttribute(name, String(value)); + } + if (text !== null) { + element.textContent = text; + } + parent.appendChild(element); + return element; +} + +function labelName(value) { + const names = { + "direct": "Direct", + "relay-minimal": "Relay minimal", + "relay-file": "Relay ATOF file", + "relay-otlp": "Relay OTLP", + "relay-minimal_vs_direct": "Minimal − direct", + "relay-file_vs_direct": "ATOF file − direct", + "relay-otlp_vs_direct": "OTLP − direct", + "file_exporter_vs_minimal": "ATOF file − minimal", + "otlp_exporter_vs_minimal": "OTLP − minimal", + "process_baseline": "Process baseline", + }; + if (names[value]) { + return names[value]; + } + if (value.endsWith("_vs_direct")) { + return `${labelName(value.slice(0, -"_vs_direct".length))} − direct`; + } + if (value.endsWith("_vs_minimal")) { + return `${labelName(value.slice(0, -"_vs_minimal".length))} − minimal`; + } + if (value.endsWith("_vs_process_baseline")) { + return `${labelName(value.slice(0, -"_vs_process_baseline".length))} − process baseline`; + } + if (value.startsWith("relay-")) { + return `Relay ${value.slice("relay-".length).replaceAll("-", " ")}`; + } + const label = value.replaceAll("_", " ").replaceAll("-", " "); + return label.charAt(0).toUpperCase() + label.slice(1); +} + +function formatMs(value) { + if (!Number.isFinite(value)) { + return "—"; + } + const magnitude = Math.abs(value); + const digits = magnitude >= 100 ? 1 : magnitude >= 10 ? 2 : 3; + return `${value.toFixed(digits)} ms`; +} + +function formatBytes(value) { + if (!Number.isFinite(value)) { + return "—"; + } + const units = ["B", "KiB", "MiB", "GiB"]; + let amount = value; + let unit = 0; + while (amount >= 1024 && unit < units.length - 1) { + amount /= 1024; + unit += 1; + } + const digits = amount >= 10 || unit === 0 ? 0 : 1; + return `${amount.toFixed(digits)} ${units[unit]}`; +} + +function formatValue(value) { + if (Array.isArray(value)) { + return value.map((item) => formatValue(item)).join("; "); + } + if (value !== null && typeof value === "object") { + return Object.entries(value) + .map(([key, item]) => `${key}: ${formatValue(item)}`) + .join("; "); + } + return String(value); +} + +function summaryCard(parent, label, value) { + const card = addElement(parent, "div", { class: "summary-card" }); + addElement(card, "span", { class: "label" }, label); + addElement(card, "strong", { class: "value" }, value); +} + +function renderKeyValueTable(parent, values) { + const rows = Object.entries(values).map(([key, value]) => [labelName(key), formatValue(value)]); + renderTable(parent, ["Field", "Value"], rows); +} + +function renderTable(parent, headers, rows, numericColumns = []) { + parent.replaceChildren(); + const table = addElement(parent, "table"); + const head = addElement(table, "thead"); + const headRow = addElement(head, "tr"); + headers.forEach((header) => addElement(headRow, "th", { scope: "col" }, header)); + const body = addElement(table, "tbody"); + for (const row of rows) { + const tableRow = addElement(body, "tr"); + row.forEach((value, index) => { + addElement(tableRow, "td", numericColumns.includes(index) ? { class: "numeric" } : {}, value); + }); + } +} + +function renderOverview() { + const environment = benchmark.environment || {}; + const parameters = benchmark.parameters || {}; + const tests = parameters.tests || []; + + const cards = byId("summary-cards"); + summaryCard(cards, "Suites", tests.join(", ") || "None"); + summaryCard(cards, "Gateway scenarios", String((benchmark.gateway || []).length)); + summaryCard(cards, "Relay", environment.relay_version || "unknown"); + summaryCard(cards, "Working tree", environment.git_dirty ? "Dirty" : "Clean"); + summaryCard(cards, "Platform", environment.platform || "unknown"); + + renderKeyValueTable(byId("environment-table"), environment); + renderKeyValueTable(byId("parameters-table"), parameters); +} + +function populateSelect(select, entries) { + select.replaceChildren(); + for (const [value, label] of entries) { + addElement(select, "option", { value }, label); + } +} + +function uniqueSorted(values, numeric = false) { + const unique = [...new Set(values)]; + return unique.sort(numeric ? (left, right) => left - right : undefined); +} + +function graphRange(values, includeZero) { + let minimum = Math.min(...values); + let maximum = Math.max(...values); + if (includeZero) { + minimum = Math.min(minimum, 0); + maximum = Math.max(maximum, 0); + } + if (minimum === maximum) { + const padding = Math.max(Math.abs(minimum) * 0.15, 0.1); + minimum -= padding; + maximum += padding; + } else { + const padding = (maximum - minimum) * 0.12; + minimum -= padding; + maximum += padding; + } + return [minimum, maximum]; +} + +function drawLineChart(svg, series, payloads, includeZero) { + svg.replaceChildren(); + const width = 920; + const height = 390; + const margin = { top: 28, right: 24, bottom: 66, left: 82 }; + const innerWidth = width - margin.left - margin.right; + const innerHeight = height - margin.top - margin.bottom; + svg.setAttribute("viewBox", `0 0 ${width} ${height}`); + + const values = series.flatMap((item) => item.values.map((point) => point.value)); + const [minimum, maximum] = graphRange(values, includeZero); + const x = (index) => margin.left + (payloads.length === 1 ? innerWidth / 2 : (index / (payloads.length - 1)) * innerWidth); + const y = (value) => margin.top + ((maximum - value) / (maximum - minimum)) * innerHeight; + + for (let index = 0; index <= 5; index += 1) { + const value = minimum + ((maximum - minimum) * index) / 5; + const position = y(value); + addSvg(svg, "line", { + class: Math.abs(value) < (maximum - minimum) / 1000 ? "zero-line" : "grid-line", + x1: margin.left, + y1: position, + x2: width - margin.right, + y2: position, + }); + addSvg(svg, "text", { class: "chart-label", x: margin.left - 12, y: position + 4, "text-anchor": "end" }, formatMs(value)); + } + + addSvg(svg, "line", { + class: "axis", + x1: margin.left, + y1: margin.top + innerHeight, + x2: width - margin.right, + y2: margin.top + innerHeight, + }); + payloads.forEach((payload, index) => { + const position = x(index); + addSvg(svg, "line", { + class: "axis", + x1: position, + y1: margin.top + innerHeight, + x2: position, + y2: margin.top + innerHeight + 6, + }); + addSvg( + svg, + "text", + { class: "chart-label", x: position, y: height - 30, "text-anchor": "middle" }, + formatBytes(payload), + ); + }); + addSvg(svg, "text", { class: "chart-label", x: margin.left + innerWidth / 2, y: height - 6, "text-anchor": "middle" }, "Request content size"); + + series.forEach((item, seriesIndex) => { + const color = colors[seriesIndex % colors.length]; + const points = item.values.map((point, index) => `${x(index)},${y(point.value)}`).join(" "); + addSvg(svg, "polyline", { + class: "line-series", + points, + stroke: color, + "stroke-dasharray": seriesIndex === 1 ? "9 5" : seriesIndex === 2 ? "3 5" : "none", + }); + item.values.forEach((point, index) => { + const circle = addSvg(svg, "circle", { + class: "chart-point", + cx: x(index), + cy: y(point.value), + r: 5, + fill: color, + }); + addSvg(circle, "title", {}, `${item.label}, ${formatBytes(point.payload)}: ${formatMs(point.value)}`); + }); + }); +} + +function renderLegend(series) { + const legend = byId("gateway-legend"); + legend.replaceChildren(); + series.forEach((item, index) => { + const entry = addElement(legend, "span", { class: "legend-item" }); + addElement(entry, "span", { + class: "legend-swatch", + style: `border-color: ${colors[index % colors.length]}`, + "aria-hidden": "true", + }); + addElement(entry, "span", {}, item.label); + }); +} + +function gatewaySeries(scenarios, view, metric, statistic) { + const selected = scenarios[0]; + const definitions = view === "absolute" + ? Object.keys(selected.absolute) + : Object.keys(selected.comparisons).filter((name) => name.endsWith(view === "minimal" ? "_vs_minimal" : "_vs_direct")); + return definitions.map((name) => ({ + name, + label: labelName(name), + values: scenarios.map((scenario) => { + const collection = view === "absolute" ? scenario.absolute : scenario.comparisons; + return { + payload: scenario.payload_bytes, + value: collection[name][metric][statistic], + summary: collection[name][metric], + }; + }), + })); +} + +function selectedGatewayScenarios() { + const provider = byId("gateway-provider").value; + const mode = byId("gateway-mode").value; + const concurrency = Number(byId("gateway-concurrency").value); + return benchmark.gateway + .filter((scenario) => scenario.provider === provider && scenario.mode === mode && scenario.concurrency === concurrency) + .sort((left, right) => left.payload_bytes - right.payload_bytes); +} + +function syncMetricOptions() { + const metric = byId("gateway-metric"); + const previous = metric.value; + const options = byId("gateway-mode").value === "streaming" + ? [["first_content", "Time to first content"], ["total", "Total response"]] + : [["total", "Total response"]]; + populateSelect(metric, options); + if (options.some(([value]) => value === previous)) { + metric.value = previous; + } +} + +function renderGateway() { + const scenarios = selectedGatewayScenarios(); + if (!scenarios.length) { + return; + } + const view = byId("gateway-view").value; + const metric = byId("gateway-metric").value; + const statistic = byId("gateway-percentile").value; + const series = gatewaySeries(scenarios, view, metric, statistic); + const payloads = scenarios.map((scenario) => scenario.payload_bytes); + const isDelta = view !== "absolute"; + drawLineChart(byId("gateway-chart"), series, payloads, isDelta); + renderLegend(series); + + const metricLabel = metric === "first_content" ? "time to first content" : "total response time"; + const viewLabel = view === "absolute" + ? "absolute latency" + : view === "minimal" + ? "variant overhead relative to minimal Relay" + : "Relay overhead relative to direct provider calls"; + byId("gateway-chart-description").textContent = + `${statistic.replace("_ms", "")} ${metricLabel}; ${viewLabel}. ` + + `Provider ${byId("gateway-provider").value}, ${byId("gateway-mode").value}, ` + + `concurrency ${byId("gateway-concurrency").value}.`; + + const rows = []; + for (const item of series) { + for (const point of item.values) { + const interval = point.summary.median_ci95_ms; + rows.push([ + item.label, + formatBytes(point.payload), + formatMs(point.summary.p50_ms), + formatMs(point.summary.p95_ms), + formatMs(point.summary.p99_ms), + interval ? `${formatMs(interval[0])} to ${formatMs(interval[1])}` : "—", + String(point.summary.samples), + ]); + } + } + renderTable( + byId("gateway-table"), + ["Path or comparison", "Payload", "p50", "p95", "p99", "Median 95% CI", "Samples"], + rows, + [2, 3, 4, 6], + ); +} + +function initializeGateway() { + if (!benchmark.gateway || !benchmark.gateway.length) { + byId("gateway-section").hidden = true; + return; + } + populateSelect( + byId("gateway-provider"), + uniqueSorted(benchmark.gateway.map((scenario) => scenario.provider)).map((value) => [value, value]), + ); + populateSelect( + byId("gateway-mode"), + uniqueSorted(benchmark.gateway.map((scenario) => scenario.mode)).map((value) => [value, value]), + ); + populateSelect( + byId("gateway-concurrency"), + uniqueSorted(benchmark.gateway.map((scenario) => scenario.concurrency), true).map((value) => [String(value), String(value)]), + ); + populateSelect(byId("gateway-percentile"), [["p50_ms", "p50"], ["p95_ms", "p95"], ["p99_ms", "p99"]]); + populateSelect(byId("gateway-view"), [ + ["direct", "Paired delta vs direct"], + ["minimal", "Paired delta vs minimal"], + ["absolute", "Absolute latency"], + ]); + syncMetricOptions(); + for (const id of [ + "gateway-provider", + "gateway-mode", + "gateway-concurrency", + "gateway-metric", + "gateway-percentile", + "gateway-view", + ]) { + byId(id).addEventListener("change", () => { + if (id === "gateway-mode") { + syncMetricOptions(); + } + renderGateway(); + }); + } + renderGateway(); +} + +function drawBarChart(svg, entries) { + svg.replaceChildren(); + const width = 920; + const rowHeight = 38; + const margin = { top: 16, right: 90, bottom: 28, left: 220 }; + const innerWidth = width - margin.left - margin.right; + const height = margin.top + margin.bottom + entries.length * rowHeight; + const maximum = Math.max(...entries.map((entry) => entry.value), 0.001) * 1.08; + svg.setAttribute("viewBox", `0 0 ${width} ${height}`); + + entries.forEach((entry, index) => { + const y = margin.top + index * rowHeight; + const barWidth = Math.max((entry.value / maximum) * innerWidth, 1); + addSvg(svg, "text", { class: "bar-label", x: margin.left - 12, y: y + 20, "text-anchor": "end" }, entry.label); + const bar = addSvg(svg, "rect", { + x: margin.left, + y: y + 5, + width: barWidth, + height: 22, + rx: 4, + fill: colors[index % colors.length], + }); + addSvg(bar, "title", {}, `${entry.label}: ${formatMs(entry.value)}`); + addSvg(svg, "text", { class: "bar-value", x: margin.left + barWidth + 8, y: y + 20 }, formatMs(entry.value)); + }); +} + +function summaryRows(collection, comparisons = false) { + return Object.entries(collection).map(([name, summary]) => { + const interval = summary.median_ci95_ms; + return [ + labelName(name), + formatMs(summary.p50_ms), + formatMs(summary.p95_ms), + formatMs(summary.p99_ms), + interval ? `${formatMs(interval[0])} to ${formatMs(interval[1])}` : "—", + String(summary.samples), + ]; + }); +} + +function renderProcessSuite(name) { + const result = benchmark[name]; + if (!result) { + byId(`${name}-section`).hidden = true; + return; + } + const entries = Object.entries(result.absolute).map(([path, summary]) => ({ + label: labelName(path), + value: summary.p50_ms, + })); + drawBarChart(byId(`${name}-chart`), entries); + const headers = ["Path", "p50", "p95", "p99", "Median 95% CI", "Samples"]; + renderTable(byId(`${name}-absolute-table`), headers, summaryRows(result.absolute), [1, 2, 3, 5]); + renderTable(byId(`${name}-comparison-table`), headers, summaryRows(result.comparisons, true), [1, 2, 3, 5]); +} + +function renderDelivery() { + if (!benchmark.exporter_delivery) { + byId("delivery-section").hidden = true; + return; + } + const cards = byId("delivery-cards"); + summaryCard(cards, "ATOF written", formatBytes(benchmark.exporter_delivery.atof_bytes)); + summaryCard(cards, "OTLP requests", Number(benchmark.exporter_delivery.otlp_requests).toLocaleString()); +} + +renderOverview(); +initializeGateway(); +renderProcessSuite("hooks"); +renderProcessSuite("startup"); +renderDelivery(); diff --git a/scripts/latency_benchmark/src/report/styles.css b/scripts/latency_benchmark/src/report/styles.css new file mode 100644 index 000000000..2a0458857 --- /dev/null +++ b/scripts/latency_benchmark/src/report/styles.css @@ -0,0 +1,434 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +:root { + color-scheme: light; + --ink: #1a1a1a; + --muted: #5b5b5b; + --line: #d6d6d6; + --panel: #ffffff; + --soft: #f5f5f5; + --accent: #76b900; + --accent-dark: #456d00; + --focus: #006eb8; + font-family: "NVIDIA Sans", Arial, Helvetica, sans-serif; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + color: var(--ink); + background: var(--panel); + font-size: 16px; + line-height: 1.6; +} + +.page-shell { + width: min(1180px, calc(100% - 64px)); + margin: 0 auto; +} + +.hero { + padding: 42px 0 36px; + border-top: 4px solid var(--accent); + border-bottom: 1px solid var(--line); + background: var(--panel); +} + +.hero h1 { + margin: 0; + font-size: clamp(2.4rem, 6vw, 3.25rem); + font-weight: 700; + line-height: 1.12; + letter-spacing: -0.035em; +} + +main { + padding: 0 0 72px; +} + +section { + margin: 0; + padding: 42px 0; + border-bottom: 1px solid var(--line); +} + +section:first-child { + padding-top: 34px; +} + +section[hidden] { + display: none; +} + +h2, +h3 { + color: var(--ink); + line-height: 1.25; +} + +h2 { + margin: 0 0 24px; + font-size: 2rem; + font-weight: 700; + letter-spacing: -0.02em; +} + +h3 { + margin: 0 0 12px; + font-size: 1.12rem; + font-weight: 700; +} + +p { + color: var(--muted); +} + +code { + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; +} + +.summary-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); + gap: 16px; +} + +.summary-card { + min-height: 104px; + padding: 17px 18px; + border: 1px solid var(--line); + border-top: 3px solid var(--accent); + background: var(--panel); +} + +.summary-card .label { + color: var(--muted); + font-size: 0.74rem; + font-weight: 700; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.summary-card .value { + display: block; + margin-top: 8px; + overflow-wrap: anywhere; + color: var(--ink); + font-size: 1.22rem; + font-weight: 700; +} + +details { + margin-top: 24px; + border: 1px solid var(--line); + background: var(--panel); +} + +summary { + padding: 14px 18px; + cursor: pointer; + color: var(--accent-dark); + font-weight: 700; +} + +.details-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 24px; + padding: 20px; + border-top: 1px solid var(--line); +} + +.split-tables { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 24px; + margin-top: 28px; +} + +.definition-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 16px; +} + +.definition-grid article { + padding: 18px 20px; + border-left: 3px solid var(--accent); + background: var(--soft); +} + +.definition-grid p, +.section-heading p, +.chart-card p, +.table-section p { + margin: 0; +} + +.callout { + margin: 24px 0 0; + padding: 16px 20px; + border-left: 3px solid var(--accent); + color: #314200; + background: #f2f8e8; +} + +.section-heading { + display: grid; + grid-template-columns: minmax(210px, 0.55fr) minmax(300px, 1.45fr); + gap: 40px; + align-items: start; + margin-bottom: 28px; +} + +.section-heading h2 { + margin-bottom: 0; +} + +.section-heading > p { + max-width: 760px; +} + +.controls { + display: grid; + grid-template-columns: repeat(3, minmax(160px, 1fr)); + gap: 16px; + margin-bottom: 24px; + padding: 20px; + border: 1px solid var(--line); + background: var(--soft); +} + +label { + display: grid; + gap: 6px; + color: var(--ink); + font-size: 0.82rem; + font-weight: 700; +} + +select { + width: 100%; + min-height: 40px; + padding: 8px 10px; + border: 1px solid #a7a7a7; + border-radius: 2px; + color: var(--ink); + background: #fff; + font: inherit; + font-size: 0.9rem; +} + +select:focus-visible, +summary:focus-visible { + outline: 3px solid rgba(0, 110, 184, 0.32); + outline-offset: 2px; +} + +.chart-card { + padding: 24px; + overflow: hidden; + border: 1px solid var(--line); + background: var(--panel); +} + +svg { + display: block; + width: 100%; + margin-top: 16px; + overflow: visible; +} + +.line-chart { + min-height: 360px; +} + +.bar-chart { + min-height: 220px; +} + +.axis, +.grid-line { + stroke: #d4d4d4; + stroke-width: 1; +} + +.grid-line { + stroke-dasharray: 3 5; +} + +.zero-line { + stroke: #747474; + stroke-width: 1.5; +} + +.chart-label { + fill: #595959; + font-size: 12px; +} + +.line-series { + fill: none; + stroke-width: 2.5; + stroke-linecap: round; + stroke-linejoin: round; +} + +.chart-point { + stroke: #fff; + stroke-width: 2; +} + +.bar-label { + fill: #262626; + font-size: 12px; +} + +.bar-value { + fill: #595959; + font-size: 12px; + font-weight: 700; +} + +.legend { + display: flex; + flex-wrap: wrap; + gap: 10px 20px; + margin-top: 8px; + color: var(--muted); + font-size: 0.82rem; +} + +.legend-item { + display: inline-flex; + gap: 8px; + align-items: center; +} + +.legend-swatch { + width: 26px; + border-top: 3px solid; +} + +.table-section, +.split-tables { + margin-top: 28px; +} + +.table-wrap { + max-width: 100%; + overflow-x: auto; + border: 1px solid var(--line); +} + +table { + width: 100%; + border-collapse: collapse; + font-size: 0.84rem; +} + +th, +td { + padding: 10px 12px; + border-bottom: 1px solid #e1e1e1; + text-align: left; + white-space: nowrap; +} + +th { + color: #333333; + background: var(--soft); + font-size: 0.72rem; + letter-spacing: 0.05em; + text-transform: uppercase; +} + +td.numeric { + text-align: right; + font-variant-numeric: tabular-nums; +} + +tr:last-child td { + border-bottom: 0; +} + +tbody tr:hover { + background: #f8fbef; +} + +footer { + padding: 24px 0 32px; + border-top: 1px solid var(--line); + color: var(--muted); + background: var(--soft); + font-size: 0.85rem; +} + +@media (max-width: 760px) { + .page-shell { + width: min(100% - 32px, 1180px); + } + + .hero { + padding: 32px 0 28px; + } + + section { + padding: 34px 0; + } + + .details-grid, + .split-tables, + .definition-grid, + .section-heading { + grid-template-columns: 1fr; + } + + .controls { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .details-grid { + padding: 16px; + } + + .line-chart { + min-height: 300px; + } +} + +@media (max-width: 560px) { + .summary-grid, + .controls { + grid-template-columns: 1fr; + } + + .hero h1 { + font-size: 2.25rem; + } + + .chart-card { + padding: 16px; + } +} + +@media print { + .hero { + padding-top: 24px; + } + + section { + break-inside: avoid; + } + + .controls, + details { + display: none; + } +} diff --git a/scripts/latency_benchmark/src/report/template.html b/scripts/latency_benchmark/src/report/template.html new file mode 100644 index 000000000..c10f343f4 --- /dev/null +++ b/scripts/latency_benchmark/src/report/template.html @@ -0,0 +1,213 @@ + + + + + + + NeMo Relay Latency Report + + + +
+
+

NeMo Relay Latency Report

+
+
+ +
+
+

Run Summary

+
+
+ Environment and resolved configuration +
+
+

Environment

+
+
+
+

Configuration

+
+
+
+
+
+ +
+

How to Read This Report

+
+
+

Absolute Latency

+

+ Elapsed wall time for one path. Use it to understand the complete + local operation, including process work where applicable. +

+
+
+

Paired Delta

+

+ The left path minus its baseline in the same measurement cycle. + Positive values mean added time. Small negative values can occur + because of scheduler and measurement noise. +

+
+
+

Percentiles

+

+ p50 is the median. p95 and p99 describe slower tail observations. + Min and max are the observed extremes and are more sensitive to + outliers. +

+
+
+

Median 95% CI

+

+ The bootstrap interval estimates uncertainty around the median + paired delta. It is not a range containing 95% of observations. +

+
+
+

+ Prefer paired added milliseconds over percentages. Loopback baselines + are very small, so a harmless absolute difference can look large as a + percentage. Do not draw performance conclusions from a smoke test. +

+
+ +
+
+
+

Gateway

+
+

+ Compares deterministic provider calls made directly and through + minimal, ATOF file-exporter, OTLP, and opt-in middleware Relay + gateways. +

+
+
+ + + + + + +
+
+

Gateway Latency by Payload Size

+

+ +
+
+
+

Selected Gateway Data

+

+ The table is the accessible numeric counterpart to the graph. The + confidence interval is available for paired median deltas only. +

+
+
+
+ +
+
+
+

Hooks

+
+

+ Measures full nemo-relay hook-forward subprocess wall + time for Codex and Claude Code. Paired deltas subtract a + nemo-relay --version process baseline. +

+
+
+

Hook Absolute p50

+

Median subprocess wall time in milliseconds.

+ +
+
+
+

Absolute Latency

+
+
+
+

Paired Delta From Process Baseline

+
+
+
+
+ +
+
+
+

Startup

+
+

+ Measures cold Relay process launch through a healthy gateway. Paired + deltas subtract the same process baseline used by the hook suite. +

+
+
+

Startup Absolute p50

+

Median process or readiness time in milliseconds.

+ +
+
+
+

Absolute Latency

+
+
+
+

Paired Delta From Process Baseline

+
+
+
+
+ +
+
+
+

Exporter Delivery

+
+

+ These counts confirm that the local ATOF and OTLP exporters delivered + data. They are correctness signals, not latency metrics. +

+
+
+
+
+ +
+
+ Generated by just latency-benchmark. Keep the + companion JSON file when sharing or analyzing this report. +
+
+ + + + + diff --git a/scripts/benchmark_coding_agent_latency/reporting.py b/scripts/latency_benchmark/src/reporting.py similarity index 90% rename from scripts/benchmark_coding_agent_latency/reporting.py rename to scripts/latency_benchmark/src/reporting.py index 49e902e8f..70add2cfe 100644 --- a/scripts/benchmark_coding_agent_latency/reporting.py +++ b/scripts/latency_benchmark/src/reporting.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any -ROOT = Path(__file__).resolve().parents[2] +ROOT = Path(__file__).resolve().parents[3] def _git_output(*args: str) -> str: @@ -40,12 +40,10 @@ def print_results(results: dict[str, Any]) -> None: headers = ("provider", "mode", "bytes", "c", "comparison", "metric", "p50", "p95", "p99") print(" ".join(f"{header:>12}" for header in headers)) for scenario in results["gateway"]: - for comparison in ( - "relay-minimal_vs_direct", - "relay-file_vs_direct", - "relay-otlp_vs_direct", - ): - for metric, summary in scenario["comparisons"][comparison].items(): + for comparison, metrics in scenario["comparisons"].items(): + if not comparison.endswith("_vs_direct"): + continue + for metric, summary in metrics.items(): values = ( scenario["provider"], scenario["mode"], diff --git a/scripts/benchmark_coding_agent_latency/servers.py b/scripts/latency_benchmark/src/servers.py similarity index 100% rename from scripts/benchmark_coding_agent_latency/servers.py rename to scripts/latency_benchmark/src/servers.py diff --git a/scripts/benchmark_coding_agent_latency/tests/__init__.py b/scripts/latency_benchmark/tests/__init__.py similarity index 100% rename from scripts/benchmark_coding_agent_latency/tests/__init__.py rename to scripts/latency_benchmark/tests/__init__.py diff --git a/scripts/latency_benchmark/tests/test_config.py b/scripts/latency_benchmark/tests/test_config.py new file mode 100644 index 000000000..1618f5bf8 --- /dev/null +++ b/scripts/latency_benchmark/tests/test_config.py @@ -0,0 +1,214 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the coding-agent latency benchmark fixture.""" + +import tempfile +import unittest +from pathlib import Path + +from scripts.latency_benchmark.src.config import ( + DEFAULT_CONFIG_PATH, + MiddlewareVariant, + build_parser, + load_config, + parse_args, +) +from scripts.latency_benchmark.src.fixtures import ( + write_agent_config, + write_mock_codex, + write_plugin_configs, +) + + +class BenchmarkConfigTests(unittest.TestCase): + def test_default_config_defines_every_suite_and_matrix_axis(self) -> None: + config = load_config(DEFAULT_CONFIG_PATH) + + self.assertEqual(config.tests, ("gateway", "hooks", "startup")) + self.assertEqual(config.providers, ("openai", "anthropic")) + self.assertEqual(config.modes, ("buffered", "streaming")) + self.assertTrue(config.payload_sizes) + self.assertTrue(config.concurrency) + self.assertEqual(config.middleware, ()) + + def test_partial_config_and_cli_arguments_override_defaults(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + custom_config = root / "quick.toml" + custom_config.write_text('tests = ["startup"]\nsamples = 7\n', encoding="utf-8") + relay_bin = root / "nemo-relay" + relay_bin.touch() + + options = parse_args( + [ + "--relay-bin", + str(relay_bin), + "--output", + str(root / "results.json"), + "--config", + str(custom_config), + "--tests", + "gateway,hooks", + "--samples", + "3", + "--concurrency", + "1", + "--providers", + "openai", + ] + ) + + self.assertEqual(options.config.tests, ("gateway", "hooks")) + self.assertEqual(options.config.samples, 3) + self.assertEqual(options.config.concurrency, (1,)) + self.assertEqual(options.config.providers, ("openai",)) + self.assertEqual(options.config.modes, ("buffered", "streaming")) + self.assertEqual(options.report, (root / "results.html").resolve()) + + def test_help_uses_just_entrypoint_and_explains_every_option(self) -> None: + help_text = build_parser().format_help() + + self.assertIn("usage: just latency-benchmark [options]", help_text) + for option in ( + "-h, --help", + "--relay-bin", + "--output", + "--report", + "--config", + "--tests", + "--providers", + "--modes", + "--payload-sizes", + "--concurrency", + "--middleware", + "--samples", + "--hook-samples", + "--startup-samples", + "--warmup", + "--response-bytes", + "--stream-chunks", + ): + with self.subTest(option=option): + self.assertIn(option, help_text) + + def test_report_path_can_be_overridden(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + relay_bin = root / "nemo-relay" + relay_bin.touch() + + options = parse_args( + [ + "--relay-bin", + str(relay_bin), + "--output", + str(root / "results.json"), + "--report", + str(root / "site" / "index.html"), + ] + ) + + self.assertEqual(options.report, (root / "site" / "index.html").resolve()) + + def test_middleware_paths_are_relative_to_the_custom_config(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + plugin_config = root / "plugins-guardrails.toml" + plugin_config.write_text("version = 1\ncomponents = []\n", encoding="utf-8") + config_path = root / "benchmark.toml" + config_path.write_text( + '[[middleware]]\nname = "guardrails"\nplugin_config = "plugins-guardrails.toml"\n', + encoding="utf-8", + ) + + config = load_config(config_path) + + self.assertEqual( + config.middleware, + (MiddlewareVariant(name="guardrails", plugin_config=plugin_config.resolve()),), + ) + + def test_cli_middleware_replaces_configured_middleware(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + relay_bin = root / "nemo-relay" + relay_bin.touch() + plugin_config = root / "plugins-redaction.toml" + plugin_config.write_text("version = 1\ncomponents = []\n", encoding="utf-8") + + options = parse_args( + [ + "--relay-bin", + str(relay_bin), + "--output", + str(root / "results.json"), + "--middleware", + f"redaction={plugin_config}", + ] + ) + + self.assertEqual( + options.config.middleware, + (MiddlewareVariant(name="redaction", plugin_config=plugin_config.resolve()),), + ) + + def test_rejects_reserved_middleware_name(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + plugin_config = root / "plugins.toml" + plugin_config.touch() + config_path = root / "invalid.toml" + config_path.write_text( + '[[middleware]]\nname = "minimal"\nplugin_config = "plugins.toml"\n', + encoding="utf-8", + ) + + with self.assertRaisesRegex(ValueError, "reserved by a default variant"): + load_config(config_path) + + def test_rejects_unknown_config_keys(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + config_path = Path(temporary) / "invalid.toml" + config_path.write_text("sample = 1\n", encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "unknown config key"): + load_config(config_path) + + def test_rejects_gateway_concurrency_greater_than_samples(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + config_path = Path(temporary) / "invalid.toml" + config_path.write_text( + 'tests = ["gateway"]\nsamples = 2\nconcurrency = [4]\n', + encoding="utf-8", + ) + + with self.assertRaisesRegex(ValueError, "samples must be greater"): + load_config(config_path) + + +class StaticFixtureTests(unittest.TestCase): + def test_materializes_templates_without_embedded_markers(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + custom_config = root / "plugins-guardrails.toml" + custom_config.write_text("version = 1\ncomponents = []\n", encoding="utf-8") + configs = write_plugin_configs( + root, + "http://127.0.0.1:4318", + (MiddlewareVariant(name="guardrails", plugin_config=custom_config),), + ) + mock_codex = write_mock_codex(root) + agent_config = write_agent_config(root, "test", mock_codex) + + rendered = "\n".join(path.read_text(encoding="utf-8") for path in (*configs.values(), agent_config)) + + self.assertEqual(mock_codex.stem, "mock-codex") + self.assertNotIn("__ATOF_OUTPUT_DIRECTORY__", rendered) + self.assertNotIn("__OTLP_ENDPOINT__", rendered) + self.assertNotIn("__CODEX_COMMAND__", rendered) + self.assertEqual(configs["relay-guardrails"], custom_config) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/latency_benchmark/tests/test_html_report.py b/scripts/latency_benchmark/tests/test_html_report.py new file mode 100644 index 000000000..5ccb28a71 --- /dev/null +++ b/scripts/latency_benchmark/tests/test_html_report.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the self-contained benchmark report.""" + +import tempfile +import unittest +from pathlib import Path +from typing import Any + +from scripts.latency_benchmark.src.html_report import render_html_report, write_html_report + + +def _summary(*, interval: bool = False) -> dict[str, Any]: + result: dict[str, Any] = { + "samples": 5, + "p50_ms": 1.0, + "p95_ms": 1.5, + "p99_ms": 1.6, + "min_ms": 0.8, + "max_ms": 1.7, + } + if interval: + result["median_ci95_ms"] = [0.9, 1.1] + return result + + +def _sample_results() -> dict[str, Any]: + absolute = { + name: {"total": _summary()} + for name in ("direct", "relay-minimal", "relay-file", "relay-otlp", "relay-guardrails") + } + comparisons = { + name: {"total": _summary(interval=True)} + for name in ( + "relay-minimal_vs_direct", + "relay-file_vs_direct", + "relay-otlp_vs_direct", + "file_exporter_vs_minimal", + "otlp_exporter_vs_minimal", + "relay-guardrails_vs_direct", + "guardrails_vs_minimal", + ) + } + process_absolute = { + "process_baseline": _summary(), + "relay-minimal": _summary(), + } + return { + "schema_version": 2, + "environment": { + "generated_at": "2026-08-05T12:00:00+00:00", + "git_commit": "0123456789abcdef", + "git_dirty": False, + "relay_version": "nemo-relay 0.1.0", + "platform": "test", + }, + "parameters": { + "tests": ["gateway", "hooks", "startup"], + "providers": ["openai"], + "modes": ["buffered"], + "samples": 5, + "payload_sizes": [4096], + "concurrency": [1], + "middleware": [{"name": "guardrails", "plugin_config": "/tmp/plugins.toml"}], + }, + "gateway": [ + { + "provider": "openai", + "mode": "buffered", + "payload_bytes": 4096, + "serialized_request_bytes": 4200, + "concurrency": 1, + "absolute": absolute, + "comparisons": comparisons, + } + ], + "hooks": { + "absolute": {"process_baseline": _summary(), "codex_minimal": _summary()}, + "comparisons": {"codex_minimal_vs_process_baseline": _summary(interval=True)}, + }, + "startup": { + "absolute": process_absolute, + "comparisons": {"relay-minimal_readiness_vs_process_baseline": _summary(interval=True)}, + }, + "exporter_delivery": {"atof_bytes": 8192, "otlp_requests": 4}, + } + + +class HtmlReportTests(unittest.TestCase): + def test_report_embeds_results_and_static_assets(self) -> None: + report = render_html_report(_sample_results()) + + self.assertIn("NeMo Relay Latency Report", report) + self.assertIn("

NeMo Relay Latency Report

", report) + self.assertNotIn("Coding-Agent Latency Report", report) + self.assertNotIn('class="eyebrow"', report) + self.assertNotIn('class="lede"', report) + self.assertIn('"git_commit":"0123456789abcdef"', report) + self.assertIn("Gateway Latency by Payload Size", report) + self.assertIn("function drawLineChart", report) + self.assertNotIn("__BENCHMARK_DATA__", report) + self.assertNotIn("__BENCHMARK_STYLES__", report) + self.assertNotIn("__BENCHMARK_SCRIPT__", report) + + def test_report_escapes_script_terminators_in_embedded_data(self) -> None: + results = _sample_results() + results["environment"]["platform"] = "" + + report = render_html_report(results) + + self.assertNotIn("