diff --git a/.gitattributes b/.gitattributes index b60c7cf49..78065eb75 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,4 @@ # SPDX-License-Identifier: Apache-2.0 examples/python-grpc-worker-plugin/nemo_relay_python_grpc_worker_example/worker.py text eol=lf +*.cmd text eol=crlf diff --git a/docs/reference/performance.mdx b/docs/reference/performance.mdx index 2ca497cb1..392fe2bc4 100644 --- a/docs/reference/performance.mdx +++ b/docs/reference/performance.mdx @@ -29,6 +29,50 @@ Use these practices when applying the concept in application or integration code - Use execution intercepts when you need to wrap real execution and sanitize guardrails when you only need to change emitted observability payloads. - Use binding-native typed wrappers and codecs when provider payload conversion would otherwise be repeated at many call sites. +## Latency Benchmark + +Use the opt-in latency benchmark to measure the local latency that Relay +adds around Codex and Claude Code traffic. The harness uses deterministic +loopback OpenAI and Anthropic providers so network and model-service latency do +not hide Relay's contribution. + +Run the benchmark with the following command: + +```bash +just latency-benchmark +``` + +The command builds the release CLI. Its default configuration measures the +following paths: + +- Direct requests to the mock provider. +- Relay with no exporter, which isolates the managed gateway pipeline. +- Relay with the ATOF file exporter. +- Relay with the OTLP exporter sending to a local OpenTelemetry HTTP receiver. +- Full `nemo-relay hook-forward` subprocesses for Codex and Claude Code. +- Cold Relay process startup through gateway readiness. + +Gateway scenarios cover OpenAI Responses and Anthropic Messages, buffered and +streaming responses, multiple request payload sizes, and multiple concurrency +levels. + +For buffered calls, inspect total latency. For streaming calls, inspect both +time to the first content delta and total stream time. The report includes +p50, p95, and p99 paired latency differences and a bootstrap 95% confidence +interval for the median. + +For setup, CLI options, configuration files, middleware variants, storage, +output paths, and troubleshooting, refer to the +[latency benchmark run guide](https://github.com/NVIDIA/NeMo-Relay/blob/main/scripts/latency_benchmark/README.md). + +Treat results as environment-specific. Record the commit, release build, +hardware, operating system, workload sizes, and sample counts when sharing a +number. Prefer added milliseconds over percentages: a small absolute increase +can look disproportionately large when the direct loopback baseline is much +faster than a real model call. Use real Codex or Claude Code runs as an +end-to-end validation, not as the primary gateway measurement, because host +startup and scheduling add unrelated variance. + ## Related Topics Use these links to continue into adjacent concepts and workflows. diff --git a/justfile b/justfile index c1f6ec630..5bd3a1e14 100644 --- a/justfile +++ b/justfile @@ -1169,6 +1169,28 @@ test-codex-plugin-e2e: test-claude-plugin-e2e: ./scripts/test-claude-plugin-e2e.sh +# Opt-in: builds the release CLI and runs configurable local latency suites. +[positional-arguments] +latency-benchmark *benchmark_args: + #!/usr/bin/env bash + set -euo pipefail + result_dir={{ quote(output_dir) }} + result_dir="${result_dir:-target/benchmark-results}" + for argument in "$@"; do + if [[ "$argument" == "-h" || "$argument" == "--help" ]]; then + exec uv run --locked python -m scripts.latency_benchmark.src "$@" + fi + done + cargo build --locked --release -p nemo-relay-cli + uv run --locked python -m scripts.latency_benchmark.src \ + --relay-bin target/release/nemo-relay \ + --output "$result_dir/nemo-relay-latency-report.json" \ + "$@" + +# Runs the fast latency benchmark fixture tests without building Relay. +test-latency-benchmark: + uv run --locked python -m pytest scripts/latency_benchmark/tests + # --set [output_dir=] [ci=true|false] test-rust: #!/usr/bin/env bash diff --git a/scripts/README.md b/scripts/README.md index 783df8f92..c4c324e88 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -25,6 +25,42 @@ These checks exercise installed coding-agent clients and are intentionally outsi - `just test-codex-plugin-e2e` - `just test-claude-plugin-e2e` +## Latency Benchmark + +Run `just latency-benchmark` to build the release CLI and compare +direct provider requests with Relay's minimal, ATOF file exporter, and OTLP +exporter configurations. The benchmark also measures full hook subprocess and +cold gateway startup time. It writes structured results under +`target/benchmark-results/nemo-relay-latency-report.json` and +`target/benchmark-results/nemo-relay-latency-report.html` by default and is +intentionally outside regular CI. Large ATOF output goes to a +`nemo-relay-latency-*` directory in the operating system's temporary location +and is removed after a normal run. + +Run `just test-latency-benchmark` to execute the fixture's fast unit tests +without building Relay. + +The defaults live in `scripts/latency_benchmark/config/default.toml`. Supply a +partial TOML file with `--config`, or override individual values on the command +line. For example, this runs only a small OpenAI gateway matrix: + +```bash +just latency-benchmark \ + --tests gateway \ + --providers openai \ + --payload-sizes 4096 \ + --concurrency 1 \ + --samples 10 +``` + +Run `just latency-benchmark --help` to list all overrides without +building Relay. The three selectable suites are `gateway`, `hooks`, and +`startup`. Each run writes machine-readable JSON and a self-contained HTML +report with graphs. Add repeatable `--middleware NAME=PATH` options to measure +custom Relay plugin configurations alongside the three default variants. Refer +to [`latency_benchmark/README.md`](latency_benchmark/README.md) for the complete +human-facing run guide. + ## Internal Layout - `docs/`: Fern reference-generation, migration cleanup, and `docs-website` branch sync helpers. Generated API reference output under `docs/reference/api/*-library-reference/` is ignored and recreated by `just docs`. diff --git a/scripts/latency_benchmark/README.md b/scripts/latency_benchmark/README.md new file mode 100644 index 000000000..f15d68838 --- /dev/null +++ b/scripts/latency_benchmark/README.md @@ -0,0 +1,300 @@ + + +# Latency Benchmark + +Use this opt-in benchmark to measure the local latency that NeMo Relay adds +around OpenAI Responses, Anthropic Messages, Codex hooks, Claude Code hooks, +and Relay process startup. The fixture runs deterministic providers on +loopback, so network and model-service latency do not hide Relay overhead. + +## Before You Run + +Run all commands from the repository root. Start with the smoke test unless you +are collecting reportable performance results. + +The default matrix can write about 25 GiB of temporary ATOF data. Keep at least +30 GiB free in the operating system's temporary directory. The benchmark +removes this data after a normal run. + +## Prerequisites + +Install the repository development prerequisites, including Rust, Python 3.11 +or newer, `uv`, and `just`. The `just` recipe builds the release-mode Relay CLI +before running the benchmark. + +## Fixture Layout + +The benchmark keeps executable source under `src/`, repeatable TOML fixtures +under `config/`, static coding-agent fixtures under `data/`, and focused unit +tests under `tests/`. Add runtime behavior to `src/` and keep fixed test input +outside executable modules. + +Run the fixture's fast unit tests with the following recipe: + +```bash +just test-latency-benchmark +``` + +The recipe runs +`uv run --locked python -m pytest scripts/latency_benchmark/tests`. Keep the +`python -m` form so imports resolve from the repository root. + +The `data/mock-codex.*` fixture is a small lifecycle stub used when Relay starts +a configured Codex command in transparent mode. It is not a simulated hook +client. The hooks suite invokes `nemo-relay hook-forward codex` and +`nemo-relay hook-forward claude` directly, so it does not launch either real +coding-agent executable and does not need a separate mock Claude command. + +## Run a Smoke Test + +Use a small matrix to verify the fixture and exporter paths: + +```bash +just latency-benchmark \ + --tests gateway \ + --providers openai \ + --modes buffered \ + --payload-sizes 4096 \ + --concurrency 1 \ + --samples 5 \ + --warmup 1 \ + --response-bytes 1024 +``` + +Do not use a smoke-test result for performance conclusions. Its sample count +is only large enough to catch functional failures. + +## Run the Default Matrix + +After you check available disk space, run the default matrix with the following +command: + +```bash +just latency-benchmark +``` + +The default configuration runs the following three suites: + +| Suite | What It Measures | +| --- | --- | +| `gateway` | Request latency through direct, minimal Relay, ATOF file exporter, and OTLP exporter paths | +| `hooks` | Full Codex and Claude Code `nemo-relay hook-forward` subprocess wall time | +| `startup` | Cold Relay process launch through a healthy gateway | + +During a run, the terminal prints an `[x/y]` indicator after each completed +benchmark test. Each gateway matrix scenario counts as one test. The hooks and +startup suites each count as one test. + +### Gateway Suite + +The gateway suite sends deterministic OpenAI Responses and Anthropic Messages +requests to a loopback provider. By default, each measurement cycle runs the +same request through four paths in a rotated order: + +- `direct` calls the mock provider without Relay. +- `relay-minimal` measures Relay without an exporter and isolates the managed + gateway pipeline. +- `relay-file` adds the ATOF file exporter. +- `relay-otlp` adds the OTLP exporter. + +The suite varies provider protocol, buffered or streaming response mode, +request-content size, and concurrency. `total` measures from request start +until the buffered body is read or the streaming response reaches end of +stream. Streaming scenarios also report `first_content`, which measures from +request start until the first content-delta event arrives. + +The primary paired comparisons subtract `direct` from each Relay path. The +`file_exporter_vs_minimal` and `otlp_exporter_vs_minimal` comparisons subtract +minimal Relay to isolate exporter overhead. Pairing measurements from the same +cycle reduces unrelated timing variation. + +### Hooks Suite + +The hooks suite measures the complete wall time of a new +`nemo-relay hook-forward` subprocess for Codex and Claude Code through minimal, +ATOF file exporter, and OTLP exporter configurations. Its paired comparisons +subtract a `nemo-relay --version` subprocess measured in the same cycle. This +process baseline estimates generic executable startup cost; it is not a no-op +hook. + +### Startup Suite + +The startup suite measures a cold Relay process from launch until its +`/healthz` endpoint reports ready for minimal, ATOF file exporter, and OTLP +exporter configurations. Its paired comparisons subtract the same +`nemo-relay --version` process baseline to make Relay-specific readiness work +easier to distinguish. + +## Configure a Run + +The benchmark resolves settings in this order: + +1. Defaults from `config/default.toml`. +2. Values from the file passed with `--config`. +3. CLI arguments, which take final precedence. + +A custom TOML file can contain only the settings that differ from the +defaults. For example: + +```toml +tests = ["gateway"] +providers = ["openai"] +modes = ["streaming"] +samples = 50 +warmup = 3 +payload_sizes = [4096, 65536] +concurrency = [1, 4] +``` + +Run the custom configuration with the following command: + +```bash +just latency-benchmark --config /path/to/benchmark.toml +``` + +Override any list from the command line with comma-separated values: + +```bash +just latency-benchmark \ + --config /path/to/benchmark.toml \ + --tests gateway,startup \ + --providers openai,anthropic \ + --modes buffered \ + --concurrency 1,4,8 +``` + +List every supported override without running the benchmark: + +```bash +just latency-benchmark --help +``` + +## Benchmark Custom Middleware + +Every run includes the minimal Relay (`relay-minimal`), ATOF file exporter +(`relay-file`), and OTLP exporter (`relay-otlp`) variants. Add middleware as an +opt-in variant by pointing the benchmark to a valid Relay plugin configuration. + +The fixture includes `config/plugins-pii-redaction.toml` for the simplest +self-contained middleware test. It installs the built-in email detector and +does not require an external service. Run the following small gateway matrix: + +```bash +just latency-benchmark \ + --tests gateway \ + --providers openai \ + --modes buffered \ + --payload-sizes 4096 \ + --concurrency 1 \ + --samples 5 \ + --warmup 1 \ + --response-bytes 1024 \ + --middleware pii-redaction=scripts/latency_benchmark/config/plugins-pii-redaction.toml +``` + +This smoke test verifies that Relay loads and executes the middleware and that +the reports include the custom variant. The deterministic benchmark payload +does not contain an email address, so use this command to test latency plumbing, +not redaction correctness. Increase the sample count before drawing performance +conclusions. + +For repeatable custom runs, add one or more `[[middleware]]` tables to a +benchmark TOML file: + +```toml +[[middleware]] +name = "pii-redaction" +plugin_config = "./plugins-pii-redaction.toml" +``` + +The `plugin_config` path is relative to the benchmark TOML file. Run the +configuration with the following command: + +```bash +just latency-benchmark --config /path/to/benchmark.toml +``` + +For a one-off run, use `--middleware NAME=PATH`. Repeat the option to add more +than one variant: + +```bash +just latency-benchmark \ + --middleware pii-redaction=/path/to/plugins-pii-redaction.toml \ + --middleware guardrails=/path/to/plugins-guardrails.toml +``` + +CLI middleware options replace the `[[middleware]]` entries from the benchmark +TOML file. Names must contain lowercase letters, digits, or hyphens and cannot +be `direct`, `minimal`, `file`, or `otlp`. Each custom variant runs across the +selected gateway, hook, and startup suites. Gateway results compare it with +both direct provider calls and `relay-minimal`. + +Custom variants increase runtime in proportion to the number of variants. A +custom plugin configuration can also write additional data, so account for its +own storage behavior separately from the default ATOF estimate. + +## Read the Results + +The command prints a terminal summary and writes both of the following +persistent files: + +- `target/benchmark-results/nemo-relay-latency-report.json` contains the + complete, machine-readable result. +- `target/benchmark-results/nemo-relay-latency-report.html` is a self-contained + report with interactive gateway graphs, hook and startup graphs, numeric + tables, metric explanations, and the resolved run environment. + +Open the HTML file directly in a browser. It embeds its styles, scripts, and +result data, so it does not require an Internet connection or a web server. +Use the following command to choose another output directory: + +```bash +just output_dir=/tmp/relay-benchmarks latency-benchmark +``` + +Use `--report` to choose a different HTML path without changing the JSON path: + +```bash +just latency-benchmark --report /tmp/nemo-relay-latency-report.html +``` + +The reports use the following statistics: + +| Metric | Meaning | +| --- | --- | +| Absolute latency | Complete elapsed wall time for one measured path | +| Paired delta | Left path minus its baseline in the same cycle; positive is slower and small negative values can be measurement noise | +| p50 | Median observation | +| p95 and p99 | Tail percentiles that show slower observations | +| Min and max | Fastest and slowest observed values; these are sensitive to outliers | +| Median 95% CI | Deterministic bootstrap uncertainty interval around the median paired delta, not a range containing 95% of observations | + +The exporter-delivery byte and request counts are correctness checks. They +confirm that the ATOF file exporter and OTLP exporter delivered data; they are +not latency metrics. If this validation fails, the command still writes the +JSON and HTML reports, records the messages in `validation_errors`, and then +exits with an error. + +When comparing variants, prefer added milliseconds over percentages. Record +the commit, release build, hardware, operating system, matrix, and sample count +with any shared result. Small loopback baselines can make harmless absolute +differences look large as percentages. + +## Troubleshoot + +Use these checks to diagnose the most common benchmark failures: + +- A loopback bind error means the environment must allow local HTTP listeners. +- An exporter-delivery error means the ATOF file exporter or OTLP exporter + delivered no benchmark events. Rerun a small gateway suite to isolate the + exporter path; Relay startup failures include their captured log output. +- A validation error names the invalid TOML or CLI value. Gateway samples must + be at least as large as every requested concurrency value. +- A normal run removes the large temporary ATOF output and its + `nemo-relay-latency-*` workspace, but a default run still needs at least 30 + GiB of free disk space while it is active. After an abrupt process + termination, remove any stale benchmark directory from the operating + system's temporary location. diff --git a/scripts/latency_benchmark/config/agent-config.toml b/scripts/latency_benchmark/config/agent-config.toml new file mode 100644 index 000000000..13d6e5224 --- /dev/null +++ b/scripts/latency_benchmark/config/agent-config.toml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[agents.codex] +command = "__CODEX_COMMAND__" diff --git a/scripts/latency_benchmark/config/default.toml b/scripts/latency_benchmark/config/default.toml new file mode 100644 index 000000000..c1c4acdeb --- /dev/null +++ b/scripts/latency_benchmark/config/default.toml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# A custom config file may contain only the values it needs to override. +tests = ["gateway", "hooks", "startup"] +providers = ["openai", "anthropic"] +modes = ["buffered", "streaming"] +samples = 100 +hook_samples = 30 +startup_samples = 20 +warmup = 3 +payload_sizes = [4096, 65536, 262144, 1048576, 4194304] +concurrency = [1, 2, 4, 8, 16] +response_bytes = 16384 +stream_chunks = 32 +middleware = [] + +[models] +openai = "gpt-5-codex" +anthropic = "claude-sonnet-4-5" + +[content] +request_fill = "p" +response_fill = "r" diff --git a/scripts/latency_benchmark/config/plugins-file.toml b/scripts/latency_benchmark/config/plugins-file.toml new file mode 100644 index 000000000..533a65759 --- /dev/null +++ b/scripts/latency_benchmark/config/plugins-file.toml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = 1 + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 3 + +[components.config.atof] +enabled = true + +[[components.config.atof.sinks]] +type = "file" +output_directory = "__ATOF_OUTPUT_DIRECTORY__" +filename = "events.jsonl" +mode = "append" diff --git a/scripts/latency_benchmark/config/plugins-minimal.toml b/scripts/latency_benchmark/config/plugins-minimal.toml new file mode 100644 index 000000000..f02170300 --- /dev/null +++ b/scripts/latency_benchmark/config/plugins-minimal.toml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = 1 +components = [] diff --git a/scripts/latency_benchmark/config/plugins-otlp.toml b/scripts/latency_benchmark/config/plugins-otlp.toml new file mode 100644 index 000000000..818d51940 --- /dev/null +++ b/scripts/latency_benchmark/config/plugins-otlp.toml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = 1 + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 3 + +[components.config.opentelemetry] +enabled = true + +[[components.config.opentelemetry.endpoints]] +type = "openinference" +endpoint = "__OTLP_ENDPOINT__" diff --git a/scripts/latency_benchmark/config/plugins-pii-redaction.toml b/scripts/latency_benchmark/config/plugins-pii-redaction.toml new file mode 100644 index 000000000..b965a69ac --- /dev/null +++ b/scripts/latency_benchmark/config/plugins-pii-redaction.toml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = 1 + +[[components]] +kind = "pii_redaction" +enabled = true + +[components.config] +version = 1 + +[[components.config.profiles]] +mode = "builtin" +priority = 80 + +[components.config.profiles.builtin] +action = "redact" +detector = "email" diff --git a/scripts/latency_benchmark/config/relay-config.toml b/scripts/latency_benchmark/config/relay-config.toml new file mode 100644 index 000000000..d51c4fe1e --- /dev/null +++ b/scripts/latency_benchmark/config/relay-config.toml @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/latency_benchmark/data/mock-codex.cmd b/scripts/latency_benchmark/data/mock-codex.cmd new file mode 100644 index 000000000..ab53eef06 --- /dev/null +++ b/scripts/latency_benchmark/data/mock-codex.cmd @@ -0,0 +1,13 @@ +@REM SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +@REM SPDX-License-Identifier: Apache-2.0 +@echo off +if "%1"=="--version" ( + echo codex-cli 0.143.0 + exit /b 0 +) +> "%BENCHMARK_GATEWAY_FILE%.tmp" echo %NEMO_RELAY_GATEWAY_URL% +move /y "%BENCHMARK_GATEWAY_FILE%.tmp" "%BENCHMARK_GATEWAY_FILE%" >nul +:wait +if exist "%BENCHMARK_STOP_FILE%" exit /b 0 +ping 127.0.0.1 -n 2 >nul +goto wait diff --git a/scripts/latency_benchmark/data/mock-codex.sh b/scripts/latency_benchmark/data/mock-codex.sh new file mode 100755 index 000000000..bf13b6350 --- /dev/null +++ b/scripts/latency_benchmark/data/mock-codex.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +if [ "${1:-}" = "--version" ]; then + printf 'codex-cli 0.143.0\n' + exit 0 +fi +gateway_file_tmp="${BENCHMARK_GATEWAY_FILE}.tmp.$$" +printf '%s' "$NEMO_RELAY_GATEWAY_URL" > "$gateway_file_tmp" +mv "$gateway_file_tmp" "$BENCHMARK_GATEWAY_FILE" +while [ ! -f "$BENCHMARK_STOP_FILE" ]; do + sleep 0.1 +done diff --git a/scripts/latency_benchmark/src/__init__.py b/scripts/latency_benchmark/src/__init__.py new file mode 100644 index 000000000..f98864006 --- /dev/null +++ b/scripts/latency_benchmark/src/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Latency benchmark fixture.""" diff --git a/scripts/latency_benchmark/src/__main__.py b/scripts/latency_benchmark/src/__main__.py new file mode 100644 index 000000000..1d7139fd3 --- /dev/null +++ b/scripts/latency_benchmark/src/__main__.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the configurable latency benchmark.""" + +from __future__ import annotations + +import sys + +from .cli import main + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("benchmark interrupted", file=sys.stderr) + raise SystemExit(130) from None diff --git a/scripts/latency_benchmark/src/benchmarks.py b/scripts/latency_benchmark/src/benchmarks.py new file mode 100644 index 000000000..51fb76ef7 --- /dev/null +++ b/scripts/latency_benchmark/src/benchmarks.py @@ -0,0 +1,328 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Gateway, hook, and process-startup benchmark suites.""" + +from __future__ import annotations + +import concurrent.futures +import contextlib +import http.client +import json +import math +import random +import subprocess +import threading +import time +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +from .fixtures import isolated_environment +from .processes import RelayProcess, TransparentRelayProcess +from .protocol import make_request, request_headers, request_path +from .servers import connection_for + + +def percentile(values: Sequence[int | float], fraction: float) -> float: + """Return a linearly interpolated percentile for a non-empty sample.""" + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1 - weight) + ordered[upper] * weight + + +def summarize_ns(values: list[int]) -> dict[str, Any]: + milliseconds = [value / 1_000_000 for value in values] + return { + "samples": len(values), + "p50_ms": round(percentile(milliseconds, 0.50), 6), + "p95_ms": round(percentile(milliseconds, 0.95), 6), + "p99_ms": round(percentile(milliseconds, 0.99), 6), + "min_ms": round(min(milliseconds), 6), + "max_ms": round(max(milliseconds), 6), + } + + +def median_confidence_interval_ns(values: list[int], *, seed: int, resamples: int = 1_000) -> list[float]: + """Return a deterministic bootstrap 95% confidence interval for the median.""" + randomizer = random.Random(seed) + medians = [] + for _ in range(resamples): + sample = [values[randomizer.randrange(len(values))] for _ in values] + medians.append(percentile(sample, 0.50) / 1_000_000) + return [ + round(percentile(medians, 0.025), 6), + round(percentile(medians, 0.975), 6), + ] + + +def perform_request( + connection: http.client.HTTPConnection, + provider: str, + body: bytes, + streaming: bool, +) -> dict[str, int]: + started = time.perf_counter_ns() + connection.request("POST", request_path(provider), body=body, headers=request_headers(provider)) + response = connection.getresponse() + if response.status != 200: + details = response.read().decode(errors="replace") + raise RuntimeError(f"benchmark request failed with HTTP {response.status}: {details}") + if not streaming: + response.read() + return {"total_ns": time.perf_counter_ns() - started} + first_content_ns = 0 + while True: + line = response.readline() + if not line: + break + if not line.startswith(b"data:"): + continue + payload = line[5:].strip() + if not payload or payload == b"[DONE]": + continue + event = json.loads(payload) + if event.get("type") in {"response.output_text.delta", "content_block_delta"}: + first_content_ns = first_content_ns or time.perf_counter_ns() - started + if first_content_ns == 0: + raise RuntimeError("stream ended before a content delta was received") + return { + "first_content_ns": first_content_ns, + "total_ns": time.perf_counter_ns() - started, + } + + +def benchmark_scenario( + urls: dict[str, str], + *, + provider: str, + model: str, + request_fill: str, + streaming: bool, + payload_bytes: int, + samples: int, + warmup: int, + concurrency: int, +) -> dict[str, Any]: + variants = tuple(urls) + body = make_request( + provider, + streaming, + payload_bytes, + model=model, + request_fill=request_fill, + ) + observations: list[dict[str, dict[str, int]]] = [] + observation_lock = threading.Lock() + barrier = threading.Barrier(concurrency) + + def worker(indices: list[int]) -> None: + connections = {name: connection_for(url) for name, url in urls.items()} + try: + for _ in range(warmup): + for name in variants: + perform_request(connections[name], provider, body, streaming) + barrier.wait() + local = [] + for index in indices: + order = list(variants) + shift = index % len(order) + order = order[shift:] + order[:shift] + local.append({name: perform_request(connections[name], provider, body, streaming) for name in order}) + with observation_lock: + observations.extend(local) + except BaseException: + barrier.abort() + raise + finally: + for connection in connections.values(): + connection.close() + + assignments = [list(range(worker_id, samples, concurrency)) for worker_id in range(concurrency)] + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = [executor.submit(worker, indices) for indices in assignments] + for future in futures: + future.result() + + metrics = ("total_ns",) if not streaming else ("first_content_ns", "total_ns") + absolute = { + name: { + metric.removesuffix("_ns"): summarize_ns([cycle[name][metric] for cycle in observations]) + for metric in metrics + } + for name in variants + } + relay_variants = tuple(name for name in variants if name != "direct") + comparison_pairs = {f"{name}_vs_direct": (name, "direct") for name in relay_variants} + for name in relay_variants: + if name == "relay-minimal": + continue + label = name.removeprefix("relay-") + comparison = { + "file": "file_exporter_vs_minimal", + "otlp": "otlp_exporter_vs_minimal", + }.get(label, f"{label}_vs_minimal") + comparison_pairs[comparison] = (name, "relay-minimal") + comparisons = {} + for comparison, (left, right) in comparison_pairs.items(): + comparisons[comparison] = {} + for metric_index, metric in enumerate(metrics): + deltas = [cycle[left][metric] - cycle[right][metric] for cycle in observations] + summary = summarize_ns(deltas) + summary["median_ci95_ms"] = median_confidence_interval_ns( + deltas, + seed=payload_bytes + concurrency * 101 + metric_index * 10_007, + ) + comparisons[comparison][metric.removesuffix("_ns")] = summary + return { + "provider": provider, + "mode": "streaming" if streaming else "buffered", + "payload_bytes": payload_bytes, + "serialized_request_bytes": len(body), + "concurrency": concurrency, + "absolute": absolute, + "comparisons": comparisons, + } + + +def run_subprocess_timed(command: list[str], *, root: Path, input_bytes: bytes | None = None) -> int: + started = time.perf_counter_ns() + result = subprocess.run( + command, + cwd=root, + env=isolated_environment(root), + input=input_bytes, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + elapsed = time.perf_counter_ns() - started + if result.returncode != 0: + details = result.stderr.decode(errors="replace").strip() + raise RuntimeError(f"command exited with status {result.returncode}: {command}\n{details}") + return elapsed + + +def benchmark_hooks( + binary: Path, + root: Path, + provider_url: str, + configs: dict[str, Path], + *, + samples: int, + warmup: int, +) -> dict[str, Any]: + relay_variants = tuple(configs) + measurements = {"process_baseline": []} + for agent in ("codex", "claude"): + for variant in relay_variants: + measurements[f"{agent}_{variant.removeprefix('relay-')}"] = [] + + with contextlib.ExitStack() as stack: + relay_urls = { + variant: stack.enter_context( + TransparentRelayProcess( + binary, + root, + provider_url, + configs[variant], + f"transparent-{variant}", + ) + ).url + for variant in relay_variants + } + + def hook(agent: str, variant: str, index: int) -> int: + variant_name = variant.removeprefix("relay-") + event_name = "sessionStart" if agent == "codex" else "SessionStart" + payload = json.dumps( + { + "session_id": f"benchmark-{agent}-{variant_name}-{index}", + "hook_event_name": event_name, + } + ).encode() + return run_subprocess_timed( + [ + str(binary), + "hook-forward", + agent, + "--gateway-url", + relay_urls[variant], + "--transparent-run", + "--fail-closed", + ], + root=root, + input_bytes=payload, + ) + + for index in range(-warmup, samples): + cycle = {"process_baseline": lambda: run_subprocess_timed([str(binary), "--version"], root=root)} + for agent in ("codex", "claude"): + for variant in relay_variants: + name = f"{agent}_{variant.removeprefix('relay-')}" + cycle[name] = lambda agent=agent, variant=variant: hook(agent, variant, index) + names = list(cycle) + random.Random(index).shuffle(names) + values = {name: cycle[name]() for name in names} + if index >= 0: + for name, value in values.items(): + measurements[name].append(value) + + baseline = measurements["process_baseline"] + result: dict[str, Any] = { + "absolute": {name: summarize_ns(values) for name, values in measurements.items()}, + "comparisons": {}, + } + for agent in ("codex", "claude"): + for variant in relay_variants: + name = f"{agent}_{variant.removeprefix('relay-')}" + deltas = [left - right for left, right in zip(measurements[name], baseline)] + summary = summarize_ns(deltas) + summary["median_ci95_ms"] = median_confidence_interval_ns(deltas, seed=len(name) * 1_009) + result["comparisons"][f"{name}_vs_process_baseline"] = summary + return result + + +def benchmark_startup( + binary: Path, + root: Path, + provider_url: str, + configs: dict[str, Path], + *, + samples: int, + warmup: int, +) -> dict[str, Any]: + relay_variants = tuple(configs) + measurements = {"process_baseline": [], **{variant: [] for variant in relay_variants}} + for index in range(-warmup, samples): + baseline = run_subprocess_timed([str(binary), "--version"], root=root) + cycle = {} + for variant in relay_variants: + with RelayProcess( + binary, + root, + provider_url, + configs[variant], + f"startup-{variant}-{index}", + ) as process: + cycle[variant] = process.startup_ns + if index >= 0: + measurements["process_baseline"].append(baseline) + for variant, value in cycle.items(): + measurements[variant].append(value) + result: dict[str, Any] = { + "absolute": {name: summarize_ns(values) for name, values in measurements.items()}, + "comparisons": {}, + } + baseline = measurements["process_baseline"] + for variant in relay_variants: + deltas = [left - right for left, right in zip(measurements[variant], baseline)] + summary = summarize_ns(deltas) + summary["median_ci95_ms"] = median_confidence_interval_ns(deltas, seed=len(variant) * 2_003) + result["comparisons"][f"{variant}_readiness_vs_process_baseline"] = summary + return result diff --git a/scripts/latency_benchmark/src/cli.py b/scripts/latency_benchmark/src/cli.py new file mode 100644 index 000000000..0c6e47a3d --- /dev/null +++ b/scripts/latency_benchmark/src/cli.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Command-line orchestration for the latency benchmark.""" + +from __future__ import annotations + +import contextlib +import json +import tempfile +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from .benchmarks import benchmark_hooks, benchmark_scenario, benchmark_startup +from .config import BenchmarkConfig, parse_args +from .fixtures import write_plugin_configs, write_relay_config +from .html_report import write_html_report +from .processes import RelayProcess +from .reporting import environment_record, print_results +from .servers import OtlpHandler, ProviderHandler, local_server + + +def _benchmark_gateway( + binary: Path, + root: Path, + provider_url: str, + configs: dict[str, Path], + config: BenchmarkConfig, + report_complete: Callable[[str], None], +) -> list[dict[str, Any]]: + with contextlib.ExitStack() as stack: + relays = { + variant: stack.enter_context(RelayProcess(binary, root, provider_url, configs[variant], variant)) + for variant in configs + } + urls = {"direct": provider_url} | {variant: relay.url for variant, relay in relays.items()} + scenarios = [] + for provider in config.providers: + model = config.openai_model if provider == "openai" else config.anthropic_model + for mode in config.modes: + streaming = mode == "streaming" + for payload_bytes in config.payload_sizes: + for concurrency in config.concurrency: + print( + f"Benchmarking {provider} {mode}, payload={payload_bytes}, concurrency={concurrency}...", + flush=True, + ) + scenarios.append( + benchmark_scenario( + urls, + provider=provider, + model=model, + request_fill=config.request_fill, + streaming=streaming, + payload_bytes=payload_bytes, + samples=config.samples, + warmup=config.warmup, + concurrency=concurrency, + ) + ) + report_complete( + f"gateway {provider} {mode}, payload={payload_bytes}, concurrency={concurrency}" + ) + return scenarios + + +def _benchmark_test_count(config: BenchmarkConfig) -> int: + gateway_tests = 0 + if "gateway" in config.tests: + gateway_tests = len(config.providers) * len(config.modes) * len(config.payload_sizes) * len(config.concurrency) + return gateway_tests + int("hooks" in config.tests) + int("startup" in config.tests) + + +def _exporter_delivery(root: Path) -> tuple[dict[str, int], list[str]]: + atof_path = root / "atof" / "events.jsonl" + atof_bytes = atof_path.stat().st_size if atof_path.is_file() else 0 + delivery = { + "atof_bytes": atof_bytes, + "otlp_requests": OtlpHandler.request_count, + } + errors = [] + if atof_bytes == 0: + errors.append("local ATOF exporter did not write benchmark events") + if OtlpHandler.request_count == 0: + errors.append("local OTLP receiver did not receive benchmark exports") + return delivery, errors + + +def run_benchmarks(binary: Path, config: BenchmarkConfig) -> dict[str, Any]: + """Run selected suites and return the versioned result document.""" + OtlpHandler.reset() + results: dict[str, Any] = { + "schema_version": 2, + "environment": environment_record(binary), + "parameters": config.parameters(), + } + completed_tests = 0 + total_tests = _benchmark_test_count(config) + + def report_complete(label: str) -> None: + nonlocal completed_tests + completed_tests += 1 + print(f"[{completed_tests}/{total_tests}] Completed {label}", flush=True) + + with tempfile.TemporaryDirectory(prefix="nemo-relay-latency-") as temporary: + root = Path(temporary) + write_relay_config(root) + with ( + local_server( + ProviderHandler, + response_bytes=config.response_bytes, + stream_chunks=config.stream_chunks, + response_fill=config.response_fill, + openai_model=config.openai_model, + anthropic_model=config.anthropic_model, + ) as provider_url, + local_server(OtlpHandler) as otlp_url, + ): + configs = write_plugin_configs(root, otlp_url, config.middleware) + if "gateway" in config.tests: + results["gateway"] = _benchmark_gateway( + binary, + root, + provider_url, + configs, + config, + report_complete, + ) + if "hooks" in config.tests: + results["hooks"] = benchmark_hooks( + binary, + root, + provider_url, + configs, + samples=config.hook_samples, + warmup=config.warmup, + ) + report_complete("hooks suite") + if "startup" in config.tests: + results["startup"] = benchmark_startup( + binary, + root, + provider_url, + configs, + samples=config.startup_samples, + warmup=config.warmup, + ) + report_complete("startup suite") + + if {"gateway", "hooks"}.intersection(config.tests): + results["exporter_delivery"], validation_errors = _exporter_delivery(root) + if validation_errors: + results["validation_errors"] = validation_errors + return results + + +def main(argv: list[str] | None = None) -> None: + options = parse_args(argv) + results = run_benchmarks(options.relay_bin, options.config) + options.output.parent.mkdir(parents=True, exist_ok=True) + options.output.write_text(json.dumps(results, indent=2, sort_keys=True) + "\n", encoding="utf-8") + write_html_report(results, options.report) + print_results(results) + print(f"\nJSON results: {options.output}") + print(f"HTML report: {options.report}") + validation_errors = [str(error) for error in results.get("validation_errors", [])] + if validation_errors: + raise RuntimeError(f"benchmark validation failed: {'; '.join(validation_errors)}") diff --git a/scripts/latency_benchmark/src/config.py b/scripts/latency_benchmark/src/config.py new file mode 100644 index 000000000..f6cf1e502 --- /dev/null +++ b/scripts/latency_benchmark/src/config.py @@ -0,0 +1,483 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Configuration loading and command-line overrides for the benchmark.""" + +from __future__ import annotations + +import argparse +import re +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +BENCHMARK_ROOT = Path(__file__).resolve().parent.parent +DATA_ROOT = BENCHMARK_ROOT / "data" +CONFIG_ROOT = BENCHMARK_ROOT / "config" +DEFAULT_CONFIG_PATH = CONFIG_ROOT / "default.toml" + +AVAILABLE_TESTS = ("gateway", "hooks", "startup") +AVAILABLE_PROVIDERS = ("openai", "anthropic") +AVAILABLE_MODES = ("buffered", "streaming") +RESERVED_MIDDLEWARE_NAMES = {"direct", "minimal", "file", "otlp"} +MIDDLEWARE_NAME_PATTERN = re.compile(r"[a-z0-9][a-z0-9-]*") +CONFIG_KEYS = { + "tests", + "providers", + "modes", + "samples", + "hook_samples", + "startup_samples", + "warmup", + "payload_sizes", + "concurrency", + "response_bytes", + "stream_chunks", + "models", + "content", + "middleware", +} +TABLE_KEYS = { + "models": {"openai", "anthropic"}, + "content": {"request_fill", "response_fill"}, +} +MIDDLEWARE_KEYS = {"name", "plugin_config"} + + +@dataclass(frozen=True) +class MiddlewareVariant: + """One opt-in Relay plugin configuration benchmarked as an extra variant.""" + + name: str + plugin_config: Path + + @property + def relay_name(self) -> str: + """Return the variant key stored in benchmark results.""" + return f"relay-{self.name}" + + def parameters(self) -> dict[str, str]: + """Return the serializable configuration recorded with results.""" + return {"name": self.name, "plugin_config": str(self.plugin_config)} + + +@dataclass(frozen=True) +class BenchmarkConfig: + """Validated benchmark matrix and sample settings.""" + + tests: tuple[str, ...] + providers: tuple[str, ...] + modes: tuple[str, ...] + samples: int + hook_samples: int + startup_samples: int + warmup: int + payload_sizes: tuple[int, ...] + concurrency: tuple[int, ...] + response_bytes: int + stream_chunks: int + openai_model: str + anthropic_model: str + request_fill: str + response_fill: str + middleware: tuple[MiddlewareVariant, ...] + + def parameters(self) -> dict[str, Any]: + """Return the configuration embedded in the JSON result.""" + return { + "tests": self.tests, + "providers": self.providers, + "modes": self.modes, + "samples": self.samples, + "hook_samples": self.hook_samples, + "startup_samples": self.startup_samples, + "warmup": self.warmup, + "payload_sizes": self.payload_sizes, + "concurrency": self.concurrency, + "response_bytes": self.response_bytes, + "stream_chunks": self.stream_chunks, + "models": { + "openai": self.openai_model, + "anthropic": self.anthropic_model, + }, + "content": { + "request_fill": self.request_fill, + "response_fill": self.response_fill, + }, + "middleware": [variant.parameters() for variant in self.middleware], + } + + +@dataclass(frozen=True) +class CliOptions: + """File paths and resolved benchmark configuration.""" + + relay_bin: Path + output: Path + report: Path + config: BenchmarkConfig + + +def _read_toml(path: Path) -> dict[str, Any]: + with path.open("rb") as config_file: + return tomllib.load(config_file) + + +def _merge_config(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + unknown = set(override) - CONFIG_KEYS + if unknown: + raise ValueError(f"unknown config key(s): {', '.join(sorted(unknown))}") + merged = dict(base) + for key, value in override.items(): + if key in {"models", "content"}: + if not isinstance(value, dict): + raise ValueError(f"[{key}] must be a TOML table") + unknown_nested = set(value) - TABLE_KEYS[key] + if unknown_nested: + raise ValueError(f"unknown [{key}] key(s): {', '.join(sorted(unknown_nested))}") + merged[key] = dict(merged.get(key, {})) | value + else: + merged[key] = value + return merged + + +def _string_tuple(value: Any, name: str, available: tuple[str, ...]) -> tuple[str, ...]: + if not isinstance(value, list) or not value or not all(isinstance(item, str) for item in value): + raise ValueError(f"{name} must be a non-empty list of strings") + values = tuple(value) + invalid = sorted(set(values) - set(available)) + if invalid: + raise ValueError(f"unknown {name}: {', '.join(invalid)}; choose from {', '.join(available)}") + if len(set(values)) != len(values): + raise ValueError(f"{name} must not contain duplicates") + return values + + +def _positive_int(value: Any, name: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + return value + + +def _positive_int_tuple(value: Any, name: str) -> tuple[int, ...]: + if not isinstance(value, list) or not value: + raise ValueError(f"{name} must be a non-empty list of positive integers") + values = tuple(_positive_int(item, name) for item in value) + if len(set(values)) != len(values): + raise ValueError(f"{name} must not contain duplicates") + return values + + +def _nonempty_string(value: Any, name: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{name} must be a non-empty string") + return value + + +def _middleware_variants(value: Any, base_directory: Path) -> tuple[MiddlewareVariant, ...]: + if not isinstance(value, list): + raise ValueError("middleware must be a list of tables") + variants = [] + names = set() + for index, item in enumerate(value): + if not isinstance(item, dict): + raise ValueError(f"middleware[{index}] must be a TOML table") + item_mapping = cast(dict[str, Any], item) + unknown = set(item_mapping) - MIDDLEWARE_KEYS + if unknown: + raise ValueError(f"unknown middleware[{index}] key(s): {', '.join(sorted(unknown))}") + name = _nonempty_string(item_mapping.get("name"), f"middleware[{index}].name") + if MIDDLEWARE_NAME_PATTERN.fullmatch(name) is None: + raise ValueError(f"middleware[{index}].name must contain only lowercase letters, digits, and hyphens") + if name in RESERVED_MIDDLEWARE_NAMES: + raise ValueError(f"middleware name is reserved by a default variant: {name}") + if name in names: + raise ValueError(f"middleware names must not contain duplicates: {name}") + names.add(name) + raw_path = _nonempty_string(item_mapping.get("plugin_config"), f"middleware[{index}].plugin_config") + plugin_config = Path(raw_path) + if not plugin_config.is_absolute(): + plugin_config = base_directory / plugin_config + plugin_config = plugin_config.resolve() + if not plugin_config.is_file(): + raise ValueError(f"middleware plugin config does not exist: {plugin_config}") + variants.append(MiddlewareVariant(name=name, plugin_config=plugin_config)) + return tuple(variants) + + +def _config_from_mapping(value: dict[str, Any], *, middleware_base: Path) -> BenchmarkConfig: + unknown = set(value) - CONFIG_KEYS + if unknown: + raise ValueError(f"unknown config key(s): {', '.join(sorted(unknown))}") + models = value.get("models") + content = value.get("content") + if not isinstance(models, dict) or not isinstance(content, dict): + raise ValueError("config must contain [models] and [content] tables") + warmup = value.get("warmup") + if not isinstance(warmup, int) or isinstance(warmup, bool) or warmup < 0: + raise ValueError("warmup must be a non-negative integer") + request_fill = _nonempty_string(content.get("request_fill"), "content.request_fill") + response_fill = _nonempty_string(content.get("response_fill"), "content.response_fill") + if len(request_fill) != 1 or len(response_fill) != 1 or not request_fill.isascii() or not response_fill.isascii(): + raise ValueError("content fill values must each contain exactly one ASCII character") + config = BenchmarkConfig( + tests=_string_tuple(value.get("tests"), "tests", AVAILABLE_TESTS), + providers=_string_tuple(value.get("providers"), "providers", AVAILABLE_PROVIDERS), + modes=_string_tuple(value.get("modes"), "modes", AVAILABLE_MODES), + samples=_positive_int(value.get("samples"), "samples"), + hook_samples=_positive_int(value.get("hook_samples"), "hook_samples"), + startup_samples=_positive_int(value.get("startup_samples"), "startup_samples"), + warmup=warmup, + payload_sizes=_positive_int_tuple(value.get("payload_sizes"), "payload_sizes"), + concurrency=_positive_int_tuple(value.get("concurrency"), "concurrency"), + response_bytes=_positive_int(value.get("response_bytes"), "response_bytes"), + stream_chunks=_positive_int(value.get("stream_chunks"), "stream_chunks"), + openai_model=_nonempty_string(models.get("openai"), "models.openai"), + anthropic_model=_nonempty_string(models.get("anthropic"), "models.anthropic"), + request_fill=request_fill, + response_fill=response_fill, + middleware=_middleware_variants(value.get("middleware"), middleware_base), + ) + if "gateway" in config.tests and max(config.concurrency) > config.samples: + raise ValueError("samples must be greater than or equal to every gateway concurrency value") + return config + + +def load_config(path: Path) -> BenchmarkConfig: + """Load the defaults and overlay a possibly partial user config.""" + defaults = _read_toml(DEFAULT_CONFIG_PATH) + resolved_path = path.resolve() + middleware_base = DEFAULT_CONFIG_PATH.parent + if resolved_path != DEFAULT_CONFIG_PATH.resolve(): + override = _read_toml(resolved_path) + defaults = _merge_config(defaults, override) + if "middleware" in override: + middleware_base = resolved_path.parent + return _config_from_mapping(defaults, middleware_base=middleware_base) + + +def _csv_strings(value: str) -> tuple[str, ...]: + values = tuple(item.strip() for item in value.split(",") if item.strip()) + if not values: + raise argparse.ArgumentTypeError("expected a comma-separated list") + return values + + +def _csv_ints(value: str) -> tuple[int, ...]: + try: + values = tuple(int(item.strip()) for item in value.split(",") if item.strip()) + except ValueError as error: + raise argparse.ArgumentTypeError("expected comma-separated positive integers") from error + if not values or any(item <= 0 for item in values): + raise argparse.ArgumentTypeError("expected comma-separated positive integers") + return values + + +def _arg_positive_int(value: str) -> int: + try: + return _positive_int(int(value), "value") + except ValueError as error: + raise argparse.ArgumentTypeError("expected a positive integer") from error + + +def _arg_nonnegative_int(value: str) -> int: + try: + number = int(value) + except ValueError as error: + raise argparse.ArgumentTypeError("expected a non-negative integer") from error + if number < 0: + raise argparse.ArgumentTypeError("expected a non-negative integer") + return number + + +def _middleware_arg(value: str) -> dict[str, str]: + name, separator, plugin_config = value.partition("=") + if not separator or not name or not plugin_config: + raise argparse.ArgumentTypeError("expected NAME=PATH") + return {"name": name, "plugin_config": plugin_config} + + +def build_parser() -> argparse.ArgumentParser: + """Build the public command-line interface for the benchmark.""" + parser = argparse.ArgumentParser( + prog="just latency-benchmark", + usage="just latency-benchmark [options]", + description=( + "Measure the local latency that NeMo Relay adds to coding-agent gateway, hook, and startup paths." + ), + epilog=( + "Configuration precedence: built-in defaults, then --config, then CLI overrides.\n" + "Example smoke test: just latency-benchmark --tests gateway " + "--providers openai --modes buffered --payload-sizes 4096 " + "--concurrency 1 --samples 5 --warmup 1" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + outputs = parser.add_argument_group("output and configuration") + outputs.add_argument( + "--relay-bin", + type=Path, + required=True, + metavar="PATH", + help="Relay CLI executable to benchmark (the just recipe supplies the release binary).", + ) + outputs.add_argument( + "--output", + type=Path, + required=True, + metavar="PATH", + help="write machine-readable JSON results to PATH (the just recipe supplies a default).", + ) + outputs.add_argument( + "--report", + type=Path, + metavar="PATH", + help="write the self-contained HTML report to PATH (default: the JSON path with an .html suffix).", + ) + outputs.add_argument( + "--config", + type=Path, + default=DEFAULT_CONFIG_PATH, + metavar="PATH", + help="overlay benchmark settings from a TOML file (default: bundled config/default.toml).", + ) + + matrix = parser.add_argument_group("suite and matrix selection") + matrix.add_argument( + "--tests", + type=_csv_strings, + metavar="LIST", + help=f"run comma-separated suites; choices: {','.join(AVAILABLE_TESTS)}.", + ) + matrix.add_argument( + "--providers", + type=_csv_strings, + metavar="LIST", + help=f"benchmark comma-separated mock provider protocols; choices: {','.join(AVAILABLE_PROVIDERS)}.", + ) + matrix.add_argument( + "--modes", + type=_csv_strings, + metavar="LIST", + help=f"benchmark comma-separated response modes; choices: {','.join(AVAILABLE_MODES)}.", + ) + matrix.add_argument( + "--payload-sizes", + type=_csv_ints, + metavar="BYTES", + help="benchmark comma-separated request-content sizes in bytes.", + ) + matrix.add_argument( + "--concurrency", + type=_csv_ints, + metavar="COUNTS", + help="benchmark comma-separated in-flight request counts; each value must not exceed --samples.", + ) + matrix.add_argument( + "--middleware", + action="append", + type=_middleware_arg, + metavar="NAME=PATH", + help=( + "add a named Relay middleware plugin config as an extra variant; repeat for multiple variants " + "and use this option to replace middleware entries from --config." + ), + ) + + sampling = parser.add_argument_group("sampling") + sampling.add_argument( + "--samples", + type=_arg_positive_int, + metavar="COUNT", + help="record COUNT gateway measurement cycles per scenario.", + ) + sampling.add_argument( + "--hook-samples", + type=_arg_positive_int, + metavar="COUNT", + help="record COUNT subprocess measurements for every hook path.", + ) + sampling.add_argument( + "--startup-samples", + type=_arg_positive_int, + metavar="COUNT", + help="record COUNT cold-start measurements for every Relay variant.", + ) + sampling.add_argument( + "--warmup", + type=_arg_nonnegative_int, + metavar="COUNT", + help="run COUNT unrecorded warmup cycles before each measured workload.", + ) + + response = parser.add_argument_group("mock provider response") + response.add_argument( + "--response-bytes", + type=_arg_positive_int, + metavar="BYTES", + help="return approximately BYTES of deterministic content from the mock provider.", + ) + response.add_argument( + "--stream-chunks", + type=_arg_positive_int, + metavar="COUNT", + help="split streaming mock responses into COUNT content-delta events.", + ) + return parser + + +def parse_args(argv: list[str] | None = None) -> CliOptions: + """Parse CLI options, applying them after values from the config file.""" + parser = build_parser() + args = parser.parse_args(argv) + + try: + config = load_config(args.config) + values = config.parameters() + values.update( + { + name: getattr(args, name) + for name in ( + "tests", + "providers", + "modes", + "samples", + "hook_samples", + "startup_samples", + "warmup", + "payload_sizes", + "concurrency", + "response_bytes", + "stream_chunks", + "middleware", + ) + if getattr(args, name) is not None + } + ) + config = _config_from_mapping( + { + **values, + "tests": list(values["tests"]), + "providers": list(values["providers"]), + "modes": list(values["modes"]), + "payload_sizes": list(values["payload_sizes"]), + "concurrency": list(values["concurrency"]), + }, + middleware_base=Path.cwd(), + ) + except (OSError, tomllib.TOMLDecodeError, ValueError) as error: + parser.error(f"invalid benchmark config: {error}") + + relay_bin = args.relay_bin.resolve() + if not relay_bin.is_file(): + parser.error(f"Relay binary does not exist: {relay_bin}") + output = args.output.resolve() + report = args.report.resolve() if args.report is not None else output.with_suffix(".html") + return CliOptions( + relay_bin=relay_bin, + output=output, + report=report, + config=config, + ) diff --git a/scripts/latency_benchmark/src/fixtures.py b/scripts/latency_benchmark/src/fixtures.py new file mode 100644 index 000000000..de8847a4f --- /dev/null +++ b/scripts/latency_benchmark/src/fixtures.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Materialize static benchmark data in an isolated workspace.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +from .config import CONFIG_ROOT, DATA_ROOT, MiddlewareVariant + +_BLOCKED_ENVIRONMENT_PREFIXES = ( + "ANTHROPIC_", + "NEMO_RELAY", + "OPENAI_", + "OTEL_", + "RUST_LOG", +) +_PROXY_ENVIRONMENT_NAMES = { + "ALL_PROXY", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", +} + + +def _read_data(name: str) -> str: + return (DATA_ROOT / name).read_text(encoding="utf-8") + + +def _read_config(name: str) -> str: + return (CONFIG_ROOT / name).read_text(encoding="utf-8") + + +def _render_config(name: str, replacements: dict[str, str]) -> str: + rendered = _read_config(name) + for marker, value in replacements.items(): + if marker not in rendered: + raise RuntimeError(f"static fixture {name} is missing marker {marker}") + rendered = rendered.replace(marker, value) + return rendered + + +def toml_string(value: str | Path) -> str: + """Encode a string using TOML-compatible JSON quoting.""" + return json.dumps(str(value)) + + +def write_relay_config(root: Path) -> Path: + path = root / "config.toml" + path.write_text(_read_config("relay-config.toml"), encoding="utf-8") + return path + + +def write_plugin_configs( + root: Path, + otlp_url: str, + middleware: tuple[MiddlewareVariant, ...] = (), +) -> dict[str, Path]: + """Write default plugin configs and add opt-in middleware variants.""" + paths = { + "relay-minimal": root / "plugins-minimal.toml", + "relay-file": root / "plugins-file.toml", + "relay-otlp": root / "plugins-otlp.toml", + } + paths["relay-minimal"].write_text(_read_config("plugins-minimal.toml"), encoding="utf-8") + + atof_dir = root / "atof" + atof_dir.mkdir() + paths["relay-file"].write_text( + _render_config("plugins-file.toml", {'"__ATOF_OUTPUT_DIRECTORY__"': toml_string(atof_dir)}), + encoding="utf-8", + ) + paths["relay-otlp"].write_text( + _render_config("plugins-otlp.toml", {'"__OTLP_ENDPOINT__"': toml_string(f"{otlp_url}/v1/traces")}), + encoding="utf-8", + ) + paths.update((variant.relay_name, variant.plugin_config) for variant in middleware) + return paths + + +def write_mock_codex(root: Path) -> Path: + """Copy the platform-specific static mock Codex client into the workspace.""" + source_name = "mock-codex.cmd" if os.name == "nt" else "mock-codex.sh" + target_name = "mock-codex.cmd" if os.name == "nt" else "mock-codex" + path = root / target_name + newline = "\r\n" if os.name == "nt" else "\n" + path.write_text(_read_data(source_name), encoding="utf-8", newline=newline) + if os.name != "nt": + path.chmod(0o755) + return path + + +def write_agent_config(root: Path, name: str, mock_codex: Path) -> Path: + path = root / f"{name}-config.toml" + path.write_text( + _render_config("agent-config.toml", {'"__CODEX_COMMAND__"': toml_string(mock_codex)}), + encoding="utf-8", + ) + return path + + +def isolated_environment(root: Path) -> dict[str, str]: + """Return an environment that cannot discover the developer's Relay state.""" + environment = os.environ.copy() + for name in tuple(environment): + normalized = name.upper() + if normalized.startswith(_BLOCKED_ENVIRONMENT_PREFIXES) or normalized in _PROXY_ENVIRONMENT_NAMES: + environment.pop(name) + environment.update( + { + "HOME": str(root / "home"), + "NO_PROXY": "127.0.0.1,localhost", + "XDG_CONFIG_HOME": str(root / "xdg-config"), + "XDG_DATA_HOME": str(root / "xdg-data"), + "NO_COLOR": "1", + "no_proxy": "127.0.0.1,localhost", + } + ) + for directory in ("home", "xdg-config", "xdg-data"): + (root / directory).mkdir(exist_ok=True) + return environment diff --git a/scripts/latency_benchmark/src/html_report.py b/scripts/latency_benchmark/src/html_report.py new file mode 100644 index 000000000..4da1c82d0 --- /dev/null +++ b/scripts/latency_benchmark/src/html_report.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Render a self-contained HTML report for benchmark results.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +REPORT_ROOT = Path(__file__).resolve().parent / "report" +TEMPLATE_PATH = REPORT_ROOT / "template.html" +STYLES_PATH = REPORT_ROOT / "styles.css" +SCRIPT_PATH = REPORT_ROOT / "report.js" + + +def _embedded_json(results: dict[str, Any]) -> str: + """Serialize JSON so it cannot terminate its script element.""" + return ( + json.dumps(results, separators=(",", ":"), sort_keys=True) + .replace("&", "\\u0026") + .replace("<", "\\u003c") + .replace(">", "\\u003e") + ) + + +def render_html_report(results: dict[str, Any]) -> str: + """Return a portable HTML report with embedded styles, data, and scripts.""" + template = TEMPLATE_PATH.read_text(encoding="utf-8") + replacements = { + "__BENCHMARK_STYLES__": STYLES_PATH.read_text(encoding="utf-8"), + "__BENCHMARK_DATA__": _embedded_json(results), + "__BENCHMARK_SCRIPT__": SCRIPT_PATH.read_text(encoding="utf-8"), + } + for marker, value in replacements.items(): + template = template.replace(marker, value) + return template + + +def write_html_report(results: dict[str, Any], path: Path) -> None: + """Write a portable HTML report to ``path``.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(render_html_report(results), encoding="utf-8") diff --git a/scripts/latency_benchmark/src/processes.py b/scripts/latency_benchmark/src/processes.py new file mode 100644 index 000000000..de109b838 --- /dev/null +++ b/scripts/latency_benchmark/src/processes.py @@ -0,0 +1,211 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Relay subprocess lifecycle helpers for latency measurements.""" + +from __future__ import annotations + +import socket +import subprocess +import time +import uuid +from pathlib import Path +from typing import IO + +from .fixtures import isolated_environment, write_agent_config, write_mock_codex +from .servers import connection_for + + +class RelayProcess: + """Run a normal Relay gateway and record readiness latency.""" + + def __init__( + self, + binary: Path, + root: Path, + provider_url: str, + plugin_config: Path, + name: str, + ) -> None: + self.binary = binary + self.root = root + self.provider_url = provider_url + self.plugin_config = plugin_config + self.name = name + self.process: subprocess.Popen[bytes] | None = None + self.url = "" + self.startup_ns = 0 + self.log_handle: IO[bytes] | None = None + + def start(self) -> None: + log_path = self.root / f"{self.name}.log" + self.log_handle = log_path.open("ab") + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + self.url = f"http://127.0.0.1:{port}" + command = [ + str(self.binary), + "--bind", + f"127.0.0.1:{port}", + "--openai-base-url", + f"{self.provider_url}/v1", + "--anthropic-base-url", + self.provider_url, + "--config", + str(self.root / "config.toml"), + "--plugin-config-path", + str(self.plugin_config), + ] + started = time.perf_counter_ns() + self.process = subprocess.Popen( + command, + cwd=self.root, + env=isolated_environment(self.root), + stdout=subprocess.DEVNULL, + stderr=self.log_handle, + ) + deadline = time.monotonic() + 15 + while not self._healthy(): + if self.process.poll() is not None: + self._raise_start_error(log_path) + if time.monotonic() >= deadline: + self.stop() + raise RuntimeError(f"timed out waiting for {self.name} readiness") + time.sleep(0.0005) + self.startup_ns = time.perf_counter_ns() - started + + def _healthy(self) -> bool: + connection = connection_for(self.url) + connection.timeout = 0.1 + try: + connection.request("GET", "/healthz") + response = connection.getresponse() + response.read() + return response.status == 200 + except OSError: + return False + finally: + connection.close() + + def stop(self) -> None: + if self.process is not None and self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + if self.log_handle is not None: + self.log_handle.close() + self.log_handle = None + + def _raise_start_error(self, log_path: Path) -> None: + if self.log_handle is not None: + self.log_handle.close() + self.log_handle = None + details = log_path.read_text(encoding="utf-8", errors="replace") + raise RuntimeError(f"{self.name} exited during startup:\n{details}") + + def __enter__(self) -> RelayProcess: + self.start() + return self + + def __exit__(self, *_: object) -> None: + self.stop() + + +class TransparentRelayProcess: + """Run Relay's transparent coding-agent path for hook benchmarks.""" + + def __init__( + self, + binary: Path, + root: Path, + provider_url: str, + plugin_config: Path, + name: str, + ) -> None: + self.binary = binary + self.root = root + self.provider_url = provider_url + self.plugin_config = plugin_config + self.name = name + self.process: subprocess.Popen[bytes] | None = None + self.url = "" + self.log_handle: IO[bytes] | None = None + self.stop_file = root / f"{name}-{uuid.uuid4().hex}.stop" + + def start(self) -> None: + gateway_file = self.root / f"{self.name}-{uuid.uuid4().hex}.gateway" + config = write_agent_config(self.root, self.name, write_mock_codex(self.root)) + log_path = self.root / f"{self.name}.log" + self.log_handle = log_path.open("ab") + environment = isolated_environment(self.root) + environment["BENCHMARK_GATEWAY_FILE"] = str(gateway_file) + environment["BENCHMARK_STOP_FILE"] = str(self.stop_file) + command = [ + str(self.binary), + "run", + "--agent", + "codex", + "--config", + str(config), + "--openai-base-url", + f"{self.provider_url}/v1", + "--anthropic-base-url", + self.provider_url, + "--plugin-config-path", + str(self.plugin_config), + ] + self.process = subprocess.Popen( + command, + cwd=self.root, + env=environment, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=self.log_handle, + ) + deadline = time.monotonic() + 15 + while True: + if gateway_file.is_file(): + gateway_url = gateway_file.read_text(encoding="utf-8").strip() + if gateway_url: + self.url = gateway_url + break + if self.process.poll() is not None: + self._raise_start_error(log_path) + if time.monotonic() >= deadline: + self.stop() + raise RuntimeError(f"timed out waiting for {self.name} transparent gateway") + time.sleep(0.001) + + def stop(self) -> None: + self.stop_file.touch() + if self.process is not None and self.process.poll() is None: + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + if self.log_handle is not None: + self.log_handle.close() + self.log_handle = None + + def _raise_start_error(self, log_path: Path) -> None: + if self.log_handle is not None: + self.log_handle.close() + self.log_handle = None + details = log_path.read_text(encoding="utf-8", errors="replace") + raise RuntimeError(f"{self.name} exited during startup:\n{details}") + + def __enter__(self) -> TransparentRelayProcess: + self.start() + return self + + def __exit__(self, *_: object) -> None: + self.stop() diff --git a/scripts/latency_benchmark/src/protocol.py b/scripts/latency_benchmark/src/protocol.py new file mode 100644 index 000000000..6a08ca47e --- /dev/null +++ b/scripts/latency_benchmark/src/protocol.py @@ -0,0 +1,229 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Deterministic OpenAI and Anthropic request/response fixtures.""" + +from __future__ import annotations + +import json +import math +import time +import uuid +from typing import Any + + +def split_text(text: str, chunks: int) -> list[str]: + chunk_size = max(1, math.ceil(len(text) / chunks)) + return [text[index : index + chunk_size] for index in range(0, len(text), chunk_size)] + + +def response_document(model: str, text: str) -> dict[str, Any]: + response_id = f"resp_{uuid.uuid4().hex}" + item_id = f"msg_{uuid.uuid4().hex}" + created = int(time.time()) + item = { + "id": item_id, + "type": "message", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": text, + "annotations": [], + "logprobs": [], + } + ], + } + return { + "id": response_id, + "object": "response", + "created_at": created, + "completed_at": created, + "status": "completed", + "background": False, + "error": None, + "incomplete_details": None, + "instructions": None, + "max_output_tokens": None, + "max_tool_calls": None, + "model": model, + "output": [item], + "parallel_tool_calls": True, + "previous_response_id": None, + "prompt_cache_key": None, + "reasoning": {"effort": "medium", "summary": None}, + "safety_identifier": None, + "service_tier": "default", + "store": False, + "temperature": None, + "text": {"format": {"type": "text"}, "verbosity": "medium"}, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": None, + "truncation": "disabled", + "usage": { + "input_tokens": 1, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 1, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 2, + }, + "user": None, + "metadata": {}, + } + + +def openai_events(model: str, text: str, chunks: int) -> list[dict[str, Any]]: + response = response_document(model, text) + response_id = response["id"] + item = response["output"][0] + item_id = item["id"] + in_progress = {**response, "completed_at": None, "status": "in_progress", "output": []} + events: list[dict[str, Any]] = [ + {"type": "response.created", "response": in_progress}, + { + "type": "response.output_item.added", + "response_id": response_id, + "output_index": 0, + "item": {**item, "status": "in_progress", "content": []}, + }, + { + "type": "response.content_part.added", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": 0, + "part": {"type": "output_text", "text": "", "annotations": [], "logprobs": []}, + }, + ] + for delta in split_text(text, chunks): + events.append( + { + "type": "response.output_text.delta", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": 0, + "delta": delta, + "logprobs": [], + } + ) + events.extend( + [ + { + "type": "response.output_text.done", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": 0, + "text": text, + "logprobs": [], + }, + { + "type": "response.content_part.done", + "response_id": response_id, + "item_id": item_id, + "output_index": 0, + "content_index": 0, + "part": item["content"][0], + }, + { + "type": "response.output_item.done", + "response_id": response_id, + "output_index": 0, + "item": item, + }, + {"type": "response.completed", "response": response}, + ] + ) + return events + + +def anthropic_events(model: str, text: str, chunks: int) -> list[dict[str, Any]]: + message_id = f"msg_{uuid.uuid4().hex}" + events: list[dict[str, Any]] = [ + { + "type": "message_start", + "message": { + "id": message_id, + "type": "message", + "role": "assistant", + "content": [], + "model": model, + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + ] + for delta in split_text(text, chunks): + events.append( + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": delta}, + } + ) + events.extend( + [ + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 1}, + }, + {"type": "message_stop"}, + ] + ) + return events + + +def make_request( + provider: str, + streaming: bool, + payload_bytes: int, + *, + model: str, + request_fill: str, +) -> bytes: + payload = request_fill * payload_bytes + if provider == "openai": + value = { + "model": model, + "input": [ + { + "role": "user", + "content": [{"type": "input_text", "text": payload}], + } + ], + "stream": streaming, + } + else: + value = { + "model": model, + "max_tokens": 1024, + "messages": [{"role": "user", "content": payload}], + "stream": streaming, + } + return json.dumps(value, separators=(",", ":")).encode() + + +def request_path(provider: str) -> str: + return "/v1/responses" if provider == "openai" else "/v1/messages" + + +def request_headers(provider: str) -> dict[str, str]: + headers = {"Content-Type": "application/json"} + if provider == "openai": + headers["Authorization"] = "Bearer relay-benchmark" + else: + headers["x-api-key"] = "relay-benchmark" + headers["anthropic-version"] = "2023-06-01" + return headers diff --git a/scripts/latency_benchmark/src/report/report.js b/scripts/latency_benchmark/src/report/report.js new file mode 100644 index 000000000..38e09c203 --- /dev/null +++ b/scripts/latency_benchmark/src/report/report.js @@ -0,0 +1,470 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +"use strict"; + +const benchmark = JSON.parse(document.getElementById("benchmark-data").textContent); +const svgNamespace = "http://www.w3.org/2000/svg"; +const colors = ["#5f8f00", "#006e9c", "#b45f06", "#7042a6", "#b3261e", "#007c70", "#725f00", "#4554a3"]; + +function byId(id) { + return document.getElementById(id); +} + +function addElement(parent, tag, attributes = {}, text = null) { + const element = document.createElement(tag); + for (const [name, value] of Object.entries(attributes)) { + element.setAttribute(name, String(value)); + } + if (text !== null) { + element.textContent = text; + } + parent.appendChild(element); + return element; +} + +function addSvg(parent, tag, attributes = {}, text = null) { + const element = document.createElementNS(svgNamespace, tag); + for (const [name, value] of Object.entries(attributes)) { + element.setAttribute(name, String(value)); + } + if (text !== null) { + element.textContent = text; + } + parent.appendChild(element); + return element; +} + +function labelName(value) { + const names = { + "direct": "Direct", + "relay-minimal": "Relay minimal", + "relay-file": "ATOF file exporter", + "relay-otlp": "OTLP exporter", + "relay-minimal_vs_direct": "Minimal − direct", + "relay-file_vs_direct": "ATOF file exporter − direct", + "relay-otlp_vs_direct": "OTLP exporter − direct", + "file_exporter_vs_minimal": "ATOF file exporter − minimal", + "otlp_exporter_vs_minimal": "OTLP exporter − minimal", + "process_baseline": "Process baseline", + }; + if (names[value]) { + return names[value]; + } + if (value.endsWith("_vs_direct")) { + return `${labelName(value.slice(0, -"_vs_direct".length))} − direct`; + } + if (value.endsWith("_vs_minimal")) { + return `${labelName(value.slice(0, -"_vs_minimal".length))} − minimal`; + } + if (value.endsWith("_vs_process_baseline")) { + return `${labelName(value.slice(0, -"_vs_process_baseline".length))} − process baseline`; + } + if (value.startsWith("relay-")) { + return `Relay ${value.slice("relay-".length).replaceAll("-", " ")}`; + } + const label = value.replaceAll("_", " ").replaceAll("-", " "); + return label.charAt(0).toUpperCase() + label.slice(1); +} + +function formatMs(value) { + if (!Number.isFinite(value)) { + return "—"; + } + const magnitude = Math.abs(value); + const digits = magnitude >= 100 ? 1 : magnitude >= 10 ? 2 : 3; + return `${value.toFixed(digits)} ms`; +} + +function formatBytes(value) { + if (!Number.isFinite(value)) { + return "—"; + } + const units = ["B", "KiB", "MiB", "GiB"]; + let amount = value; + let unit = 0; + while (amount >= 1024 && unit < units.length - 1) { + amount /= 1024; + unit += 1; + } + const digits = amount >= 10 || unit === 0 ? 0 : 1; + return `${amount.toFixed(digits)} ${units[unit]}`; +} + +function formatValue(value) { + if (Array.isArray(value)) { + return value.map((item) => formatValue(item)).join("; "); + } + if (value !== null && typeof value === "object") { + return Object.entries(value) + .map(([key, item]) => `${key}: ${formatValue(item)}`) + .join("; "); + } + return String(value); +} + +function summaryCard(parent, label, value) { + const card = addElement(parent, "div", { class: "summary-card" }); + addElement(card, "span", { class: "label" }, label); + addElement(card, "strong", { class: "value" }, value); +} + +function renderKeyValueTable(parent, values) { + const rows = Object.entries(values).map(([key, value]) => [labelName(key), formatValue(value)]); + renderTable(parent, ["Field", "Value"], rows); +} + +function renderTable(parent, headers, rows, numericColumns = []) { + parent.replaceChildren(); + const table = addElement(parent, "table"); + const head = addElement(table, "thead"); + const headRow = addElement(head, "tr"); + headers.forEach((header) => addElement(headRow, "th", { scope: "col" }, header)); + const body = addElement(table, "tbody"); + for (const row of rows) { + const tableRow = addElement(body, "tr"); + row.forEach((value, index) => { + addElement(tableRow, "td", numericColumns.includes(index) ? { class: "numeric" } : {}, value); + }); + } +} + +function renderOverview() { + const environment = benchmark.environment || {}; + const parameters = benchmark.parameters || {}; + const tests = parameters.tests || []; + + const cards = byId("summary-cards"); + summaryCard(cards, "Suites", tests.join(", ") || "None"); + summaryCard(cards, "Gateway scenarios", String((benchmark.gateway || []).length)); + summaryCard(cards, "Relay", environment.relay_version || "unknown"); + summaryCard(cards, "Working tree", environment.git_dirty ? "Dirty" : "Clean"); + summaryCard(cards, "Platform", environment.platform || "unknown"); + + renderKeyValueTable(byId("environment-table"), environment); + renderKeyValueTable(byId("parameters-table"), parameters); +} + +function populateSelect(select, entries) { + select.replaceChildren(); + for (const [value, label] of entries) { + addElement(select, "option", { value }, label); + } +} + +function uniqueSorted(values, numeric = false) { + const unique = [...new Set(values)]; + return unique.sort(numeric ? (left, right) => left - right : undefined); +} + +function graphRange(values, includeZero) { + let minimum = Math.min(...values); + let maximum = Math.max(...values); + if (includeZero) { + minimum = Math.min(minimum, 0); + maximum = Math.max(maximum, 0); + } + if (minimum === maximum) { + const padding = Math.max(Math.abs(minimum) * 0.15, 0.1); + minimum -= padding; + maximum += padding; + } else { + const padding = (maximum - minimum) * 0.12; + minimum -= padding; + maximum += padding; + } + return [minimum, maximum]; +} + +function drawLineChart(svg, series, payloads, includeZero) { + svg.replaceChildren(); + const width = 920; + const height = 390; + const margin = { top: 28, right: 24, bottom: 66, left: 82 }; + const innerWidth = width - margin.left - margin.right; + const innerHeight = height - margin.top - margin.bottom; + svg.setAttribute("viewBox", `0 0 ${width} ${height}`); + + const values = series.flatMap((item) => item.values.map((point) => point.value)); + const [minimum, maximum] = graphRange(values, includeZero); + const x = (index) => margin.left + (payloads.length === 1 ? innerWidth / 2 : (index / (payloads.length - 1)) * innerWidth); + const y = (value) => margin.top + ((maximum - value) / (maximum - minimum)) * innerHeight; + + for (let index = 0; index <= 5; index += 1) { + const value = minimum + ((maximum - minimum) * index) / 5; + const position = y(value); + addSvg(svg, "line", { + class: Math.abs(value) < (maximum - minimum) / 1000 ? "zero-line" : "grid-line", + x1: margin.left, + y1: position, + x2: width - margin.right, + y2: position, + }); + addSvg(svg, "text", { class: "chart-label", x: margin.left - 12, y: position + 4, "text-anchor": "end" }, formatMs(value)); + } + + addSvg(svg, "line", { + class: "axis", + x1: margin.left, + y1: margin.top + innerHeight, + x2: width - margin.right, + y2: margin.top + innerHeight, + }); + payloads.forEach((payload, index) => { + const position = x(index); + addSvg(svg, "line", { + class: "axis", + x1: position, + y1: margin.top + innerHeight, + x2: position, + y2: margin.top + innerHeight + 6, + }); + addSvg( + svg, + "text", + { class: "chart-label", x: position, y: height - 30, "text-anchor": "middle" }, + formatBytes(payload), + ); + }); + addSvg(svg, "text", { class: "chart-label", x: margin.left + innerWidth / 2, y: height - 6, "text-anchor": "middle" }, "Request content size"); + + series.forEach((item, seriesIndex) => { + const color = colors[seriesIndex % colors.length]; + const points = item.values.map((point, index) => `${x(index)},${y(point.value)}`).join(" "); + addSvg(svg, "polyline", { + class: "line-series", + points, + stroke: color, + "stroke-dasharray": seriesIndex === 1 ? "9 5" : seriesIndex === 2 ? "3 5" : "none", + }); + item.values.forEach((point, index) => { + const circle = addSvg(svg, "circle", { + class: "chart-point", + cx: x(index), + cy: y(point.value), + r: 5, + fill: color, + }); + addSvg(circle, "title", {}, `${item.label}, ${formatBytes(point.payload)}: ${formatMs(point.value)}`); + }); + }); +} + +function renderLegend(series) { + const legend = byId("gateway-legend"); + legend.replaceChildren(); + series.forEach((item, index) => { + const entry = addElement(legend, "span", { class: "legend-item" }); + addElement(entry, "span", { + class: "legend-swatch", + style: `border-color: ${colors[index % colors.length]}`, + "aria-hidden": "true", + }); + addElement(entry, "span", {}, item.label); + }); +} + +function gatewaySeries(scenarios, view, metric, statistic) { + const selected = scenarios[0]; + const definitions = view === "absolute" + ? Object.keys(selected.absolute) + : Object.keys(selected.comparisons).filter((name) => name.endsWith(view === "minimal" ? "_vs_minimal" : "_vs_direct")); + return definitions.map((name) => ({ + name, + label: labelName(name), + values: scenarios.map((scenario) => { + const collection = view === "absolute" ? scenario.absolute : scenario.comparisons; + return { + payload: scenario.payload_bytes, + value: collection[name][metric][statistic], + summary: collection[name][metric], + }; + }), + })); +} + +function selectedGatewayScenarios() { + const provider = byId("gateway-provider").value; + const mode = byId("gateway-mode").value; + const concurrency = Number(byId("gateway-concurrency").value); + return benchmark.gateway + .filter((scenario) => scenario.provider === provider && scenario.mode === mode && scenario.concurrency === concurrency) + .sort((left, right) => left.payload_bytes - right.payload_bytes); +} + +function syncMetricOptions() { + const metric = byId("gateway-metric"); + const previous = metric.value; + const options = byId("gateway-mode").value === "streaming" + ? [["first_content", "Time to first content"], ["total", "Total response"]] + : [["total", "Total response"]]; + populateSelect(metric, options); + if (options.some(([value]) => value === previous)) { + metric.value = previous; + } +} + +function renderGateway() { + const scenarios = selectedGatewayScenarios(); + if (!scenarios.length) { + return; + } + const view = byId("gateway-view").value; + const metric = byId("gateway-metric").value; + const statistic = byId("gateway-percentile").value; + const series = gatewaySeries(scenarios, view, metric, statistic); + const payloads = scenarios.map((scenario) => scenario.payload_bytes); + const isDelta = view !== "absolute"; + drawLineChart(byId("gateway-chart"), series, payloads, isDelta); + renderLegend(series); + + const metricLabel = metric === "first_content" ? "time to first content" : "total response time"; + const viewLabel = view === "absolute" + ? "absolute latency" + : view === "minimal" + ? "variant overhead relative to minimal Relay" + : "Relay overhead relative to direct provider calls"; + byId("gateway-chart-description").textContent = + `${statistic.replace("_ms", "")} ${metricLabel}; ${viewLabel}. ` + + `Provider ${byId("gateway-provider").value}, ${byId("gateway-mode").value}, ` + + `concurrency ${byId("gateway-concurrency").value}.`; + + const rows = []; + for (const item of series) { + for (const point of item.values) { + const interval = point.summary.median_ci95_ms; + rows.push([ + item.label, + formatBytes(point.payload), + formatMs(point.summary.p50_ms), + formatMs(point.summary.p95_ms), + formatMs(point.summary.p99_ms), + interval ? `${formatMs(interval[0])} to ${formatMs(interval[1])}` : "—", + String(point.summary.samples), + ]); + } + } + renderTable( + byId("gateway-table"), + ["Path or comparison", "Payload", "p50", "p95", "p99", "Median 95% CI", "Samples"], + rows, + [2, 3, 4, 6], + ); +} + +function initializeGateway() { + if (!benchmark.gateway || !benchmark.gateway.length) { + byId("gateway-section").hidden = true; + return; + } + populateSelect( + byId("gateway-provider"), + uniqueSorted(benchmark.gateway.map((scenario) => scenario.provider)).map((value) => [value, value]), + ); + populateSelect( + byId("gateway-mode"), + uniqueSorted(benchmark.gateway.map((scenario) => scenario.mode)).map((value) => [value, value]), + ); + populateSelect( + byId("gateway-concurrency"), + uniqueSorted(benchmark.gateway.map((scenario) => scenario.concurrency), true).map((value) => [String(value), String(value)]), + ); + populateSelect(byId("gateway-percentile"), [["p50_ms", "p50"], ["p95_ms", "p95"], ["p99_ms", "p99"]]); + populateSelect(byId("gateway-view"), [ + ["direct", "Paired delta vs direct"], + ["minimal", "Paired delta vs minimal"], + ["absolute", "Absolute latency"], + ]); + syncMetricOptions(); + for (const id of [ + "gateway-provider", + "gateway-mode", + "gateway-concurrency", + "gateway-metric", + "gateway-percentile", + "gateway-view", + ]) { + byId(id).addEventListener("change", () => { + if (id === "gateway-mode") { + syncMetricOptions(); + } + renderGateway(); + }); + } + renderGateway(); +} + +function drawBarChart(svg, entries) { + svg.replaceChildren(); + const width = 920; + const rowHeight = 38; + const margin = { top: 16, right: 90, bottom: 28, left: 220 }; + const innerWidth = width - margin.left - margin.right; + const height = margin.top + margin.bottom + entries.length * rowHeight; + const maximum = Math.max(...entries.map((entry) => entry.value), 0.001) * 1.08; + svg.setAttribute("viewBox", `0 0 ${width} ${height}`); + + entries.forEach((entry, index) => { + const y = margin.top + index * rowHeight; + const barWidth = Math.max((entry.value / maximum) * innerWidth, 1); + addSvg(svg, "text", { class: "bar-label", x: margin.left - 12, y: y + 20, "text-anchor": "end" }, entry.label); + const bar = addSvg(svg, "rect", { + x: margin.left, + y: y + 5, + width: barWidth, + height: 22, + rx: 4, + fill: colors[index % colors.length], + }); + addSvg(bar, "title", {}, `${entry.label}: ${formatMs(entry.value)}`); + addSvg(svg, "text", { class: "bar-value", x: margin.left + barWidth + 8, y: y + 20 }, formatMs(entry.value)); + }); +} + +function summaryRows(collection) { + return Object.entries(collection).map(([name, summary]) => { + const interval = summary.median_ci95_ms; + return [ + labelName(name), + formatMs(summary.p50_ms), + formatMs(summary.p95_ms), + formatMs(summary.p99_ms), + interval ? `${formatMs(interval[0])} to ${formatMs(interval[1])}` : "—", + String(summary.samples), + ]; + }); +} + +function renderProcessSuite(name) { + const result = benchmark[name]; + if (!result) { + byId(`${name}-section`).hidden = true; + return; + } + const entries = Object.entries(result.absolute).map(([path, summary]) => ({ + label: labelName(path), + value: summary.p50_ms, + })); + drawBarChart(byId(`${name}-chart`), entries); + const headers = ["Path", "p50", "p95", "p99", "Median 95% CI", "Samples"]; + renderTable(byId(`${name}-absolute-table`), headers, summaryRows(result.absolute), [1, 2, 3, 5]); + renderTable(byId(`${name}-comparison-table`), headers, summaryRows(result.comparisons), [1, 2, 3, 5]); +} + +function renderDelivery() { + if (!benchmark.exporter_delivery) { + byId("delivery-section").hidden = true; + return; + } + const cards = byId("delivery-cards"); + summaryCard(cards, "ATOF written", formatBytes(benchmark.exporter_delivery.atof_bytes)); + summaryCard(cards, "OTLP requests", Number(benchmark.exporter_delivery.otlp_requests).toLocaleString()); +} + +renderOverview(); +initializeGateway(); +renderProcessSuite("hooks"); +renderProcessSuite("startup"); +renderDelivery(); diff --git a/scripts/latency_benchmark/src/report/styles.css b/scripts/latency_benchmark/src/report/styles.css new file mode 100644 index 000000000..45753b432 --- /dev/null +++ b/scripts/latency_benchmark/src/report/styles.css @@ -0,0 +1,435 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +:root { + color-scheme: light; + --ink: #1a1a1a; + --muted: #5b5b5b; + --line: #d6d6d6; + --panel: #ffffff; + --soft: #f5f5f5; + --accent: #76b900; + --accent-dark: #456d00; + --focus: #006eb8; + + font-family: "NVIDIA Sans", Arial, Helvetica, sans-serif; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + color: var(--ink); + background: var(--panel); + font-size: 16px; + line-height: 1.6; +} + +.page-shell { + width: min(1180px, calc(100% - 64px)); + margin: 0 auto; +} + +.hero { + padding: 42px 0 36px; + border-top: 4px solid var(--accent); + border-bottom: 1px solid var(--line); + background: var(--panel); +} + +.hero h1 { + margin: 0; + font-size: clamp(2.4rem, 6vw, 3.25rem); + font-weight: 700; + line-height: 1.12; + letter-spacing: -0.035em; +} + +main { + padding: 0 0 72px; +} + +section { + margin: 0; + padding: 42px 0; + border-bottom: 1px solid var(--line); +} + +section:first-child { + padding-top: 34px; +} + +section[hidden] { + display: none; +} + +h2, +h3 { + color: var(--ink); + line-height: 1.25; +} + +h2 { + margin: 0 0 24px; + font-size: 2rem; + font-weight: 700; + letter-spacing: -0.02em; +} + +h3 { + margin: 0 0 12px; + font-size: 1.12rem; + font-weight: 700; +} + +p { + color: var(--muted); +} + +code { + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; +} + +.summary-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); + gap: 16px; +} + +.summary-card { + min-height: 104px; + padding: 17px 18px; + border: 1px solid var(--line); + border-top: 3px solid var(--accent); + background: var(--panel); +} + +.summary-card .label { + color: var(--muted); + font-size: 0.74rem; + font-weight: 700; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.summary-card .value { + display: block; + margin-top: 8px; + overflow-wrap: anywhere; + color: var(--ink); + font-size: 1.22rem; + font-weight: 700; +} + +details { + margin-top: 24px; + border: 1px solid var(--line); + background: var(--panel); +} + +summary { + padding: 14px 18px; + cursor: pointer; + color: var(--accent-dark); + font-weight: 700; +} + +.details-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 24px; + padding: 20px; + border-top: 1px solid var(--line); +} + +.split-tables { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 24px; + margin-top: 28px; +} + +.definition-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 16px; +} + +.definition-grid article { + padding: 18px 20px; + border-left: 3px solid var(--accent); + background: var(--soft); +} + +.definition-grid p, +.section-heading p, +.chart-card p, +.table-section p { + margin: 0; +} + +.callout { + margin: 24px 0 0; + padding: 16px 20px; + border-left: 3px solid var(--accent); + color: #314200; + background: #f2f8e8; +} + +.section-heading { + display: grid; + grid-template-columns: minmax(210px, 0.55fr) minmax(300px, 1.45fr); + gap: 40px; + align-items: start; + margin-bottom: 28px; +} + +.section-heading h2 { + margin-bottom: 0; +} + +.section-heading > p { + max-width: 760px; +} + +.controls { + display: grid; + grid-template-columns: repeat(3, minmax(160px, 1fr)); + gap: 16px; + margin-bottom: 24px; + padding: 20px; + border: 1px solid var(--line); + background: var(--soft); +} + +label { + display: grid; + gap: 6px; + color: var(--ink); + font-size: 0.82rem; + font-weight: 700; +} + +select { + width: 100%; + min-height: 40px; + padding: 8px 10px; + border: 1px solid #a7a7a7; + border-radius: 2px; + color: var(--ink); + background: #fff; + font: inherit; + font-size: 0.9rem; +} + +select:focus-visible, +summary:focus-visible { + outline: 3px solid rgba(0, 110, 184, 0.32); + outline-offset: 2px; +} + +.chart-card { + padding: 24px; + overflow: hidden; + border: 1px solid var(--line); + background: var(--panel); +} + +svg { + display: block; + width: 100%; + margin-top: 16px; + overflow: visible; +} + +.line-chart { + min-height: 360px; +} + +.bar-chart { + min-height: 220px; +} + +.axis, +.grid-line { + stroke: #d4d4d4; + stroke-width: 1; +} + +.grid-line { + stroke-dasharray: 3 5; +} + +.zero-line { + stroke: #747474; + stroke-width: 1.5; +} + +.chart-label { + fill: #595959; + font-size: 12px; +} + +.line-series { + fill: none; + stroke-width: 2.5; + stroke-linecap: round; + stroke-linejoin: round; +} + +.chart-point { + stroke: #fff; + stroke-width: 2; +} + +.bar-label { + fill: #262626; + font-size: 12px; +} + +.bar-value { + fill: #595959; + font-size: 12px; + font-weight: 700; +} + +.legend { + display: flex; + flex-wrap: wrap; + gap: 10px 20px; + margin-top: 8px; + color: var(--muted); + font-size: 0.82rem; +} + +.legend-item { + display: inline-flex; + gap: 8px; + align-items: center; +} + +.legend-swatch { + width: 26px; + border-top: 3px solid; +} + +.table-section, +.split-tables { + margin-top: 28px; +} + +.table-wrap { + max-width: 100%; + overflow-x: auto; + border: 1px solid var(--line); +} + +table { + width: 100%; + border-collapse: collapse; + font-size: 0.84rem; +} + +th, +td { + padding: 10px 12px; + border-bottom: 1px solid #e1e1e1; + text-align: left; + white-space: nowrap; +} + +th { + color: #333333; + background: var(--soft); + font-size: 0.72rem; + letter-spacing: 0.05em; + text-transform: uppercase; +} + +td.numeric { + text-align: right; + font-variant-numeric: tabular-nums; +} + +tr:last-child td { + border-bottom: 0; +} + +tbody tr:hover { + background: #f8fbef; +} + +footer { + padding: 24px 0 32px; + border-top: 1px solid var(--line); + color: var(--muted); + background: var(--soft); + font-size: 0.85rem; +} + +@media (max-width: 760px) { + .page-shell { + width: min(100% - 32px, 1180px); + } + + .hero { + padding: 32px 0 28px; + } + + section { + padding: 34px 0; + } + + .details-grid, + .split-tables, + .definition-grid, + .section-heading { + grid-template-columns: 1fr; + } + + .controls { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .details-grid { + padding: 16px; + } + + .line-chart { + min-height: 300px; + } +} + +@media (max-width: 560px) { + .summary-grid, + .controls { + grid-template-columns: 1fr; + } + + .hero h1 { + font-size: 2.25rem; + } + + .chart-card { + padding: 16px; + } +} + +@media print { + .hero { + padding-top: 24px; + } + + section { + break-inside: avoid; + } + + .controls, + details { + display: none; + } +} diff --git a/scripts/latency_benchmark/src/report/template.html b/scripts/latency_benchmark/src/report/template.html new file mode 100644 index 000000000..2ba6e4b71 --- /dev/null +++ b/scripts/latency_benchmark/src/report/template.html @@ -0,0 +1,213 @@ + + + + + + + NeMo Relay Latency Report + + + +
+
+

NeMo Relay Latency Report

+
+
+ +
+
+

Run Summary

+
+
+ Environment and resolved configuration +
+
+

Environment

+
+
+
+

Configuration

+
+
+
+
+
+ +
+

How to Read This Report

+
+
+

Absolute Latency

+

+ Elapsed wall time for one path. Use it to understand the complete + local operation, including process work where applicable. +

+
+
+

Paired Delta

+

+ The left path minus its baseline in the same measurement cycle. + Positive values mean added time. Small negative values can occur + because of scheduler and measurement noise. +

+
+
+

Percentiles

+

+ p50 is the median. p95 and p99 describe slower tail observations. + Min and max are the observed extremes and are more sensitive to + outliers. +

+
+
+

Median 95% CI

+

+ The bootstrap interval estimates uncertainty around the median + paired delta. It is not a range containing 95% of observations. +

+
+
+

+ Prefer paired added milliseconds over percentages. Loopback baselines + are very small, so a harmless absolute difference can look large as a + percentage. Do not draw performance conclusions from a smoke test. +

+
+ +
+
+
+

Gateway

+
+

+ Compares deterministic provider calls made directly and through + minimal Relay, the ATOF file exporter, the OTLP exporter, and + opt-in middleware Relay gateways. +

+
+
+ + + + + + +
+
+

Gateway Latency by Payload Size

+

+ +
+
+
+

Selected Gateway Data

+

+ The table is the accessible numeric counterpart to the graph. The + confidence interval is available for paired median deltas only. +

+
+
+
+ +
+
+
+

Hooks

+
+

+ Measures full nemo-relay hook-forward subprocess wall + time for Codex and Claude Code. Paired deltas subtract a + nemo-relay --version process baseline. +

+
+
+

Hook Absolute p50

+

Median subprocess wall time in milliseconds.

+ +
+
+
+

Absolute Latency

+
+
+
+

Paired Delta From Process Baseline

+
+
+
+
+ +
+
+
+

Startup

+
+

+ Measures cold Relay process launch through a healthy gateway. Paired + deltas subtract the same process baseline used by the hook suite. +

+
+
+

Startup Absolute p50

+

Median process or readiness time in milliseconds.

+ +
+
+
+

Absolute Latency

+
+
+
+

Paired Delta From Process Baseline

+
+
+
+
+ +
+
+
+

Exporter Delivery

+
+

+ These counts confirm that the ATOF file exporter and OTLP exporter + delivered data. They are correctness signals, not latency metrics. +

+
+
+
+
+ + + + + + + diff --git a/scripts/latency_benchmark/src/reporting.py b/scripts/latency_benchmark/src/reporting.py new file mode 100644 index 000000000..2e4f7fe8b --- /dev/null +++ b/scripts/latency_benchmark/src/reporting.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Benchmark environment metadata and terminal reporting.""" + +from __future__ import annotations + +import datetime as dt +import platform +import subprocess +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[3] + + +def _git_output(*args: str) -> str: + result = subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=False) + return result.stdout.strip() if result.returncode == 0 else "unknown" + + +def environment_record(binary: Path) -> dict[str, Any]: + version = subprocess.run([str(binary), "--version"], capture_output=True, text=True, check=True).stdout.strip() + git_status = _git_output("status", "--porcelain") + return { + "generated_at": dt.datetime.now(dt.UTC).isoformat(), + "git_commit": _git_output("rev-parse", "HEAD"), + "git_dirty": git_status not in {"", "unknown"}, + "relay_version": version, + "platform": platform.platform(), + "machine": platform.machine(), + "processor": platform.processor(), + "python": platform.python_version(), + } + + +def print_results(results: dict[str, Any]) -> None: + """Print only the result sections produced by selected test suites.""" + if "gateway" in results: + print("\nGateway incremental latency (milliseconds; negative values are measurement noise)") + headers = ("provider", "mode", "bytes", "c", "comparison", "metric", "p50", "p95", "p99") + print(" ".join(f"{header:>12}" for header in headers)) + for scenario in results["gateway"]: + for comparison, metrics in scenario["comparisons"].items(): + if not comparison.endswith("_vs_direct"): + continue + for metric, summary in metrics.items(): + values = ( + scenario["provider"], + scenario["mode"], + str(scenario["payload_bytes"]), + str(scenario["concurrency"]), + comparison.replace("relay-", "").replace("_vs_direct", ""), + metric, + f"{summary['p50_ms']:.3f}", + f"{summary['p95_ms']:.3f}", + f"{summary['p99_ms']:.3f}", + ) + print(" ".join(f"{value:>12}" for value in values)) + + if "hooks" in results: + print("\nHook subprocess wall time (p50 milliseconds)") + for name, summary in results["hooks"]["absolute"].items(): + print(f" {name:<24} {summary['p50_ms']:>9.3f}") + + if "startup" in results: + print("\nCold process/readiness time (p50 milliseconds)") + for name, summary in results["startup"]["absolute"].items(): + print(f" {name:<24} {summary['p50_ms']:>9.3f}") + + if "exporter_delivery" in results: + delivery = results["exporter_delivery"] + print("\nExporter delivery verification") + print(f" ATOF bytes written {delivery['atof_bytes']:>12}") + print(f" OTLP requests received {delivery['otlp_requests']:>12}") + + if "validation_errors" in results: + print("\nValidation errors") + for error in results["validation_errors"]: + print(f" {error}") diff --git a/scripts/latency_benchmark/src/servers.py b/scripts/latency_benchmark/src/servers.py new file mode 100644 index 000000000..a7a88fb4e --- /dev/null +++ b/scripts/latency_benchmark/src/servers.py @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Loopback provider and OTLP servers used by the benchmark.""" + +from __future__ import annotations + +import contextlib +import http.client +import json +import socket +import threading +import uuid +from collections.abc import Iterable, Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, ClassVar +from urllib.parse import urlparse + +from .protocol import anthropic_events, openai_events, response_document + + +class QuietThreadingServer(ThreadingHTTPServer): + daemon_threads = True + allow_reuse_address = True + + +class ProviderHandler(BaseHTTPRequestHandler): + """Serve deterministic OpenAI Responses and Anthropic Messages payloads.""" + + protocol_version = "HTTP/1.1" + + def setup(self) -> None: + super().setup() + self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + del format, args + + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("content-length", "0")) + request = json.loads(self.rfile.read(length) or b"{}") + path = urlparse(self.path).path + text = getattr(self.server, "response_fill") * getattr(self.server, "response_bytes") + chunks = getattr(self.server, "stream_chunks") + if path.endswith("/responses"): + self._openai(request, text, chunks) + elif path.endswith("/messages"): + self._anthropic(request, text, chunks) + else: + self.send_error(404) + + def _openai(self, request: dict[str, Any], text: str, chunks: int) -> None: + model = request.get("model", getattr(self.server, "openai_model")) + if request.get("stream", False): + frames = [ + f"data: {json.dumps(event, separators=(',', ':'))}\n\n".encode() + for event in openai_events(model, text, chunks) + ] + frames.append(b"data: [DONE]\n\n") + self._send_stream(frames) + return + self._send_json(response_document(model, text)) + + def _anthropic(self, request: dict[str, Any], text: str, chunks: int) -> None: + model = request.get("model", getattr(self.server, "anthropic_model")) + if request.get("stream", False): + frames = [] + for event in anthropic_events(model, text, chunks): + name = event["type"] + frames.append(f"event: {name}\ndata: {json.dumps(event, separators=(',', ':'))}\n\n".encode()) + self._send_stream(frames) + return + self._send_json( + { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": text}], + "model": model, + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ) + + def _send_json(self, value: dict[str, Any]) -> None: + body = json.dumps(value, separators=(",", ":")).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _send_stream(self, frames: Iterable[bytes]) -> None: + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + for frame in frames: + self.wfile.write(f"{len(frame):X}\r\n".encode()) + self.wfile.write(frame) + self.wfile.write(b"\r\n") + self.wfile.flush() + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + + +class OtlpHandler(BaseHTTPRequestHandler): + """Accept OTLP requests and track whether the exporter delivered data.""" + + protocol_version = "HTTP/1.1" + request_count: ClassVar[int] = 0 + request_count_lock: ClassVar[threading.Lock] = threading.Lock() + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + del format, args + + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("content-length", "0")) + self.rfile.read(length) + with self.request_count_lock: + type(self).request_count += 1 + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + @classmethod + def reset(cls) -> None: + with cls.request_count_lock: + cls.request_count = 0 + + +@contextlib.contextmanager +def local_server(handler: type[BaseHTTPRequestHandler], **attributes: Any) -> Iterator[str]: + server = QuietThreadingServer(("127.0.0.1", 0), handler) + for name, value in attributes.items(): + setattr(server, name, value) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + server.server_close() + thread.join() + + +def connection_for(url: str) -> http.client.HTTPConnection: + parsed = urlparse(url) + if parsed.hostname is None: + raise ValueError(f"URL does not contain a host: {url}") + return http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=30) diff --git a/scripts/latency_benchmark/tests/__init__.py b/scripts/latency_benchmark/tests/__init__.py new file mode 100644 index 000000000..f11314bc5 --- /dev/null +++ b/scripts/latency_benchmark/tests/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the latency benchmark.""" diff --git a/scripts/latency_benchmark/tests/test_benchmarks.py b/scripts/latency_benchmark/tests/test_benchmarks.py new file mode 100644 index 000000000..c7d5285b6 --- /dev/null +++ b/scripts/latency_benchmark/tests/test_benchmarks.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for latency benchmark measurement coordination.""" + +import threading +import unittest +from unittest import mock + +from scripts.latency_benchmark.src import benchmarks + + +class GatewayBenchmarkTests(unittest.TestCase): + def test_rotates_variants_by_sample_index(self) -> None: + calls_by_thread: dict[int, list[str]] = {} + calls_lock = threading.Lock() + + def connection_for(url: str) -> mock.Mock: + return mock.Mock(url=url) + + def perform_request(connection: mock.Mock, *_args: object) -> dict[str, int]: + with calls_lock: + calls_by_thread.setdefault(threading.get_ident(), []).append(connection.url) + return {"total_ns": 1} + + urls = { + "direct": "direct", + "relay-minimal": "relay-minimal", + "relay-file": "relay-file", + "relay-otlp": "relay-otlp", + } + with ( + mock.patch.object(benchmarks, "connection_for", side_effect=connection_for), + mock.patch.object(benchmarks, "make_request", return_value=b"request"), + mock.patch.object(benchmarks, "perform_request", side_effect=perform_request), + ): + benchmarks.benchmark_scenario( + urls, + provider="openai", + model="benchmark-model", + request_fill="x", + streaming=False, + payload_bytes=4096, + samples=4, + warmup=0, + concurrency=4, + ) + + self.assertEqual( + {tuple(calls) for calls in calls_by_thread.values()}, + { + ("direct", "relay-minimal", "relay-file", "relay-otlp"), + ("relay-minimal", "relay-file", "relay-otlp", "direct"), + ("relay-file", "relay-otlp", "direct", "relay-minimal"), + ("relay-otlp", "direct", "relay-minimal", "relay-file"), + }, + ) + + def test_aborts_barrier_when_a_worker_fails_during_warmup(self) -> None: + barrier = mock.Mock() + connection = mock.Mock() + + with ( + mock.patch.object(benchmarks.threading, "Barrier", return_value=barrier), + mock.patch.object(benchmarks, "connection_for", return_value=connection), + mock.patch.object(benchmarks, "make_request", return_value=b"request"), + mock.patch.object(benchmarks, "perform_request", side_effect=RuntimeError("warmup failed")), + ): + with self.assertRaisesRegex(RuntimeError, "warmup failed"): + benchmarks.benchmark_scenario( + {"direct": "http://127.0.0.1:8000"}, + provider="openai", + model="benchmark-model", + request_fill="x", + streaming=False, + payload_bytes=4096, + samples=2, + warmup=1, + concurrency=2, + ) + + barrier.abort.assert_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/latency_benchmark/tests/test_cli.py b/scripts/latency_benchmark/tests/test_cli.py new file mode 100644 index 000000000..e3ad43348 --- /dev/null +++ b/scripts/latency_benchmark/tests/test_cli.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for latency benchmark orchestration.""" + +import contextlib +import io +import json +import tempfile +import unittest +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from scripts.latency_benchmark.src import cli +from scripts.latency_benchmark.src.config import DEFAULT_CONFIG_PATH, load_config + + +class BenchmarkOrchestrationTests(unittest.TestCase): + def test_counts_gateway_scenarios_and_process_suites(self) -> None: + config = load_config(DEFAULT_CONFIG_PATH) + + expected_gateway = ( + len(config.providers) * len(config.modes) * len(config.payload_sizes) * len(config.concurrency) + ) + + self.assertEqual(cli._benchmark_test_count(config), expected_gateway + 2) + + def test_reports_progress_for_completed_suites(self) -> None: + config = replace(load_config(DEFAULT_CONFIG_PATH), tests=("hooks", "startup")) + servers = [ + contextlib.nullcontext("http://127.0.0.1:8000"), + contextlib.nullcontext("http://127.0.0.1:4318"), + ] + + with ( + mock.patch.object(cli, "environment_record", return_value={}), + mock.patch.object(cli, "local_server", side_effect=servers), + mock.patch.object(cli, "write_relay_config"), + mock.patch.object(cli, "write_plugin_configs", return_value={}), + mock.patch.object(cli, "benchmark_hooks", return_value={}), + mock.patch.object(cli, "benchmark_startup", return_value={}), + contextlib.redirect_stdout(io.StringIO()) as output, + ): + cli.run_benchmarks(Path("nemo-relay"), config) + + self.assertIn("[1/2] Completed hooks suite", output.getvalue()) + self.assertIn("[2/2] Completed startup suite", output.getvalue()) + + def test_reports_completed_gateway_scenario(self) -> None: + config = replace( + load_config(DEFAULT_CONFIG_PATH), + providers=("openai",), + modes=("buffered",), + payload_sizes=(4096,), + concurrency=(1,), + ) + report_complete = mock.Mock() + + with ( + tempfile.TemporaryDirectory() as temporary, + mock.patch.object(cli, "benchmark_scenario", return_value={"scenario": "complete"}), + contextlib.redirect_stdout(io.StringIO()), + ): + scenarios = cli._benchmark_gateway( + Path("nemo-relay"), + Path(temporary), + "http://127.0.0.1:8000", + {}, + config, + report_complete, + ) + + self.assertEqual(scenarios, [{"scenario": "complete"}]) + report_complete.assert_called_once_with("gateway openai buffered, payload=4096, concurrency=1") + + def test_records_exporter_validation_failures(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + with mock.patch.object(cli.OtlpHandler, "request_count", 0): + delivery, errors = cli._exporter_delivery(Path(temporary)) + + self.assertEqual(delivery, {"atof_bytes": 0, "otlp_requests": 0}) + self.assertEqual( + errors, + [ + "local ATOF exporter did not write benchmark events", + "local OTLP receiver did not receive benchmark exports", + ], + ) + + def test_main_writes_results_before_raising_validation_error(self) -> None: + results = { + "schema_version": 2, + "validation_errors": ["local OTLP receiver did not receive benchmark exports"], + } + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + options = SimpleNamespace( + relay_bin=root / "nemo-relay", + config=mock.Mock(), + output=root / "results.json", + report=root / "results.html", + ) + with ( + mock.patch.object(cli, "parse_args", return_value=options), + mock.patch.object(cli, "run_benchmarks", return_value=results), + contextlib.redirect_stdout(io.StringIO()), + ): + with self.assertRaisesRegex(RuntimeError, "benchmark validation failed"): + cli.main([]) + + self.assertEqual(json.loads(options.output.read_text(encoding="utf-8")), results) + self.assertTrue(options.report.is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/latency_benchmark/tests/test_config.py b/scripts/latency_benchmark/tests/test_config.py new file mode 100644 index 000000000..dc61fe900 --- /dev/null +++ b/scripts/latency_benchmark/tests/test_config.py @@ -0,0 +1,250 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the latency benchmark fixture.""" + +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from scripts.latency_benchmark.src import fixtures +from scripts.latency_benchmark.src.config import ( + DEFAULT_CONFIG_PATH, + MiddlewareVariant, + build_parser, + load_config, + parse_args, +) +from scripts.latency_benchmark.src.fixtures import ( + isolated_environment, + write_agent_config, + write_mock_codex, + write_plugin_configs, +) + + +class BenchmarkConfigTests(unittest.TestCase): + def test_default_config_defines_every_suite_and_matrix_axis(self) -> None: + config = load_config(DEFAULT_CONFIG_PATH) + + self.assertEqual(config.tests, ("gateway", "hooks", "startup")) + self.assertEqual(config.providers, ("openai", "anthropic")) + self.assertEqual(config.modes, ("buffered", "streaming")) + self.assertTrue(config.payload_sizes) + self.assertTrue(config.concurrency) + self.assertEqual(config.middleware, ()) + + def test_partial_config_and_cli_arguments_override_defaults(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + custom_config = root / "quick.toml" + custom_config.write_text('tests = ["startup"]\nsamples = 7\n', encoding="utf-8") + relay_bin = root / "nemo-relay" + relay_bin.touch() + + options = parse_args( + [ + "--relay-bin", + str(relay_bin), + "--output", + str(root / "results.json"), + "--config", + str(custom_config), + "--tests", + "gateway,hooks", + "--samples", + "3", + "--concurrency", + "1", + "--providers", + "openai", + ] + ) + + self.assertEqual(options.config.tests, ("gateway", "hooks")) + self.assertEqual(options.config.samples, 3) + self.assertEqual(options.config.concurrency, (1,)) + self.assertEqual(options.config.providers, ("openai",)) + self.assertEqual(options.config.modes, ("buffered", "streaming")) + self.assertEqual(options.report, (root / "results.html").resolve()) + + def test_help_uses_just_entrypoint_and_explains_every_option(self) -> None: + help_text = build_parser().format_help() + + self.assertIn("usage: just latency-benchmark [options]", help_text) + for option in ( + "-h, --help", + "--relay-bin", + "--output", + "--report", + "--config", + "--tests", + "--providers", + "--modes", + "--payload-sizes", + "--concurrency", + "--middleware", + "--samples", + "--hook-samples", + "--startup-samples", + "--warmup", + "--response-bytes", + "--stream-chunks", + ): + with self.subTest(option=option): + self.assertIn(option, help_text) + + def test_report_path_can_be_overridden(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + relay_bin = root / "nemo-relay" + relay_bin.touch() + + options = parse_args( + [ + "--relay-bin", + str(relay_bin), + "--output", + str(root / "results.json"), + "--report", + str(root / "site" / "index.html"), + ] + ) + + self.assertEqual(options.report, (root / "site" / "index.html").resolve()) + + def test_middleware_paths_are_relative_to_the_custom_config(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + plugin_config = root / "plugins-guardrails.toml" + plugin_config.write_text("version = 1\ncomponents = []\n", encoding="utf-8") + config_path = root / "benchmark.toml" + config_path.write_text( + '[[middleware]]\nname = "guardrails"\nplugin_config = "plugins-guardrails.toml"\n', + encoding="utf-8", + ) + + config = load_config(config_path) + + self.assertEqual( + config.middleware, + (MiddlewareVariant(name="guardrails", plugin_config=plugin_config.resolve()),), + ) + + def test_cli_middleware_replaces_configured_middleware(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + relay_bin = root / "nemo-relay" + relay_bin.touch() + plugin_config = root / "plugins-redaction.toml" + plugin_config.write_text("version = 1\ncomponents = []\n", encoding="utf-8") + + options = parse_args( + [ + "--relay-bin", + str(relay_bin), + "--output", + str(root / "results.json"), + "--middleware", + f"redaction={plugin_config}", + ] + ) + + self.assertEqual( + options.config.middleware, + (MiddlewareVariant(name="redaction", plugin_config=plugin_config.resolve()),), + ) + + def test_rejects_reserved_middleware_name(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + plugin_config = root / "plugins.toml" + plugin_config.touch() + config_path = root / "invalid.toml" + config_path.write_text( + '[[middleware]]\nname = "minimal"\nplugin_config = "plugins.toml"\n', + encoding="utf-8", + ) + + with self.assertRaisesRegex(ValueError, "reserved by a default variant"): + load_config(config_path) + + def test_rejects_unknown_config_keys(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + config_path = Path(temporary) / "invalid.toml" + config_path.write_text("sample = 1\n", encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "unknown config key"): + load_config(config_path) + + def test_rejects_gateway_concurrency_greater_than_samples(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + config_path = Path(temporary) / "invalid.toml" + config_path.write_text( + 'tests = ["gateway"]\nsamples = 2\nconcurrency = [4]\n', + encoding="utf-8", + ) + + with self.assertRaisesRegex(ValueError, "samples must be greater"): + load_config(config_path) + + +class StaticFixtureTests(unittest.TestCase): + def test_materializes_templates_without_embedded_markers(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + custom_config = root / "plugins-guardrails.toml" + custom_config.write_text("version = 1\ncomponents = []\n", encoding="utf-8") + configs = write_plugin_configs( + root, + "http://127.0.0.1:4318", + (MiddlewareVariant(name="guardrails", plugin_config=custom_config),), + ) + mock_codex = write_mock_codex(root) + agent_config = write_agent_config(root, "test", mock_codex) + + rendered = "\n".join(path.read_text(encoding="utf-8") for path in (*configs.values(), agent_config)) + + self.assertEqual(mock_codex.stem, "mock-codex") + self.assertNotIn("__ATOF_OUTPUT_DIRECTORY__", rendered) + self.assertNotIn("__OTLP_ENDPOINT__", rendered) + self.assertNotIn("__CODEX_COMMAND__", rendered) + self.assertEqual(configs["relay-guardrails"], custom_config) + + def test_materializes_windows_mock_with_crlf_line_endings(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + with mock.patch.object(fixtures.os, "name", "nt"): + mock_codex = write_mock_codex(root) + contents = mock_codex.read_bytes() + + self.assertIn(b"\r\n", contents) + self.assertNotIn(b"\n", contents.replace(b"\r\n", b"")) + + def test_isolated_environment_removes_values_that_can_skew_results(self) -> None: + inherited = { + "ANTHROPIC_API_KEY": "secret", + "HTTP_PROXY": "http://proxy.example", + "NEMO_RELAY_CONFIG": "/tmp/developer-config.toml", + "OPENAI_API_KEY": "secret", + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector.example", + "PATH": "/usr/bin", + "RUST_LOG": "debug", + "https_proxy": "http://proxy.example", + } + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + with mock.patch.dict(fixtures.os.environ, inherited, clear=True): + environment = isolated_environment(root) + + self.assertEqual(environment["PATH"], "/usr/bin") + self.assertEqual(environment["NO_PROXY"], "127.0.0.1,localhost") + self.assertEqual(environment["no_proxy"], "127.0.0.1,localhost") + for name in inherited.keys() - {"PATH"}: + with self.subTest(name=name): + self.assertNotIn(name, environment) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/latency_benchmark/tests/test_html_report.py b/scripts/latency_benchmark/tests/test_html_report.py new file mode 100644 index 000000000..179c5e5ab --- /dev/null +++ b/scripts/latency_benchmark/tests/test_html_report.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the self-contained benchmark report.""" + +import tempfile +import unittest +from pathlib import Path +from typing import Any + +from scripts.latency_benchmark.src.html_report import render_html_report, write_html_report + + +def _summary(*, interval: bool = False) -> dict[str, Any]: + result: dict[str, Any] = { + "samples": 5, + "p50_ms": 1.0, + "p95_ms": 1.5, + "p99_ms": 1.6, + "min_ms": 0.8, + "max_ms": 1.7, + } + if interval: + result["median_ci95_ms"] = [0.9, 1.1] + return result + + +def _sample_results() -> dict[str, Any]: + absolute = { + name: {"total": _summary()} + for name in ("direct", "relay-minimal", "relay-file", "relay-otlp", "relay-guardrails") + } + comparisons = { + name: {"total": _summary(interval=True)} + for name in ( + "relay-minimal_vs_direct", + "relay-file_vs_direct", + "relay-otlp_vs_direct", + "file_exporter_vs_minimal", + "otlp_exporter_vs_minimal", + "relay-guardrails_vs_direct", + "guardrails_vs_minimal", + ) + } + process_absolute = { + "process_baseline": _summary(), + "relay-minimal": _summary(), + } + return { + "schema_version": 2, + "environment": { + "generated_at": "2026-08-05T12:00:00+00:00", + "git_commit": "0123456789abcdef", + "git_dirty": False, + "relay_version": "nemo-relay 0.1.0", + "platform": "test", + }, + "parameters": { + "tests": ["gateway", "hooks", "startup"], + "providers": ["openai"], + "modes": ["buffered"], + "samples": 5, + "payload_sizes": [4096], + "concurrency": [1], + "middleware": [{"name": "guardrails", "plugin_config": "/tmp/plugins.toml"}], + }, + "gateway": [ + { + "provider": "openai", + "mode": "buffered", + "payload_bytes": 4096, + "serialized_request_bytes": 4200, + "concurrency": 1, + "absolute": absolute, + "comparisons": comparisons, + } + ], + "hooks": { + "absolute": {"process_baseline": _summary(), "codex_minimal": _summary()}, + "comparisons": {"codex_minimal_vs_process_baseline": _summary(interval=True)}, + }, + "startup": { + "absolute": process_absolute, + "comparisons": {"relay-minimal_readiness_vs_process_baseline": _summary(interval=True)}, + }, + "exporter_delivery": {"atof_bytes": 8192, "otlp_requests": 4}, + } + + +class HtmlReportTests(unittest.TestCase): + def test_report_embeds_results_and_static_assets(self) -> None: + report = render_html_report(_sample_results()) + + self.assertIn("NeMo Relay Latency Report", report) + self.assertIn("

NeMo Relay Latency Report

", report) + self.assertNotIn("Coding-Agent Latency Report", report) + self.assertNotIn('class="eyebrow"', report) + self.assertNotIn('class="lede"', report) + self.assertIn('"git_commit":"0123456789abcdef"', report) + self.assertIn("Gateway Latency by Payload Size", report) + self.assertIn('"relay-file": "ATOF file exporter"', report) + self.assertIn('"relay-otlp": "OTLP exporter"', report) + self.assertIn("function drawLineChart", report) + self.assertNotIn("__BENCHMARK_DATA__", report) + self.assertNotIn("__BENCHMARK_STYLES__", report) + self.assertNotIn("__BENCHMARK_SCRIPT__", report) + + def test_report_escapes_script_terminators_in_embedded_data(self) -> None: + results = _sample_results() + results["environment"]["platform"] = "" + + report = render_html_report(results) + + self.assertNotIn("