diff --git a/.github/workflows/hardware.yml b/.github/workflows/hardware.yml index f81f655..489a920 100644 --- a/.github/workflows/hardware.yml +++ b/.github/workflows/hardware.yml @@ -17,12 +17,17 @@ jobs: - uses: taiki-e/install-action@just - run: just test-bpf-live - run: just test-cupti-live-cuda12 + - run: just test-nvtx-live-cuda12 - run: just test-cupti-live-cuda12-min - run: just test-injection-live-cuda12 - run: just test-multisource-live-cuda12 - run: just test-cupti-live + - run: just test-nvtx-live - run: just test-injection-live - run: just test-multisource-live + - run: just test-pytorch-live + - run: just test-pytorch-cuda-live - run: just benchmark-gpu - run: just benchmark-aggregate - run: just benchmark-multiprocess + - run: just benchmark-pytorch diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8381bfe..869c69d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -86,3 +86,7 @@ jobs: with: files: dist/*.tar.gz* generate_release_notes: true + - name: Verify public release archive + env: + XPROBE_RELEASE_REPOSITORY: ${{ github.repository }} + run: scripts/verify-public-release.sh "${GITHUB_REF_NAME#v}" diff --git a/AGENTS.md b/AGENTS.md index c408b15..0acc444 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,13 +47,12 @@ ## Agent workflow - Follow `skills/xprobe-measure-latency/SKILL.md` for measurement tasks. -- Classify a workload as CPU-only or GPU/mixed before choosing collectors. - `discover` is only for CUDA context holders; CPU-only work proceeds with its - selected PID and host selectors. For GPU/mixed work, progress from readiness - and baseline through a broad, representative bounded inventory, - evidence-based selector narrowing, validation, and one detailed bounded - measurement per stated hypothesis. Scope breadth and capture duration are - independent. Do not guess selectors. +- Choose the shortest Skill route supported by the evidence. Existing artifacts + do not require live readiness checks; known selectors may proceed directly to + validation and bounded measurement. Use broad-to-narrow inventory only when + selectors are unknown. Classify live work as CPU-only or GPU/mixed before + choosing collectors; `discover` is only for CUDA context holders. Scope + breadth and capture duration are independent. Do not guess selectors. - Read evidence pairs, artifact analysis, stream identity, collection quality, and profiler overhead before interpreting latency. Summed concurrent GPU duration is not wall time. @@ -86,4 +85,6 @@ - Run `just benchmark-aggregate` for aggregate inventory hot-path or capacity changes. - Run `just benchmark-multiprocess` for concurrent worker orchestration changes. +- Run `just benchmark-pytorch` for PyTorch workflow or framework-overhead + changes. - Use emoji conventional commits, for example `🐛 fix: restore target registers`. diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f38fd7..34eb397 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.20) -project(xprobe VERSION 0.4.0 LANGUAGES C) +project(xprobe VERSION 0.4.1 LANGUAGES C) include(CTest) diff --git a/Cargo.lock b/Cargo.lock index 5febe2a..cdcc629 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -461,7 +461,7 @@ dependencies = [ [[package]] name = "xprobe-cli" -version = "0.4.0" +version = "0.4.1" dependencies = [ "clap", "serde_json", @@ -474,7 +474,7 @@ dependencies = [ [[package]] name = "xprobe-collector" -version = "0.4.0" +version = "0.4.1" dependencies = [ "libbpf-rs", "serde_json", @@ -483,7 +483,7 @@ dependencies = [ [[package]] name = "xprobe-core" -version = "0.4.0" +version = "0.4.1" dependencies = [ "cpp_demangle", "nix", @@ -494,7 +494,7 @@ dependencies = [ [[package]] name = "xprobe-correlator" -version = "0.4.0" +version = "0.4.1" dependencies = [ "regex", "serde_json", @@ -503,7 +503,7 @@ dependencies = [ [[package]] name = "xprobe-exporter" -version = "0.4.0" +version = "0.4.1" dependencies = [ "serde_json", "xprobe-protocol", @@ -511,7 +511,7 @@ dependencies = [ [[package]] name = "xprobe-protocol" -version = "0.4.0" +version = "0.4.1" dependencies = [ "schemars", "serde", diff --git a/Cargo.toml b/Cargo.toml index 5a08b1f..393cac2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.4.0" +version = "0.4.1" edition = "2024" license = "Apache-2.0" repository = "https://github.com/itdevwu/xprobe" diff --git a/README.md b/README.md index da2d179..84baecb 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ only installation action required from the user: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.0/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ --global ``` diff --git a/benchmarks/pytorch/run.py b/benchmarks/pytorch/run.py new file mode 100755 index 0000000..9e2f4e9 --- /dev/null +++ b/benchmarks/pytorch/run.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import pathlib +import statistics +import subprocess +import sys +import tempfile +import time + + +ROUNDS = 3 +TIMED_SECONDS = 1.5 +CAPTURE_SECONDS = 1.5 + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--image") + parser.add_argument("--pytorch-env") + parser.add_argument("--inner", action="store_true") + parser.add_argument("--xprobe", default="/workspace/target/debug/xprobe") + args = parser.parse_args() + if args.inner: + run_inner(pathlib.Path(args.xprobe)) + else: + run_container(args) + + +def run_container(args: argparse.Namespace) -> None: + if not args.image: + raise SystemExit("--image is required") + workspace = pathlib.Path(__file__).resolve().parents[2] + python = "python3" + environment_arguments = [] + if args.pytorch_env: + pytorch_env = pathlib.Path(args.pytorch_env).resolve() + python = "/opt/xprobe-pytorch/bin/python" + environment_arguments = [ + "--volume", + f"{pytorch_env}:/opt/xprobe-pytorch:ro", + ] + completed = subprocess.run( + [ + "docker", + "run", + "--rm", + "--gpus", + "all", + "--cap-add", + "SYS_PTRACE", + "--security-opt", + "seccomp=unconfined", + "--volume", + f"{workspace}:/workspace:ro", + *environment_arguments, + "--workdir", + "/workspace", + args.image, + python, + "/workspace/benchmarks/pytorch/run.py", + "--inner", + ], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + sys.stdout.write(completed.stdout) + sys.stderr.write(completed.stderr) + raise SystemExit(completed.returncode) + lines = [line for line in completed.stdout.splitlines() if line.startswith("{")] + if not lines: + raise AssertionError(f"PyTorch benchmark emitted no JSON report:\n{completed.stdout}") + print(json.dumps(json.loads(lines[-1]), sort_keys=True)) + + +def run_inner(xprobe: pathlib.Path) -> None: + workspace = pathlib.Path("/workspace") + with tempfile.TemporaryDirectory(prefix="xprobe-pytorch-benchmark-") as directory: + root = pathlib.Path(directory) + agent = root / "libxprobe-cupti.so" + metrics = root / "metrics.json" + build_agent(workspace, agent) + + baseline = [run_timed(workspace, profile=False) for _ in range(ROUNDS)] + profiler = [run_timed(workspace, profile=True) for _ in range(ROUNDS)] + + workload = start_workload(workspace, metrics, agent) + try: + assert workload.stdout is not None + ready = workload.stdout.readline() + if not ready: + raise AssertionError("PyTorch benchmark workload exited before readiness") + workload_metadata = json.loads(ready) + wait_for_metrics(metrics) + idle_rates = [sample_rate(metrics, TIMED_SECONDS) for _ in range(ROUNDS)] + validation = validate(xprobe, workload.pid) + captures = [ + capture(xprobe, workload.pid, metrics, agent) for _ in range(ROUNDS) + ] + finally: + workload.terminate() + workload.wait(timeout=10) + + baseline_rate = statistics.median( + result["iterations_per_second"] for result in baseline + ) + idle_rate = statistics.median(idle_rates) + captured_rate = statistics.median(rate for rate, _ in captures) + profiler_rate = statistics.median( + result["iterations_per_second"] for result in profiler + ) + xprobe_ratio = baseline_rate / captured_rate + idle_ratio = baseline_rate / idle_rate + profiler_ratio = baseline_rate / profiler_rate + assert xprobe_ratio < 25.0, xprobe_ratio + assert profiler_ratio < 25.0, profiler_ratio + + capture_results = [result for _, result in captures] + for result in capture_results: + assert result["ok"] is True, result + assert result["status"] == "completed", result + assert result["collection"]["completeness"] == "complete", result + assert result["collection"]["dropped_events"] == 0, result + assert result["measurement"]["samples"]["matched"] > 0, result + assert result["correlation"]["confidence"] == "exact", result + + collection = capture_results[-1]["collection"] + print( + json.dumps( + { + "schema_version": "2.0", + "ok": True, + "environment": workload_metadata, + "workload": { + "operation": "torch.mm_256x256_and_stream_synchronize", + "rounds": ROUNDS, + "seconds_per_rate_round": TIMED_SECONDS, + }, + "throughput": { + "baseline_iterations_per_second": baseline_rate, + "idle_agent_iterations_per_second": idle_rate, + "xprobe_iterations_per_second": captured_rate, + "pytorch_profiler_iterations_per_second": profiler_rate, + }, + "overhead": { + "metric": "baseline_throughput_over_observed_throughput", + "idle_agent_ratio": idle_ratio, + "xprobe_ratio": xprobe_ratio, + "pytorch_profiler_ratio": profiler_ratio, + "project_target_ratio": 1.05, + "project_target_met": xprobe_ratio <= 1.05, + "profiler_subscriber_isolation": "separate_process", + }, + "validation": { + "agent_activation": validation["requirements"][ + "agent_activation" + ], + "policy": validation["policy_recommendation"]["policy"], + "valid": validation["valid"], + }, + "collection": { + "completeness": collection["completeness"], + "dropped_events": collection["dropped_events"], + "cupti": collection["cupti"], + "matched_samples": capture_results[-1]["measurement"]["samples"][ + "matched" + ], + "unmatched_start_samples": capture_results[-1]["measurement"][ + "samples" + ]["unmatched_start"], + "unmatched_end_samples": capture_results[-1]["measurement"][ + "samples" + ]["unmatched_end"], + "ambiguous_samples": capture_results[-1]["measurement"][ + "samples" + ]["ambiguous"], + }, + }, + sort_keys=True, + ) + ) + + +def run_timed(workspace: pathlib.Path, profile: bool) -> dict: + command = [ + sys.executable, + str(workspace / "benchmarks/pytorch/workload.py"), + "--timed", + "--seconds", + str(TIMED_SECONDS), + ] + if profile: + command.append("--profile") + completed = subprocess.run(command, check=True, capture_output=True, text=True) + return json.loads(completed.stdout) + + +def start_workload( + workspace: pathlib.Path, + metrics: pathlib.Path, + agent: pathlib.Path, +) -> subprocess.Popen[str]: + environment = os.environ.copy() + environment["NVTX_INJECTION64_PATH"] = str(agent) + return subprocess.Popen( + [ + sys.executable, + "-u", + str(workspace / "benchmarks/pytorch/workload.py"), + "--serve", + "--metrics", + str(metrics), + ], + env=environment, + stdout=subprocess.PIPE, + text=True, + ) + + +def wait_for_metrics(path: pathlib.Path) -> None: + deadline = time.monotonic() + 10 + while not path.is_file(): + if time.monotonic() >= deadline: + raise TimeoutError("timed out waiting for PyTorch benchmark metrics") + time.sleep(0.01) + + +def read_metrics(path: pathlib.Path) -> dict: + return json.loads(path.read_text()) + + +def sample_rate(path: pathlib.Path, seconds: float) -> float: + before = read_metrics(path) + time.sleep(seconds) + after = read_metrics(path) + count = after["count"] - before["count"] + elapsed_ns = after["timestamp_ns"] - before["timestamp_ns"] + assert count > 0 and elapsed_ns > 0, {"before": before, "after": after} + return count * 1_000_000_000 / elapsed_ns + + +def validate(xprobe: pathlib.Path, pid: int) -> dict: + result = run_xprobe( + xprobe, + [ + "validate", + "--pid", + str(pid), + "--from", + "cuda:runtime_api:cudaStreamSynchronize:entry", + "--to", + "cuda:runtime_api:cudaStreamSynchronize:exit", + "--match", + "exact", + ], + ) + assert result["valid"] is True, result + assert result["target"]["pid"] == pid, result + assert result["requirements"]["agent_activation"] == "already_loaded", result + assert result["policy_recommendation"]["policy"] == "exact", result + return result + + +def capture( + xprobe: pathlib.Path, + pid: int, + metrics: pathlib.Path, + agent: pathlib.Path, +) -> tuple[float, dict]: + process = subprocess.Popen( + [ + str(xprobe), + "measure", + "--pid", + str(pid), + "--agent", + str(agent), + "--from", + "cuda:runtime_api:cudaStreamSynchronize:entry", + "--to", + "cuda:runtime_api:cudaStreamSynchronize:exit", + "--match", + "exact", + "--duration-ms", + str(int(CAPTURE_SECONDS * 1000)), + "--max-events", + "8192", + "--timeout-ms", + "10000", + "--json", + "--non-interactive", + "--no-color", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + time.sleep(0.2) + rate = sample_rate(metrics, 1.0) + stdout, stderr = process.communicate(timeout=15) + if process.returncode != 0: + raise AssertionError( + f"xprobe benchmark capture failed:\n{stdout}\n{stderr}" + ) + return rate, json.loads(stdout) + + +def run_xprobe(xprobe: pathlib.Path, arguments: list[str]) -> dict: + completed = subprocess.run( + [ + str(xprobe), + *arguments, + "--json", + "--non-interactive", + "--no-color", + ], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + raise AssertionError( + f"xprobe benchmark command failed:\n{completed.stdout}\n{completed.stderr}" + ) + return json.loads(completed.stdout) + + +def build_agent(workspace: pathlib.Path, output: pathlib.Path) -> None: + subprocess.run( + [ + "gcc", + "-std=c11", + "-D_GNU_SOURCE", + "-DXPROBE_HAS_CUPTI=1", + "-fPIC", + "-shared", + "-pthread", + "-O2", + "-Wall", + "-Wextra", + "-Wpedantic", + "-Werror", + f"-I{workspace / 'cupti/include'}", + "-isystem", + "/usr/local/cuda/include", + str(workspace / "cupti/src/cupti_agent.c"), + "-L/usr/local/cuda/lib64", + "-Wl,-rpath,/usr/local/cuda/lib64", + "-lcupti", + "-o", + str(output), + ], + check=True, + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/pytorch/workload.py b/benchmarks/pytorch/workload.py new file mode 100755 index 0000000..ca312a4 --- /dev/null +++ b/benchmarks/pytorch/workload.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +import argparse +import contextlib +import json +import pathlib +import time + + +def initialize(): + import torch + + torch.cuda.init() + device = torch.device("cuda") + stream = torch.cuda.current_stream() + left = torch.randn(256, 256, device=device) + right = torch.randn(256, 256, device=device) + for _ in range(100): + torch.mm(left, right) + stream.synchronize() + torch.cuda.nvtx.range_push("xprobe_pytorch_benchmark_init") + torch.cuda.nvtx.range_pop() + return torch, stream, left, right + + +def iteration(torch, stream, left, right) -> None: + torch.mm(left, right) + stream.synchronize() + time.sleep(0.001) + + +def metadata(torch) -> dict: + return { + "cuda": torch.version.cuda, + "device": torch.cuda.get_device_name(), + "pytorch": torch.__version__, + } + + +def run_timed(seconds: float, profile: bool) -> None: + torch, stream, left, right = initialize() + profiler = ( + torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ] + ) + if profile + else contextlib.nullcontext() + ) + count = 0 + with profiler: + started_ns = time.monotonic_ns() + deadline_ns = started_ns + int(seconds * 1_000_000_000) + while time.monotonic_ns() < deadline_ns: + iteration(torch, stream, left, right) + count += 1 + if profile: + profiler.step() + finished_ns = time.monotonic_ns() + elapsed_ns = finished_ns - started_ns + print( + json.dumps( + { + **metadata(torch), + "elapsed_ns": elapsed_ns, + "iterations": count, + "iterations_per_second": count * 1_000_000_000 / elapsed_ns, + "profile": profile, + }, + sort_keys=True, + ), + flush=True, + ) + + +def write_metrics(path: pathlib.Path, count: int) -> None: + replacement = path.with_name(f"{path.name}.next") + replacement.write_text( + json.dumps( + { + "count": count, + "timestamp_ns": time.monotonic_ns(), + } + ) + ) + replacement.replace(path) + + +def serve(metrics_path: pathlib.Path) -> None: + torch, stream, left, right = initialize() + count = 0 + write_metrics(metrics_path, count) + print(json.dumps(metadata(torch), sort_keys=True), flush=True) + while True: + iteration(torch, stream, left, right) + count += 1 + if count % 8 == 0: + write_metrics(metrics_path, count) + + +def main() -> None: + parser = argparse.ArgumentParser() + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--serve", action="store_true") + mode.add_argument("--timed", action="store_true") + parser.add_argument("--metrics", type=pathlib.Path) + parser.add_argument("--seconds", type=float, default=1.5) + parser.add_argument("--profile", action="store_true") + args = parser.parse_args() + if args.serve: + if args.metrics is None: + raise SystemExit("--metrics is required with --serve") + serve(args.metrics) + else: + run_timed(args.seconds, args.profile) + + +if __name__ == "__main__": + main() diff --git a/docs/agent-integration.md b/docs/agent-integration.md index 5c4333f..af595f2 100644 --- a/docs/agent-integration.md +++ b/docs/agent-integration.md @@ -17,7 +17,7 @@ repairs the matching xprobe CLI itself: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.0/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ --global ``` @@ -26,9 +26,10 @@ For automation, add `--agent codex|claude-code|cursor --copy --yes`. Omit install its whole directory so its references, examples, and analysis script remain available. -The Skill checks, installs, and verifies the CLI before it first runs `doctor`. -It then maps a representative workload broadly, uses artifact evidence to narrow -selectors, and includes `scripts/analyze_trace.py` for deterministic kernel, +The Skill routes setup, completed-artifact analysis, known-boundary measurement, +unknown CPU/GPU investigation, and multi-process work independently. It checks +or installs the CLI only for live work, uses broad inventory only when selectors +are unknown, and includes `scripts/analyze_trace.py` for deterministic kernel, copy, overlap, stream, and gap summaries. The xprobe repository tests installation with `skills` CLI 1.5.20 in isolated home directories. This pinned test protects released behavior while the @@ -51,8 +52,8 @@ just test-skill-install The test requires the visible command set to be exactly `doctor`, `discover`, `validate`, and `measure`. It invokes the first three in strict JSON mode, checks injection requirements, verifies schemas, exercises the bundled trace -analyzer, and checks that the Skill uses only the four-command bounded workflow -and inspects result quality/evidence. +analyzer, and checks adaptive task routing, bounded live collection, mutation +guards, and result quality/evidence. The installation test uses the real third-party CLI in isolated home directories and verifies byte-for-byte copies for Codex, Claude Code, and Cursor. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index de13de5..162789c 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -61,3 +61,18 @@ Its JSON keeps validation, command timing, quality, artifact metadata, and baseline-versus-collection throughput per worker. Perturbation is reported without a fixed pass ratio because shared GPU scheduling and first injection cost vary by workload and worker count. + +Run the framework-level PyTorch benchmark with: + +```bash +just benchmark-pytorch +``` + +It measures the same warmed-up `torch.mm` plus stream-synchronization workload +without an Agent, with the Agent idle, during a bounded xprobe capture, and with +PyTorch Profiler. PyTorch Profiler runs in a separate process so the two CUPTI +subscribers never coexist. The benchmark reports throughput ratios, validation, +matched/unmatched/ambiguous samples, completeness, drops, retained capacity and +buffer utilization. A broad `25x` ceiling catches broken instrumentation; the +reported `1.05` project target remains an observation rather than a compatibility +promise. diff --git a/docs/development.md b/docs/development.md index b2aa794..b42d717 100644 --- a/docs/development.md +++ b/docs/development.md @@ -43,6 +43,10 @@ versioned installer. It also rejects CLI or Agent ELF dependencies above runner change from silently raising it. The archive test also accepts a mocked glibc 2.34 runtime, rejects 2.33, installs into a temporary prefix, runs the packaged binary, verifies both Agents and shared resources, and uninstalls it. +After publishing the GitHub release, `scripts/verify-public-release.sh` downloads +the public archive and checksum again, repeats the installation test, and +inspects every shipped ELF. This final gate verifies the artifact users can +actually download rather than the workflow's local copy. ## eBPF tests @@ -65,27 +69,32 @@ requires Docker daemon access and grants the container `BPF`, syscalls. It does not use `--privileged`, does not require GPU access, mounts the workspace read-only, and removes the container after the test. -Resolve real CPython, native extension, and libtorch C++ symbols with a Mamba -environment containing PyTorch: +Resolve real CPython, native extension, and libtorch C++ symbols with a local +Python environment containing PyTorch: ```bash PYTORCH_PYTHON=/path/to/env/bin/python just test-pytorch-symbols ``` Run the corresponding live `torch.mm` entry/return measurement in the pinned -BPF container: +NVIDIA PyTorch container: ```bash -PYTORCH_ENV=/path/to/env just test-pytorch-live +just test-pytorch-live ``` -With a CUDA-enabled PyTorch environment, run eager matrix multiplication, -convolution, compiled Triton, bidirectional transfer, selected-kernel, and +By default the live recipes use a pinned NVIDIA PyTorch image with Ubuntu 24.04 +and CUDA 12.9, so hardware CI does not depend on a runner-local Mamba +environment. During local development, set `PYTORCH_ENV=/path/to/env` to mount +an existing environment into the already-pinned CUDA fixtures instead of +pulling the PyTorch image. Run eager matrix multiplication, convolution, +compiled Triton, bidirectional transfer, selected-kernel, and stream-synchronization profiling on the local GPU. The test also wraps an eager operation in an NVTX range and verifies exact range-ID matching: ```bash -PYTORCH_ENV=/path/to/env just test-pytorch-cuda-live +just test-pytorch-cuda-live +just benchmark-pytorch ``` ## GPU checks diff --git a/docs/installation.md b/docs/installation.md index 9e5657c..3faeff3 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -13,7 +13,7 @@ user needs to run: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.0/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ --global ``` @@ -28,7 +28,7 @@ The versioned bootstrap installs to `~/.local` without root access: ```bash curl --proto '=https' --tlsv1.2 -fsSL \ - https://raw.githubusercontent.com/itdevwu/xprobe/v0.4.0/install.sh | sh + https://raw.githubusercontent.com/itdevwu/xprobe/v0.4.1/install.sh | sh ``` The bootstrap downloads the release archive and its SHA256 file, verifies the @@ -47,7 +47,7 @@ prefix, download the script and pass `--prefix`: ```bash curl --proto '=https' --tlsv1.2 -fsSLO \ - https://raw.githubusercontent.com/itdevwu/xprobe/v0.4.0/install.sh + https://raw.githubusercontent.com/itdevwu/xprobe/v0.4.1/install.sh sh install.sh --prefix /opt/xprobe ``` @@ -59,7 +59,7 @@ script with `sudo`. The installer never elevates privileges itself. For a fully explicit archive workflow: ```bash -version=0.4.0 +version=0.4.1 base=https://github.com/itdevwu/xprobe/releases/download/v$version archive=xprobe-$version-linux-x86_64.tar.gz @@ -92,7 +92,7 @@ missing or damaged, manually install the complete version-matched directory: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.0/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ --global ``` @@ -101,7 +101,7 @@ installation names the target explicitly: ```bash npx --yes skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.0/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ --agent codex --global --copy --yes ``` diff --git a/install.sh b/install.sh index 70935f5..e631737 100755 --- a/install.sh +++ b/install.sh @@ -2,7 +2,7 @@ set -eu repository=${XPROBE_REPOSITORY:-itdevwu/xprobe} -version=${XPROBE_VERSION:-0.4.0} +version=${XPROBE_VERSION:-0.4.1} if [ -n "${XPROBE_PREFIX:-}" ]; then prefix=$XPROBE_PREFIX elif [ -n "${HOME:-}" ]; then @@ -19,7 +19,7 @@ Install a released xprobe binary and its CUDA Agents. Usage: install.sh [--version VERSION] [--prefix DIR] [--uninstall] Options: - --version VERSION Release to install (default: 0.4.0) + --version VERSION Release to install (default: 0.4.1) --prefix DIR Installation prefix (default: $HOME/.local) --uninstall Remove xprobe from the selected prefix -h, --help Show this help diff --git a/justfile b/justfile index a06ca28..eaa145d 100644 --- a/justfile +++ b/justfile @@ -5,6 +5,7 @@ cuda12_devel_image := "nvidia/cuda:12.9.1-devel-ubuntu24.04@sha256:020bc241a6287 cuda12_compat_build_image := "nvidia/cuda:12.9.1-devel-ubuntu22.04@sha256:bd4e2680a261c212f1e2fea241606f71497dc67a417f73175d794ec8212b5ba8" cuda12_min_devel_image := "nvidia/cuda:12.0.1-devel-ubuntu22.04@sha256:0632323ec456b33654d489f3ddd336f3b3ea1c87e6421a91a37f6768e659f08c" cuda13_devel_image := "nvcr.io/nvidia/cuda:13.3.0-devel-ubuntu24.04@sha256:69e9e39eb8fe2cda271654a0f5eac2f1bb946b2fb9c460eb19c7c3c155f4e64e" +pytorch_image := "nvcr.io/nvidia/pytorch:25.06-py3@sha256:3cb18e2c438db8af2d3a659ca27fac5da328640261c38c48a34edcd223c38af9" default: @just --list @@ -31,7 +32,7 @@ test-skill-install: test-install: sh -n install.sh tests/install/test_install.sh - bash -n scripts/check-glibc-ceiling.sh tests/install/test_glibc_ceiling.sh + bash -n scripts/check-glibc-ceiling.sh scripts/verify-public-release.sh tests/install/test_glibc_ceiling.sh tests/install/test_glibc_ceiling.sh tests/install/test_install.sh @@ -73,10 +74,10 @@ test-pytorch-symbols: build python3 tests/integration/test_pytorch_symbols.py --python "${PYTORCH_PYTHON:?set PYTORCH_PYTHON to a Python with torch}" test-pytorch-live: build - python3 tests/integration/test_pytorch.py "{{cuda_smoke_image}}" "${PYTORCH_ENV:?set PYTORCH_ENV to a Mamba environment with torch}" + if [[ -n "${PYTORCH_ENV:-}" ]]; then python3 tests/integration/test_pytorch.py "{{cuda_smoke_image}}" "${PYTORCH_ENV}"; else python3 tests/integration/test_pytorch.py "{{pytorch_image}}"; fi test-pytorch-cuda-live: build - python3 tests/integration/test_pytorch_cuda.py --image "{{cuda12_devel_image}}" --pytorch-env "${PYTORCH_ENV:?set PYTORCH_ENV to a Mamba environment with torch}" + if [[ -n "${PYTORCH_ENV:-}" ]]; then python3 tests/integration/test_pytorch_cuda.py --image "{{cuda12_devel_image}}" --pytorch-env "${PYTORCH_ENV}"; else python3 tests/integration/test_pytorch_cuda.py --image "{{pytorch_image}}"; fi test-nvtx-live: build python3 tests/integration/test_nvtx.py --image "{{cuda13_devel_image}}" @@ -93,6 +94,9 @@ benchmark-aggregate: benchmark-multiprocess: python3 benchmarks/cuda-multiprocess/run.py "{{cuda13_devel_image}}" +benchmark-pytorch: build + if [[ -n "${PYTORCH_ENV:-}" ]]; then python3 benchmarks/pytorch/run.py --image "{{cuda12_devel_image}}" --pytorch-env "${PYTORCH_ENV}"; else python3 benchmarks/pytorch/run.py --image "{{pytorch_image}}"; fi + fmt: cargo fmt --all diff --git a/scripts/verify-public-release.sh b/scripts/verify-public-release.sh new file mode 100755 index 0000000..7315f3e --- /dev/null +++ b/scripts/verify-public-release.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: verify-public-release.sh " >&2 + exit 2 +fi + +version=${1#v} +[[ ${version} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { + echo "invalid release version: ${version}" >&2 + exit 2 +} + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +repository=${XPROBE_RELEASE_REPOSITORY:-itdevwu/xprobe} +package=xprobe-${version}-linux-x86_64 +release_url=https://github.com/${repository}/releases/download/v${version} +temporary=$(mktemp -d) +trap 'rm -rf "${temporary}"' EXIT HUP INT TERM + +archive=${temporary}/${package}.tar.gz +checksum=${archive}.sha256 + +curl --fail --location --proto '=https' --tlsv1.2 \ + --retry 5 --retry-delay 2 --retry-all-errors \ + --output "${archive}" "${release_url}/${package}.tar.gz" +curl --fail --location --proto '=https' --tlsv1.2 \ + --retry 5 --retry-delay 2 --retry-all-errors \ + --output "${checksum}" "${release_url}/${package}.tar.gz.sha256" +( + cd "${temporary}" + sha256sum --check "$(basename "${checksum}")" +) + +"${root}/tests/install/test_install.sh" "${archive}" + +extracted=${temporary}/extracted +mkdir -p "${extracted}" +tar -xzf "${archive}" -C "${extracted}" +package_root=${extracted}/${package} +[[ -d ${package_root} ]] || { + echo "release archive does not contain ${package}" >&2 + exit 1 +} + +cli=${package_root}/bin/xprobe +cuda12=${package_root}/lib/xprobe/cuda12/libxprobe-cupti.so +cuda13=${package_root}/lib/xprobe/cuda13/libxprobe-cupti.so +for binary in "${cli}" "${cuda12}" "${cuda13}"; do + [[ -f ${binary} ]] || { echo "release ELF is missing: ${binary}" >&2; exit 1; } +done + +mapfile -d '' candidates < <(find "${package_root}" -type f -print0) +elfs=() +for candidate in "${candidates[@]}"; do + if readelf -h "${candidate}" >/dev/null 2>&1; then + elfs+=("${candidate}") + fi +done +[[ ${#elfs[@]} -eq 3 ]] || { + echo "expected 3 shipped ELFs, found ${#elfs[@]}" >&2 + printf '%s\n' "${elfs[@]}" >&2 + exit 1 +} +for binary in "${elfs[@]}"; do + "${root}/scripts/check-glibc-ceiling.sh" "${binary}" 2.34 +done + +verify_agent() { + local agent=$1 + local major=$2 + local dynamic + dynamic=$(readelf -d "${agent}") + grep -Fq "Shared library: [libcupti.so.${major}]" <<<"${dynamic}" || { + echo "${agent} is not linked to libcupti.so.${major}" >&2 + exit 1 + } + if grep -Eq '\((RPATH|RUNPATH)\)' <<<"${dynamic}"; then + echo "${agent} contains a build-time RPATH or RUNPATH" >&2 + exit 1 + fi +} + +verify_agent "${cuda12}" 12 +verify_agent "${cuda13}" 13 +[[ $("${cli}" --version) == "xprobe ${version}" ]] || { + echo "release CLI version does not match ${version}" >&2 + exit 1 +} + +printf 'Verified public xprobe %s archive, installation, and 3 shipped ELFs\n' \ + "${version}" diff --git a/skills/xprobe-measure-latency/SKILL.md b/skills/xprobe-measure-latency/SKILL.md index a437d98..7d37fc1 100644 --- a/skills/xprobe-measure-latency/SKILL.md +++ b/skills/xprobe-measure-latency/SKILL.md @@ -1,110 +1,104 @@ --- name: xprobe-measure-latency -description: Investigate unknown Linux CPU/CUDA latency with bounded xprobe captures, derive selectors from trace evidence, analyze multi-stream GPU activity, and measure host functions, CUDA APIs, kernels, memory copies, memory sets, and event-to-event gaps. Use when an agent must profile a running process, narrow a performance regression, inspect an xprobe JSONL artifact, or decide when duration evidence should hand off to Nsight Compute or another microarchitectural profiler. +description: Profile live or recorded Linux CPU and NVIDIA CUDA workloads with bounded xprobe evidence. Use when an agent needs to install or repair xprobe, inspect an existing xprobe JSONL artifact, inventory unknown CPU/GPU activity, validate and measure a known function, syscall, CUDA API, kernel, transfer, synchronization point, NVTX range, or event-to-event latency, compare selected worker processes, investigate a performance regression, or decide when to hand an isolated kernel or host span to another profiler. --- -# Investigate latency with xprobe - -Use JSON mode to make a wide, coarse workload inventory before measuring one -boundary narrowly and finely. Read [references/setup.md](references/setup.md) -to install or repair the CLI and this Skill. Read -[references/investigation.md](references/investigation.md) before profiling an -unknown workload. Read [references/result-quality.md](references/result-quality.md) -before interpreting correlation, clocks, concurrency, or overhead. The exact -CLI and selector syntax is in [references/cli-contract.md](references/cli-contract.md). -For more than one selected process, follow -[references/multi-process.md](references/multi-process.md). - -## Workflow - -1. Run `xprobe --version`. When the command is absent or not 0.4.0, read - [references/setup.md](references/setup.md) and install or repair the CLI - yourself; do not ask the user to perform a separate CLI installation. - Confirm every JSON response has `schema_version: "2.0"`; do not assume a - pre-0.3 or future protocol is compatible with this Skill. -2. Establish an application-level latency baseline, process readiness, and - warmup. Classify the workload before selecting collectors: for CPU-only work, - choose the owning PID and skip CUDA discovery; for GPU or mixed work, wait - for CUDA context creation and JIT warmup before discovery. Keep a repeatable - request or batch trigger ready for the measurement window. -3. Run `xprobe doctor --json --non-interactive --no-color`. Check individual - capabilities; `ok: true` only means diagnosis completed. -4. For GPU or mixed work, run `xprobe discover --pid ROOT_PID --limit 200 --json - --non-interactive --no-color`. It returns NVML-confirmed CUDA context holders - under that process tree. Choose a worker from workload, PID/start-time, - command line, and GPU UUID evidence. When several ranks are relevant, retain - every selected PID plus process start time and use the multi-process workflow. - For CPU-only work, do not run `discover`; continue with the selected process - PID. -5. Map GPU or mixed work before choosing a name. Validate broad kernel, memcpy, - or memset activity endpoints, then collect one bounded, representative coarse - inventory per event family with `measure --aggregate --duration-ms ...`. - For CPU-only work, use existing application evidence or a bounded system - summary to choose a function, named syscall, or tracepoint family before - detailed collection; do not require CUDA or CUPTI, and do not begin with an - unfiltered high-rate raw tracepoint. Scope breadth and collection duration are - independent: keep the selector broad where a bounded aggregate exists, choose - a duration that covers the workload cycle being diagnosed, and give - `--max-groups` headroom. For defensibly homogeneous workers, inventory one - representative worker and apply its evidence-derived narrow selector to all - selected workers. -6. Use aggregate names, selector hints, counts, duration totals and bounds, and - transfer bytes to form one narrow hypothesis. For an exact GPU artifact, run - `scripts/analyze_trace.py` and use launch variants, stream distribution, busy - union, overlap factor, and adjacent gaps. Read - [references/trace-analysis.md](references/trace-analysis.md) when interpreting - the report. When the application already marks the narrowed operation with a - bounded ASCII NVTX range, that range can provide exact application-level - start/end boundaries. For CPU-only work, use resolved host selectors or - filtered syscall/tracepoint evidence to form the hypothesis instead. -7. Run one read-only `xprobe validate` per selected worker. Compare every - response target with the PID plus process start time retained from discovery - before mutation, and stop that worker when `valid` is false. If - `agent_activation` is `injection_required`, disclose that `measure` will - ptrace the target and leave the CUPTI shared object mapped. Use - `policy_recommendation` explicitly; xprobe never changes policy for the - caller. If it is `startup_required`, restart that worker with - `NVTX_INJECTION64_PATH` pointing to the matching xprobe CUPTI Agent before - its first NVTX call, reacquire PID plus start time, and validate again. - Online injection cannot retrofit NVTX dispatch into an initialized process. -8. Run one bounded `xprobe measure` for that hypothesis. Set samples or duration, - timeout, and max-events; write `--events-out` when the capture may need audit - or offline re-correlation. Use a versioned `--spec FILE` containing the stable - target identity. For multiple workers, launch the independent calls - concurrently, preserve per-worker outputs and failures, and never correlate - across process artifacts. -9. Check `status`, matched/unmatched/ambiguous/dropped counts, collection - completeness, buffer utilization, clock alignment, estimated error, - correlation method/confidence/score, warnings, and every evidence pair. -10. Repeat only with a stated reason: select another event family, narrow the - selector, select another worker or stream, change an explicitly compatible - policy, or test the next boundary. - Recheck application latency after profiling and report observed overhead. - -For completed captures, replace `--pid` with one or more `--input` arguments. -Begin with the [coarse kernel inventory](examples/coarse-kernel-inventory.json) -or [coarse memcpy inventory](examples/coarse-memcpy-inventory.json), then use the -[kernel duration](examples/kernel-duration.json), -[same-stream gap](examples/same-stream-kernel-gap.json), -[host span](examples/host-function-span.json), and -[syscall duration](examples/syscall-duration.json), -[memcpy duration](examples/memcpy-duration.json) specs, plus the -[CUDA synchronization API](examples/cuda-api-duration.json) shape, after -replacing target identity and selectors. Each bounded call answers one -hypothesis; orchestration remains the agent framework's responsibility. - -## Stop conditions - -- Stop on target reuse, permission failure, invalid selectors, unavailable - collectors, drops, incomplete capture, unknown clock alignment, or unexamined - ambiguity. Treat an NVTX ARM-time feature mismatch as a startup failure rather - than retrying online injection. Read structured `details`, `hints`, and any - failed-capture artifact. -- Do not claim request causality from `first-after` or `nearest`. Do not compare - or sum events across streams as if they were serial. -- Stop using xprobe once evidence isolates time inside one kernel. Kernel - duration cannot explain warp stalls, cache misses, occupancy, instruction mix, - or Tensor Core utilization; hand that question to NCU or PC sampling. -- Avoid continuous or repeated exploratory capture in one production process. - Use representative bounded inventories, narrow formal measurements, and a - post-profile baseline. +# Profile workloads with xprobe + +Choose the shortest route that answers the user's question. Do not run the full +investigation playbook when the target, selectors, or completed artifact already +provide the missing evidence. + +## Route the task + +- **Existing artifact**: Read [references/trace-analysis.md](references/trace-analysis.md) + and [references/result-quality.md](references/result-quality.md). Analyze the + artifact directly or use `measure --input` to test compatible selectors or a + policy. Skip installation, `doctor`, `discover`, and live attachment unless a + separate live capture is actually needed. +- **Known live boundary**: Read [references/cli-contract.md](references/cli-contract.md) + and [references/result-quality.md](references/result-quality.md). Confirm the + target identity, validate the supplied selectors, and run a bounded measure. + Do not run a broad inventory solely to satisfy a checklist. +- **Unknown CPU workload**: Read + [references/investigation.md](references/investigation.md), classify the + suspected host boundary, and narrow from application, symbol, syscall, or + tracepoint evidence. Do not run CUDA discovery. +- **Unknown GPU or mixed workload**: Read + [references/investigation.md](references/investigation.md). Establish + readiness, discover CUDA context holders, collect only the broad bounded + inventories needed by the question, derive selectors from evidence, then + validate and measure narrowly. +- **Multiple processes**: Also read + [references/multi-process.md](references/multi-process.md). Select relevant + PID/start-time identities and run independent bounded commands concurrently + when aligned capture windows matter. Keep every result and artifact separate. +- **Setup or repair**: Read [references/setup.md](references/setup.md) only when + live commands are needed and the CLI is absent, incompatible, or unhealthy. + A completed-artifact analysis does not require a local collector. + +For live work, classify the selected path as CPU-only or GPU/mixed before +choosing collectors. Run `doctor` when capability is unknown or a command +reports an environment failure; it is not a prerequisite for every valid +offline or already-diagnosed workflow. + +## Preserve these invariants + +- Use JSON mode and require schema version `2.0`. Treat malformed output and + unknown schema versions as errors. +- Identify each live target by PID plus procfs start time. Recheck the identity + around validation and attachment; never substitute a newly observed PID. +- Run read-only `validate` before every live measurement or target mutation. + Use its explicit policy recommendation, but never change policy silently. +- Bound every capture by samples or duration, timeout, and exact-event or + aggregate-group capacity. Scope breadth and capture duration are independent. +- When validation reports `injection_required`, disclose that `measure` will + ptrace the process and leave the CUPTI shared object mapped. When it reports + `startup_required` for NVTX, restart with the matching Agent before the first + NVTX call; online injection cannot retrofit initialized NVTX dispatch. +- Inspect status, collection completeness, buffer utilization, unmatched, + ambiguous, and dropped counts, clock alignment and estimated error, + correlation method, confidence, and every evidence pair before interpreting + a result. +- Keep stream and process identity in every claim. Summed concurrent GPU + duration is not wall time; temporal correlation is not exact causality. + +## Choose collection depth from evidence + +Use broad-to-narrow collection when selectors are unknown. Keep the broad scope +representative but collect aggregate kernel, memcpy, or memset families only +when they could answer the current question. Use selector hints, counts, +duration totals, transfer bytes, and bounds to state a narrower hypothesis. + +When a trustworthy selector is supplied by the user, application, an NVTX +range, a previous artifact, or a matching build's symbol inspection, validate +it directly. A failed validation is evidence to revise the selector; it is not +a reason to run unrelated inventories. + +Use one bounded live capture per stated hypothesis when source activation or +capture windows differ. A single exact Event JSONL artifact may support several +offline correlations without reattaching. Independent worker captures or +non-conflicting hypotheses may run concurrently when the caller can preserve +their bounds, outputs, failures, and perturbation separately. + +Record an application baseline when the question concerns a regression, +slowdown, or profiler overhead. Warm up readiness-sensitive framework/JIT work +before selecting a representative window. Do not require a new baseline for +schema validation or a purely offline artifact question. + +For exact GPU artifacts, run `scripts/analyze_trace.py` and inspect launch +variants, stream distribution, `busy_union_ns`, overlap factor, and adjacent +gaps. Aggregate output has no event ordering and cannot be re-correlated. + +## Stop or hand off deliberately + +Stop on target reuse, permission failure, invalid selectors, unavailable +required collectors, drops, incomplete capture, or a clock/correlation problem +that invalidates the intended claim. Preserve failed-capture artifacts and +structured details instead of reporting partial success. A quality limitation +that does not affect the requested same-domain claim may be reported explicitly +rather than treated as a universal stop condition. + +Stop using xprobe once evidence isolates unexplained time inside one kernel; +use NCU or PC sampling for microarchitectural behavior. Use a CPU sampling +profiler when the unresolved time is inside an uninstrumented host span. diff --git a/skills/xprobe-measure-latency/agents/openai.yaml b/skills/xprobe-measure-latency/agents/openai.yaml index 1619cb2..db83b67 100644 --- a/skills/xprobe-measure-latency/agents/openai.yaml +++ b/skills/xprobe-measure-latency/agents/openai.yaml @@ -1,4 +1,4 @@ interface: - display_name: "Xprobe Latency Measurement" - short_description: "Route CPU and GPU latency investigations" - default_prompt: "Use $xprobe-measure-latency to classify this workload as CPU-only or GPU/mixed, then measure the suspected latency boundary." + display_name: "Xprobe Workload Profiling" + short_description: "Route bounded CPU and GPU profiling tasks" + default_prompt: "Use $xprobe-measure-latency to choose the shortest evidence-based route for this profiling task." diff --git a/skills/xprobe-measure-latency/references/setup.md b/skills/xprobe-measure-latency/references/setup.md index 5eb5e76..cc20863 100644 --- a/skills/xprobe-measure-latency/references/setup.md +++ b/skills/xprobe-measure-latency/references/setup.md @@ -7,8 +7,10 @@ command unless their environment prevents the agent from writing a usable prefix ## Check and bootstrap xprobe -Check the executable first. Continue only when it reports 0.4.0; otherwise run -the bootstrap below: +For live work, check the executable first. This Skill supports xprobe `0.4.x` +with schema version `2.0`; install the current release when the CLI is absent, +outside that range, or fails its required capability checks. Offline analysis +of an existing schema-v2 artifact does not require an installed CLI. ```bash if command -v xprobe >/dev/null 2>&1; then @@ -21,7 +23,7 @@ installing under `~/.local`: ```bash curl --proto '=https' --tlsv1.2 -fsSL \ - https://raw.githubusercontent.com/itdevwu/xprobe/v0.4.0/install.sh \ + https://raw.githubusercontent.com/itdevwu/xprobe/v0.4.1/install.sh \ -o /tmp/xprobe-install.sh sh /tmp/xprobe-install.sh export PATH="$HOME/.local/bin:$PATH" @@ -44,7 +46,7 @@ host glibc instead. This is a local-use fallback, not permission to weaken the release package's `GLIBC_2.34` ceiling. ```bash -git clone --depth 1 --branch v0.4.0 https://github.com/itdevwu/xprobe.git +git clone --depth 1 --branch v0.4.1 https://github.com/itdevwu/xprobe.git cd xprobe mamba env create --file environment.yml mamba run -n xprobe-dev just build @@ -82,12 +84,12 @@ online injection as a fallback for an already initialized NVTX process. ## Repair the Skill only when needed The user normally installed this Skill before invoking the agent. When its files -are missing or the version is not 0.4.0, install the complete version-matched -directory through the Agent Skills CLI: +are missing or incompatible with xprobe `0.4.x`, install the complete current +release directory through the Agent Skills CLI: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.0/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ --global ``` @@ -95,7 +97,7 @@ For non-interactive automation, select the host explicitly: ```bash npx --yes skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.4.0/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.4.1/skills/xprobe-measure-latency \ --agent codex --global --copy --yes ``` diff --git a/tests/agent-contract/test_contract.py b/tests/agent-contract/test_contract.py index 9d5672c..83b42cb 100755 --- a/tests/agent-contract/test_contract.py +++ b/tests/agent-contract/test_contract.py @@ -50,16 +50,34 @@ def check_skill(workspace: pathlib.Path) -> None: assert re.search(r"^description: .+", frontmatter.group(1), re.MULTILINE) normalized_skill = re.sub(r"\s+", " ", skill) - ordered_steps = [ - "xprobe --version", - "xprobe doctor", - "xprobe discover", - "xprobe validate", - "xprobe measure", - "Check `status`", - ] - positions = [normalized_skill.index(step) for step in ordered_steps] - assert positions == sorted(positions) + for route in ( + "Existing artifact", + "Known live boundary", + "Unknown CPU workload", + "Unknown GPU or mixed workload", + "Multiple processes", + "Setup or repair", + ): + assert route in normalized_skill + for adaptive_rule in ( + "Choose the shortest route", + "Skip installation, `doctor`, `discover`, and live attachment", + "Do not run a broad inventory solely to satisfy a checklist", + "Do not run CUDA discovery", + "collect only the broad bounded inventories needed by the question", + "A completed-artifact analysis does not require a local collector", + "Run `doctor` when capability is unknown", + ): + assert adaptive_rule in normalized_skill + for invariant in ( + "schema version `2.0`", + "PID plus procfs start time", + "Run read-only `validate` before every live measurement", + "Bound every capture", + "leave the CUPTI shared object mapped", + "temporal correlation is not exact causality", + ): + assert invariant in normalized_skill for quality_field in ( "unmatched", "ambiguous", @@ -69,19 +87,16 @@ def check_skill(workspace: pathlib.Path) -> None: "clock alignment", "correlation method", "confidence", - "evidence", + "evidence pair", ): assert quality_field in normalized_skill for investigation_step in ( - "application-level latency baseline", "CPU-only", "GPU or mixed", - "do not run `discover`", - "representative coarse inventory", - "Scope breadth and collection duration are independent", + "Scope breadth and capture duration are independent", "scripts/analyze_trace.py", "selector hints", - "busy union", + "busy_union_ns", "overlap factor", "NCU or PC sampling", ): @@ -116,6 +131,11 @@ def check_skill(workspace: pathlib.Path) -> None: assert modes == {"exact", "aggregate"} assert {"exact", "first_after", "stack_nested", "stream_order"} <= policies + openai_yaml = (skill_root / "agents/openai.yaml").read_text() + assert 'display_name: "Xprobe Workload Profiling"' in openai_yaml + assert 'short_description: "Route bounded CPU and GPU profiling tasks"' in openai_yaml + assert "$xprobe-measure-latency" in openai_yaml + investigation = (skill_root / "references/investigation.md").read_text() quality = (skill_root / "references/result-quality.md").read_text() multi_process = (skill_root / "references/multi-process.md").read_text() @@ -165,7 +185,8 @@ def check_skill(workspace: pathlib.Path) -> None: ): assert required in normalized_trace_analysis for required in ( - "v0.4.0/install.sh", + "v0.4.1/install.sh", + "xprobe `0.4.x`", "npx skills@1 add", "xprobe --version", "xprobe doctor", @@ -185,6 +206,7 @@ def check_skill(workspace: pathlib.Path) -> None: "GLIBC_2.34 ceiling", "downloading the public archive", "transient infrastructure failure", + "Choose the shortest Skill route", ): assert required in engineering_rules diff --git a/tests/integration/test_pytorch.py b/tests/integration/test_pytorch.py index c52b8e6..6cef360 100644 --- a/tests/integration/test_pytorch.py +++ b/tests/integration/test_pytorch.py @@ -5,11 +5,21 @@ def main() -> None: - if len(sys.argv) != 3: - raise SystemExit("usage: test_pytorch.py ") + if len(sys.argv) not in {2, 3}: + raise SystemExit( + "usage: test_pytorch.py [pytorch-environment]" + ) workspace = pathlib.Path(__file__).resolve().parents[2] - pytorch_env = pathlib.Path(sys.argv[2]).resolve() + python = "python3" + environment_arguments = [] + if len(sys.argv) == 3: + pytorch_env = pathlib.Path(sys.argv[2]).resolve() + python = "/opt/xprobe-pytorch/bin/python" + environment_arguments = [ + "--volume", + f"{pytorch_env}:/opt/xprobe-pytorch:ro", + ] completed = subprocess.run( [ "docker", @@ -27,15 +37,14 @@ def main() -> None: "seccomp=unconfined", "--volume", f"{workspace}:/workspace:ro", - "--volume", - f"{pytorch_env}:/opt/xprobe-pytorch:ro", + *environment_arguments, "--workdir", "/workspace", sys.argv[1], - "/opt/xprobe-pytorch/bin/python", + python, "/workspace/tests/integration/test_pytorch_symbols.py", "--python", - "/opt/xprobe-pytorch/bin/python", + python, "--xprobe", "/workspace/target/debug/xprobe", "--measure", diff --git a/tests/integration/test_pytorch_cuda.py b/tests/integration/test_pytorch_cuda.py index d7f8c94..f98d7b0 100644 --- a/tests/integration/test_pytorch_cuda.py +++ b/tests/integration/test_pytorch_cuda.py @@ -23,10 +23,18 @@ def main() -> None: def run_container(args: argparse.Namespace) -> None: - if not args.image or not args.pytorch_env: - raise SystemExit("--image and --pytorch-env are required") + if not args.image: + raise SystemExit("--image is required") workspace = pathlib.Path(__file__).resolve().parents[2] - pytorch_env = pathlib.Path(args.pytorch_env).resolve() + python = "python3" + environment_arguments = [] + if args.pytorch_env: + pytorch_env = pathlib.Path(args.pytorch_env).resolve() + python = "/opt/xprobe-pytorch/bin/python" + environment_arguments = [ + "--volume", + f"{pytorch_env}:/opt/xprobe-pytorch:ro", + ] completed = subprocess.run( [ "docker", @@ -40,12 +48,11 @@ def run_container(args: argparse.Namespace) -> None: "seccomp=unconfined", "--volume", f"{workspace}:/workspace:ro", - "--volume", - f"{pytorch_env}:/opt/xprobe-pytorch:ro", + *environment_arguments, "--workdir", "/workspace", args.image, - "/opt/xprobe-pytorch/bin/python", + python, "/workspace/tests/integration/test_pytorch_cuda.py", "--inner", ], @@ -220,6 +227,12 @@ def inventory( ) -> dict: set_mode(mode_path, mode) time.sleep(0.1) + validate_pair( + xprobe, + pid, + f"cuda:{activity}_start", + f"cuda:{activity}_end", + ) return run_xprobe( xprobe, [ @@ -256,6 +269,7 @@ def exact_measure( ) -> dict: set_mode(mode_path, mode) time.sleep(0.1) + validate_pair(xprobe, pid, start_selector, end_selector) return run_xprobe( xprobe, [ @@ -280,6 +294,33 @@ def exact_measure( ) +def validate_pair( + xprobe: pathlib.Path, + pid: int, + start_selector: str, + end_selector: str, +) -> dict: + result = run_xprobe( + xprobe, + [ + "validate", + "--pid", + str(pid), + "--from", + start_selector, + "--to", + end_selector, + "--match", + "exact", + ], + ) + assert result["valid"] is True, result + assert result["target"]["pid"] == pid, result + assert result["policy_recommendation"]["policy"] == "exact", result + assert result["requirements"]["agent_activation"] == "already_loaded", result + return result + + def run_xprobe(xprobe: pathlib.Path, arguments: list[str]) -> dict: completed = subprocess.run( [ diff --git a/tests/integration/test_pytorch_symbols.py b/tests/integration/test_pytorch_symbols.py index 9da2e29..2348117 100755 --- a/tests/integration/test_pytorch_symbols.py +++ b/tests/integration/test_pytorch_symbols.py @@ -125,6 +125,37 @@ def measure( object_path: str, ) -> dict: selector = f"uprobe:{object_path}:symbol={MM_SYMBOL}" + validated = subprocess.run( + [ + binary, + "validate", + "--pid", + str(pid), + "--from", + f"{selector}:entry", + "--to", + f"{selector}:return", + "--match", + "stack-nested", + "--json", + "--non-interactive", + "--no-color", + ], + cwd=workspace, + check=False, + capture_output=True, + text=True, + ) + if validated.returncode != 0: + raise AssertionError( + f"validate failed:\n{validated.stdout}\n{validated.stderr}" + ) + validation = json.loads(validated.stdout) + assert validation["valid"] is True, validation + assert validation["target"]["pid"] == pid, validation + assert ( + validation["policy_recommendation"]["policy"] == "stack_nested" + ), validation completed = subprocess.run( [ binary,