From 2db01926476a2dd93640c76e089787287575b45b Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Tue, 13 Oct 2026 15:51:01 +0000 Subject: [PATCH 01/13] =?UTF-8?q?fix(agentx):=20role-aware,=20window-scope?= =?UTF-8?q?d=20ATOM=20cache=20rates=20with=20validity=20/=20=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=20ATOM=20=E7=BC=93=E5=AD=98=E5=91=BD=E4=B8=AD?= =?UTF-8?q?=E7=8E=87=E5=8F=A3=E5=BE=84=E5=B9=B6=E6=A0=87=E6=B3=A8=E6=9C=89?= =?UTF-8?q?=E6=95=88=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ATOM agentic result adapter summed prefix_cache_full_tokens over prefill and decode endpoints, reported lmcache_loaded_tokens (transfer volume) as CPU reuse, and set overall equal to GPU. The AIPerf metrics section also spans warmup. - Take endpoint roles from the launcher inputs (SRT_PREFILL/DECODE/AGG_ENDPOINTS). - Count prefill (or unified) endpoints over timeslices inside the profiling window. - CPU = prefix_cache_offload_tokens / full; external aliases CPU; overall = GPU + CPU. - Emit server_metrics.cache_validity so consumers know whether the rates are a complete, same-scope actual observation; refuse invalid or unprovable inputs. Refs: run 36734492117, #3543, #3158 --- .../infx/results/agentic/__init__.py | 10 +- .../infx/results/agentic/backends/atom.py | 360 ++++++++++++-- .../infx/results/agentic/backends/base.py | 1 + .../infx/results/agentic/server_metrics.py | 83 +++- .../infx/tests/bench/test_agentic_command.py | 101 +++- .../agentic/test_atom_cache_accounting.py | 442 ++++++++++++++++++ .../agentic/test_process_agentic_result.py | 336 ++++++++----- 7 files changed, 1193 insertions(+), 140 deletions(-) create mode 100644 inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py diff --git a/inferencex-e2e/infx/results/agentic/__init__.py b/inferencex-e2e/infx/results/agentic/__init__.py index 7cb92a82ff..ba84e85324 100644 --- a/inferencex-e2e/infx/results/agentic/__init__.py +++ b/inferencex-e2e/infx/results/agentic/__init__.py @@ -9,7 +9,12 @@ from infx.results.topology import Parallelism, validate_parallelism from .request_metrics import compute_request_metrics -from .server_metrics import compute_server_metrics +from .server_metrics import ( + compute_server_metrics, + deployment_from_env, + endpoint_roles_from_env, + request_scope_from_records, +) def _env_int(env: Mapping[str, str], name: str, default: int = 0) -> int: @@ -216,6 +221,9 @@ def build_result( framework=framework, records=records, server_logs=server_logs, + endpoint_roles=endpoint_roles_from_env(env), + request_scope=request_scope_from_records(records, request_accounting), + deployment=deployment_from_env(env), ) if "total_tput_tps" in request_flat and num_gpus > 0: diff --git a/inferencex-e2e/infx/results/agentic/backends/atom.py b/inferencex-e2e/infx/results/agentic/backends/atom.py index a5c8247dbe..6e83c0f0d3 100644 --- a/inferencex-e2e/infx/results/agentic/backends/atom.py +++ b/inferencex-e2e/infx/results/agentic/backends/atom.py @@ -2,12 +2,26 @@ from __future__ import annotations +import math +from collections.abc import Mapping from typing import Any -from infx.results.agentic.common import gauge_stat, normalize_fraction, rate, sum_stat +from infx.results.agentic.common import gauge_stat, normalize_fraction, series_stat, sum_stat from .base import ServerMetricsBackend, counter_int +# Prefix reuse happens where prompts are prefilled. A P/D decode worker re-counts the same +# prompts in prefix_cache_full_tokens on first admission without serving any reuse. +ACCOUNTED_ROLES = frozenset({"prefill", "unified"}) +FULL = "prefix_cache_full_tokens" +CACHED = "prefix_cache_cached_tokens" +OFFLOAD = "prefix_cache_offload_tokens" +LOADED = "lmcache_loaded_tokens" +REQUESTS = "prefix_cache_requests" +_COUNTERS = (FULL, CACHED, OFFLOAD, LOADED, REQUESTS) +VALIDITY_SCHEMA = "atom-cache-validity/1" +PRICING_STATUSES = frozenset({"VALID_COMPLETE", "VALID_ZERO"}) + def _atom_names(suffix: str) -> list[str]: """Accept direct worker metrics and Atomesh's colon-normalized form.""" @@ -19,8 +33,270 @@ def _atom_counter_names(stem: str) -> list[str]: return [*_atom_names(stem), *_atom_names(f"{stem}_total")] +def _series_by_endpoint( + section: Mapping[str, Any], stem: str +) -> tuple[dict[str, dict[str, Any]], list[str]]: + """One series per endpoint across all name aliases; a second one is a duplicate.""" + by_endpoint: dict[str, dict[str, Any]] = {} + duplicates: list[str] = [] + for name in _atom_counter_names(stem): + entry = section.get(name) + if not isinstance(entry, dict): + continue + for series in entry.get("series") or []: + url = series.get("endpoint_url") if isinstance(series, dict) else None + if not isinstance(url, str): + continue + if url in by_endpoint: + duplicates.append(url) + by_endpoint[url] = series + return by_endpoint, duplicates + + +def _slices(series: dict[str, Any]) -> list[tuple[int, int, float]] | None: + """Ordered, non-overlapping, finite, non-negative AIPerf timeslices, or None.""" + raw = series.get("timeslices") + if not isinstance(raw, list) or not raw: + return None + out: list[tuple[int, int, float]] = [] + for item in raw: + if not isinstance(item, dict): + return None + lo, hi, value = item.get("start_ns"), item.get("end_ns"), item.get("total") + if not isinstance(lo, int) or not isinstance(hi, int) or hi <= lo: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + if not math.isfinite(value) or value < 0: + return None + out.append((lo, hi, float(value))) + out.sort() + if any(out[i][0] < out[i - 1][1] for i in range(1, len(out))): + return None + return out + + +def _window_parts(slices: list[tuple[int, int, float]], start: int, end: int) -> dict[str, Any]: + """Split slice totals around [start, end]; flag sampling gaps inside the window.""" + parts = {"before": 0.0, "start_straddle": 0.0, "inside": 0.0, "end_straddle": 0.0} + covering = [s for s in slices if s[1] > start and s[0] < end] + gap = ( + not covering + or covering[0][0] > start + or covering[-1][1] < end + or any(covering[i][0] != covering[i - 1][1] for i in range(1, len(covering))) + ) + for lo, hi, value in slices: + if hi <= start: + parts["before"] += value + elif lo >= start and hi <= end: + parts["inside"] += value + else: + if lo < start < hi: + parts["start_straddle"] += value + if lo < end < hi and not lo < start < hi: + parts["end_straddle"] += value + parts["gap"] = gap + return parts + + +def account_cache( + metrics: dict[str, dict[str, Any]], + *, + framework: str, + endpoint_roles: Mapping[str, str] | None, + profiling_window: tuple[int, int] | None, + warmup_metrics: Mapping[str, Any] | None = None, + request_scope: Mapping[str, Any] | None = None, + disagg: bool | None = None, + kv_offloading: str | None = None, +) -> dict[str, Any]: + """Prefix-cache reuse on prefill (or unified) workers inside the profiling window. + + GPU = cached/full and CPU = offload/full on the same endpoints and timeslices. ATOM records + LMCache reuse in prefix_cache_offload_tokens, disjoint from the HBM `cached` counter, so + overall = GPU + CPU. lmcache_loaded_tokens is transfer volume and stays diagnostic only. + `validity` tells consumers whether the rates are a complete, same-scope actual observation + (`pricing_eligible`); warnings alone are not the guard. + """ + result: dict[str, Any] = dict.fromkeys( + ("gpu", "cpu", "overall", "full", "cached", "offload", "loaded", "requests") + ) + reasons: list[str] = [] + validity: dict[str, Any] = { + "schema": VALIDITY_SCHEMA, + "status": None, + "pricing_eligible": False, + "observed_scope": None, + "requested_scope": "profiling_successful_requests", + "cpu_component": None, + "roles_source": None, + "accounted_endpoints": [], + "window": None, + "request_scope": None, + "diagnostics": {}, + "reasons": reasons, + } + result.update(validity=validity, warnings=[]) + + def refuse(status: str, reason: str) -> dict[str, Any]: + validity["status"] = status + reasons.append(reason) + result["warnings"].append(f"atom cache accounting: {reason}") + return result + + series: dict[str, dict[str, dict[str, Any]]] = {} + for stem in _COUNTERS: + series[stem], duplicates = _series_by_endpoint(metrics, stem) + if duplicates: + return refuse("INVALID_COUNTER", f"duplicate {stem} series for {duplicates}") + observed = sorted(series[FULL]) + if not observed: + return refuse("MISSING_OBSERVATION", "no prefix_cache_full_tokens series") + + if endpoint_roles: + unmapped = [url for url in observed if url not in endpoint_roles] + accounted = sorted(u for u, role in endpoint_roles.items() if role in ACCOUNTED_ROLES) + missing = [url for url in accounted if url not in observed] + if unmapped or missing or not accounted: + return refuse( + "ROLE_OR_COVERAGE_UNRESOLVED", + "endpoint roles do not cover the scraped endpoints " + f"(unmapped={unmapped}, missing={missing})", + ) + validity["roles_source"] = "launcher_env" + elif disagg is False and len(observed) == 1: + accounted = observed + validity["roles_source"] = "single_endpoint_non_disagg" + else: + return refuse( + "ROLE_OR_COVERAGE_UNRESOLVED", + f"{framework or 'atom'}: no launcher endpoint roles and not a proven single unified " + "instance (SRT_PREFILL_ENDPOINTS/SRT_DECODE_ENDPOINTS or SRT_AGG_ENDPOINTS needed)", + ) + validity["accounted_endpoints"] = accounted + + for stem in (FULL, CACHED): + if any(url not in series[stem] for url in accounted): + return refuse("MISSING_OBSERVATION", f"{stem} missing on accounted endpoints") + cumulative = {} + for stem in _COUNTERS: + values = [series_stat(series[stem][u]) for u in accounted if u in series[stem]] + if values and len(values) == len(accounted) and None not in values: + cumulative[f"atom:{stem}"] = float(sum(values)) + validity["diagnostics"]["cumulative_stats_total"] = cumulative + + if profiling_window is None: + return refuse("WINDOW_UNRESOLVED", "no summary.phase_time_ranges.profiling window") + start, end = profiling_window + totals: dict[str, float | None] = {} + boundary: list[str] = [] + for stem in _COUNTERS: + present = [u for u in accounted if u in series[stem]] + if not present: + totals[stem] = None + continue + if len(present) != len(accounted): + totals[stem] = None + continue + inside = 0.0 + for url in present: + slices = _slices(series[stem][url]) + if slices is None: + return refuse("WINDOW_UNRESOLVED", f"{stem}@{url}: timeslices missing or invalid") + parts = _window_parts(slices, start, end) + if parts["gap"]: + return refuse("WINDOW_UNRESOLVED", f"{stem}@{url}: sampling gap in the window") + inside += parts["inside"] + if stem in (LOADED, REQUESTS): + continue + if parts["end_straddle"] > 0: + boundary.append(f"{stem}@{url}: activity in the slice straddling profiling end") + if parts["start_straddle"] > 0: + warm_series, _ = _series_by_endpoint(warmup_metrics or {}, stem) + warm = series_stat(warm_series[url]) if url in warm_series else None + if warm is None or parts["before"] + parts["start_straddle"] != warm: + boundary.append( + f"{stem}@{url}: activity in the slice straddling profiling start is not " + "proven warmup-only by warmup_metrics" + ) + totals[stem] = inside + validity["observed_scope"] = "profiling_window_interior_timeslices" + validity["window"] = {"start_ns": start, "end_ns": end, "boundary_issues": boundary} + + full, cached = totals[FULL], totals[CACHED] + offload, loaded, requests = totals[OFFLOAD], totals[LOADED], totals[REQUESTS] + if full is None or cached is None: + return refuse("MISSING_OBSERVATION", "cached/full counters missing on accounted endpoints") + if offload is not None: + validity["cpu_component"] = "MEASURED" + elif (kv_offloading or "").lower() == "none": + validity["cpu_component"] = "NO_CPU_TIER_CONFIRMED" + offload = 0.0 + else: + validity["cpu_component"] = "MISSING" + # The invariants hold on every CPU branch, including the contract zero set just above. + if not math.isfinite(full) or full <= 0: + return refuse("INVALID_COUNTER", "no prompt tokens in the accounting window") + if not math.isfinite(cached) or not 0 <= cached <= full: + return refuse( + "INVALID_COUNTER", f"cached tokens ({cached:.0f}) outside [0, full ({full:.0f})]" + ) + if offload is not None and ( + not math.isfinite(offload) or offload < 0 or not cached + offload <= full + ): + return refuse( + "INVALID_COUNTER", + f"cached+offload ({cached + offload:.0f}) exceeds full tokens ({full:.0f})", + ) + result.update(full=full, cached=cached, offload=offload, loaded=loaded, requests=requests) + result["gpu"] = cached / full + if offload is not None: + result.update(cpu=offload / full, overall=(cached + offload) / full) + + scope = dict(request_scope or {}) + client_requests = scope.get("profiling_requests") + client_tokens = scope.get("profiling_prompt_tokens") + conserved = ( + requests is not None + and client_tokens is not None + and requests == client_requests + and full == client_tokens + and not scope.get("error_records") + and not scope.get("cancelled_records") + ) + validity["request_scope"] = { + "server_window_admissions": requests, + "server_window_prompt_tokens": full, + **scope, + "conserved": conserved, + } + + if boundary: + reasons.extend(boundary) + validity["status"] = "WINDOW_UNRESOLVED" + elif validity["cpu_component"] == "MISSING": + reasons.append("CPU reuse counter missing and no confirmed absence of a CPU tier") + validity["status"] = "PARTIAL_COMPONENT" + elif not conserved: + reasons.append( + "server window admissions/prompt tokens do not match the successful profiling " + f"requests ({requests} vs {client_requests}; {full} vs {client_tokens})" + ) + validity["status"] = "REQUEST_SCOPE_UNRESOLVED" + else: + validity["status"] = "VALID_ZERO" if cached + offload == 0 else "VALID_COMPLETE" + validity["pricing_eligible"] = validity["status"] in PRICING_STATUSES + if not validity["pricing_eligible"]: + result["warnings"].append( + f"atom cache accounting: {validity['status']}; rates are telemetry, not pricing input" + ) + return result + + class AtomBackend(ServerMetricsBackend): name = "atom" + accepts_context = True def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: metric_names = set(metrics) @@ -33,6 +309,8 @@ def populate( metrics: dict[str, dict[str, Any]], flat: dict[str, Any], nested: dict[str, Any], + *, + context: Mapping[str, Any] | None = None, ) -> None: prompt_total = sum_stat( metrics, @@ -47,31 +325,26 @@ def populate( flat["total_prompt_tokens"] = counter_int(prompt_total) flat["total_generation_tokens"] = counter_int(generation_total) - cached_tokens = sum_stat( + context = context or {} + accounting = account_cache( metrics, - _atom_counter_names("prefix_cache_cached_tokens"), - preferred_keys=("total", "sum", "max", "avg"), + framework=str(context.get("framework") or ""), + endpoint_roles=context.get("endpoint_roles"), + profiling_window=context.get("profiling_window"), + warmup_metrics=context.get("warmup_metrics"), + request_scope=context.get("request_scope"), + disagg=context.get("disagg"), + kv_offloading=context.get("kv_offloading"), ) - full_tokens = sum_stat( - metrics, - _atom_counter_names("prefix_cache_full_tokens"), - preferred_keys=("total", "sum", "max", "avg"), - ) - cache_hit_rate = rate(cached_tokens, full_tokens) - external_tokens = sum_stat( - metrics, - _atom_counter_names("lmcache_loaded_tokens"), - preferred_keys=("total", "sum", "max", "avg"), - ) - external_hit_rate = rate(external_tokens, full_tokens) - # ATOM's admitted cache counter may already include a completed - # LMCache load, so do not add external tokens a second time. - overall_hit_rate = cache_hit_rate - - flat["server_gpu_cache_hit_rate"] = cache_hit_rate - flat["server_cpu_cache_hit_rate"] = external_hit_rate - flat["server_external_cache_hit_rate"] = external_hit_rate - flat["server_overall_cache_hit_rate"] = overall_hit_rate + sink = context.get("warnings") + if isinstance(sink, list): + sink.extend(accounting["warnings"]) + + flat["server_gpu_cache_hit_rate"] = accounting["gpu"] + flat["server_cpu_cache_hit_rate"] = accounting["cpu"] + # LocalCPUBackend is ATOM's only external tier, so external aliases the CPU reuse. + flat["server_external_cache_hit_rate"] = accounting["cpu"] + flat["server_overall_cache_hit_rate"] = accounting["overall"] flat["gpu_kv_cache_usage_pct"] = normalize_fraction( gauge_stat( metrics, @@ -83,16 +356,39 @@ def populate( nested["cache"].update( { - "gpu_cache_hit_rate": cache_hit_rate, - "cpu_cache_hit_rate": external_hit_rate, - "external_cache_hit_rate": external_hit_rate, - "overall_cache_hit_rate": overall_hit_rate, - "prefix_cache_hits": cached_tokens, - "prefix_cache_queries": full_tokens, - "external_prefix_cache_hits": external_tokens, - "external_prefix_cache_queries": full_tokens, + "gpu_cache_hit_rate": accounting["gpu"], + "cpu_cache_hit_rate": accounting["cpu"], + "external_cache_hit_rate": accounting["cpu"], + "overall_cache_hit_rate": accounting["overall"], + "prefix_cache_hits": accounting["cached"], + "prefix_cache_queries": accounting["full"], + "external_prefix_cache_hits": accounting["offload"], + "external_prefix_cache_queries": ( + accounting["full"] if accounting["offload"] is not None else None + ), } ) + nested["cache_validity"] = accounting["validity"] + by_source = nested["tokens"]["prompt_by_source"] + if accounting["full"] is not None and accounting["offload"] is not None: + by_source.update( + { + "gpu_cache_hit": accounting["cached"], + "cpu_or_external_cache_hit": accounting["offload"], + "computed": accounting["full"] - accounting["cached"] - accounting["offload"], + } + ) + by_source["raw"] = { + f"atom:{stem}": accounting[key] + for stem, key in ( + (FULL, "full"), + (CACHED, "cached"), + (OFFLOAD, "offload"), + (LOADED, "loaded"), + (REQUESTS, "requests"), + ) + if accounting[key] is not None + } nested["kv_cache"]["gpu_usage_pct"] = flat["gpu_kv_cache_usage_pct"] nested["tokens"].update( { diff --git a/inferencex-e2e/infx/results/agentic/backends/base.py b/inferencex-e2e/infx/results/agentic/backends/base.py index 427a4c1606..370897dd92 100644 --- a/inferencex-e2e/infx/results/agentic/backends/base.py +++ b/inferencex-e2e/infx/results/agentic/backends/base.py @@ -67,6 +67,7 @@ def empty_server_metrics(present: bool, metric_count: int) -> dict[str, Any]: "frontend_cached_tokens": None, "frontend_input_tokens": None, }, + "cache_validity": None, "kv_cache": { "gpu_usage_pct": None, "gpu_total_tokens": None, diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index aa4e7c64d7..01a1693140 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import Any from .backends import detect_backend @@ -12,6 +12,69 @@ empty_server_metrics, ) from .common import index_server_metrics +from .request_metrics import extract_per_record_ints + + +def request_scope_from_records( + records: list[dict[str, Any]], request_accounting: Mapping[str, Any] +) -> dict[str, Any]: + """Client-side profiling cohort in server token units (``usage_prompt_tokens``). + + ``profiling_prompt_tokens`` is None when any successful record lacks usage, so a partial + sum is never compared with server counters. + """ + prompt = extract_per_record_ints(records, "usage_prompt_tokens") + return { + "profiling_requests": len(records), + "profiling_prompt_tokens": sum(prompt) if len(prompt) == len(records) else None, + "error_records": request_accounting.get("records_error_dropped"), + "cancelled_records": sum( + 1 for record in records if (record.get("metadata") or {}).get("was_cancelled") + ), + } + + +def deployment_from_env(env: Mapping[str, str]) -> dict[str, Any]: + """Launcher-declared topology facts; absent or unrecognised values stay unknown (None).""" + disagg = {"true": True, "false": False}.get((env.get("DISAGG") or "").lower()) + return {"disagg": disagg, "kv_offloading": env.get("KV_OFFLOADING") or None} + + +def endpoint_roles_from_env(env: Mapping[str, str]) -> dict[str, str] | None: + """Map each scraped metrics URL to its launcher role, or None when roles are unknown. + + Mirrors ``infx.bench.agentic.replay._server_metrics_urls``: aggregated workers are + ``unified``; otherwise prefill endpoints precede decode endpoints. Caller-supplied + AIPERF_SERVER_METRICS_URLS and Dynamo frontends carry no roles, and an endpoint listed + under two roles is ambiguous. + """ + if env.get("AIPERF_SERVER_METRICS_URLS") or env.get("SRTCTL_FRONTEND_TYPE") == "dynamo": + return None + if env.get("SRT_AGG_ENDPOINTS"): + groups = [("unified", env["SRT_AGG_ENDPOINTS"])] + else: + groups = [ + ("prefill", env.get("SRT_PREFILL_ENDPOINTS", "")), + ("decode", env.get("SRT_DECODE_ENDPOINTS", "")), + ] + roles: dict[str, str] = {} + for role, endpoints in groups: + for endpoint in endpoints.split(","): + if endpoint and roles.setdefault(f"http://{endpoint}/metrics", role) != role: + return None + return roles or None + + +def _profiling_window(server_metrics: dict[str, Any]) -> tuple[int, int] | None: + summary = server_metrics.get("summary") + ranges = summary.get("phase_time_ranges") if isinstance(summary, dict) else None + profiling = ranges.get("profiling") if isinstance(ranges, dict) else None + if not isinstance(profiling, dict): + return None + start, end = profiling.get("start_ns"), profiling.get("end_ns") + if isinstance(start, int) and isinstance(end, int) and start < end: + return start, end + return None def compute_server_metrics( @@ -20,6 +83,9 @@ def compute_server_metrics( framework: str, records: list[dict[str, Any]], server_logs: Iterable[str | None] = (), + endpoint_roles: Mapping[str, str] | None = None, + request_scope: Mapping[str, Any] | None = None, + deployment: Mapping[str, Any] | None = None, ) -> tuple[dict[str, Any], dict[str, Any], list[str]]: metrics = index_server_metrics(server_metrics) flat = SERVER_CACHE_FLAT_FIELDS.copy() @@ -46,7 +112,20 @@ def compute_server_metrics( f"framework={framework.lower()!r}, metric_names=[{preview}]" ) nested["adapter"] = adapter.name - adapter.populate(metrics, flat, nested) + if getattr(adapter, "accepts_context", False): + warmup = server_metrics.get("warmup_metrics") + context = { + "framework": framework, + "endpoint_roles": endpoint_roles, + "profiling_window": _profiling_window(server_metrics), + "warmup_metrics": warmup if isinstance(warmup, dict) else None, + "request_scope": request_scope, + **dict(deployment or {}), + "warnings": warnings, + } + adapter.populate(metrics, flat, nested, context=context) + else: + adapter.populate(metrics, flat, nested) if framework.lower() == "dynamo-sglang" and adapter.name == "sglang": # Dynamo can export replicated TP-rank gauges for one logical KV pool. diff --git a/inferencex-e2e/infx/tests/bench/test_agentic_command.py b/inferencex-e2e/infx/tests/bench/test_agentic_command.py index 080ca3a6e4..06fa13a4f6 100644 --- a/inferencex-e2e/infx/tests/bench/test_agentic_command.py +++ b/inferencex-e2e/infx/tests/bench/test_agentic_command.py @@ -145,7 +145,9 @@ def test_standalone_srt_slurm_point_needs_no_kv_offload_declaration(tmp_path): WINDOW = { - "IS_MULTINODE": "true", "ENABLE_AGENTX_POWER": "1", "REQUIRE_POWER": "0", + "IS_MULTINODE": "true", + "ENABLE_AGENTX_POWER": "1", + "REQUIRE_POWER": "0", "SRT_MEASUREMENT_WINDOW_DIR": "/w", } MISSING = {"IS_MULTINODE": "true", "ENABLE_AGENTX_POWER": "1", "REQUIRE_POWER": "1"} @@ -404,3 +406,100 @@ def frontend(method: str, path: str) -> tuple[int, object]: assert "fake aiperf: profiled 1 request" in result.stdout aggregate = json.loads((tmp_path / "agentx.json").read_text()) assert (aggregate["conc"], aggregate["num_requests_total"]) == (8, 1) + + +# The replay writes ATOM P/D counters for exactly the endpoints the launcher told it to scrape. +# The prefill endpoint is the one the servers were started with (SRT_PREFILL_ENDPOINTS), not a +# position in argv. +FAKE_ATOM_AIPERF_SCRIPT = ( + FAKE_AIPERF_SCRIPT + + r""" +record["metrics"]["usage_prompt_tokens"] = {"value": 100, "unit": "tokens"} +(out / "profile_export.jsonl").write_text(json.dumps(record) + "\n") +urls = argv[argv.index("--server-metrics") + 1 :] +urls = urls[: next((i for i, url in enumerate(urls) if url.startswith("--")), len(urls))] +prefill = "http://" + os.environ["SRT_PREFILL_ENDPOINTS"] + "/metrics" +edges = [(0, 1000), (1000, 2500), (2500, 4000)] +counters = { + "prefix_cache_full_tokens": (0, 40, 60), + "prefix_cache_cached_tokens": (0, 20, 40), + "prefix_cache_offload_tokens": (0, 5, 5), + "prefix_cache_requests": (0, 1, 0), +} +def series(url, values): + slices = [{"start_ns": lo, "end_ns": hi, "total": v} for (lo, hi), v in zip(edges, values)] + return {"endpoint_url": url, "stats": {"total": sum(values)}, "timeslices": slices} +metrics = { + f"atom:{name}": { + "type": "counter", + "series": [series(url, values if url == prefill else (0, 50, 50)) for url in urls], + } + for name, values in counters.items() +} +summary = {"phase_time_ranges": {"profiling": {"start_ns": 1000, "end_ns": 4000}}} +(out / "server_metrics_export.json").write_text(json.dumps({"summary": summary, "metrics": metrics})) +""" +) + + +def test_srt_shim_hands_launcher_endpoint_roles_to_the_atom_producer(tmp_path, http_server): + tools = tmp_path / "tools" + tools.mkdir() + executable(tools / "uv", FAKE_UV) + (tools / "python3").symlink_to(sys.executable) + script = tmp_path / "fake_aiperf.py" + script.write_text(FAKE_ATOM_AIPERF_SCRIPT) + + def frontend(method: str, path: str) -> tuple[int, object]: + if path == "/v1/models": + return 200, {"data": [{"id": "served/model"}]} + return (405 if path == "/v1/chat/completions" else 200), {} + + workers = { + f"{side}_{key}": value + for side in ("PREFILL", "DECODE") + for key, value in { + "NUM_WORKERS": "1", + "TP": "8", + "PP_SIZE": "1", + "DCP_SIZE": "1", + "PCP_SIZE": "1", + "EP": "1", + "DP_ATTN": "false", + }.items() + } + env = _point( + tmp_path, + PATH=f"{tools}:/usr/bin:/bin", + HOME=str(tmp_path), + FAKE_AIPERF_SCRIPT=str(script), + REPLAY_ARGV=str(tmp_path / "replay-argv"), + HF_HUB_CACHE=str(tmp_path / "hf"), + AIPERF_RUNTIME_DIR=str(tmp_path / "runtime"), + EVAL_ONLY="true", + AIPERF_SERVER_URL=http_server(frontend), + SERVED_MODEL_NAME="served/model", + EVAL_ENDPOINT_READY_TIMEOUT_SECONDS="30", + EVAL_MODEL_STABILIZATION_SECONDS="0", + FRAMEWORK="atom-disagg", + DISAGG="true", + SRT_PREFILL_ENDPOINTS="10.0.0.1:6100", + SRT_DECODE_ENDPOINTS="10.0.0.2:6100", + **workers, + ) + + result = subprocess.run( + ["bash", str(SHIM)], env=env, capture_output=True, text=True, timeout=120, check=False + ) + + assert result.returncode == 0, result.stderr + argv = shlex.split((tmp_path / "replay-argv").read_text()) + scraped = argv[argv.index("--server-metrics") + 1 : argv.index("--server-metrics") + 3] + assert scraped == ["http://10.0.0.1:6100/metrics", "http://10.0.0.2:6100/metrics"] + server_metrics = json.loads((tmp_path / "agentx.json").read_text())["server_metrics"] + validity = server_metrics["cache_validity"] + assert validity["roles_source"] == "launcher_env" + assert validity["accounted_endpoints"] == ["http://10.0.0.1:6100/metrics"] + assert (validity["status"], validity["pricing_eligible"]) == ("VALID_COMPLETE", True) + cache = server_metrics["cache"] + assert (cache["gpu_cache_hit_rate"], cache["cpu_cache_hit_rate"]) == (0.6, 0.1) diff --git a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py new file mode 100644 index 0000000000..90f67ee35c --- /dev/null +++ b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py @@ -0,0 +1,442 @@ +from __future__ import annotations + +import json +import math +from pathlib import Path + +import pytest + +from infx.bench.agentic.replay import _server_metrics_urls +from infx.results.agentic.server_metrics import compute_server_metrics, endpoint_roles_from_env +from infx.tests.results.agentic.test_process_agentic_result import ( + _assert_stable_server_metrics_schema, + _run_processor, + _write_fixture, +) + +PREFILL = "http://10.0.0.1:6100/metrics" +DECODE = "http://10.0.0.2:6100/metrics" +ROLES = {PREFILL: "prefill", DECODE: "decode"} +WINDOW = (1_000, 4_000) +# warmup | profiling | profiling | tail: no slice straddles the window edges. +EDGES = ((0, 1_000), (1_000, 2_500), (2_500, 4_000), (4_000, 5_000)) +ZERO = (0.0, 0.0, 0.0, 0.0) +SCOPE = {"profiling_requests": 5, "profiling_prompt_tokens": 1000, "error_records": 0} +PD = {"disagg": True, "kv_offloading": "dram"} +RATE_KEYS = ( + "gpu_cache_hit_rate", + "cpu_cache_hit_rate", + "external_cache_hit_rate", + "overall_cache_hit_rate", +) +NO_RATES = (None, None, None, None) + + +def _series(url: str, values: tuple, edges: tuple = EDGES) -> dict: + slices = [{"start_ns": lo, "end_ns": hi, "total": v} for (lo, hi), v in zip(edges, values)] + return {"endpoint_url": url, "stats": {"total": sum(values)}, "timeslices": slices} + + +def _export(counters: dict[str, dict[str, tuple]], edges: tuple = EDGES, window=WINDOW) -> dict: + metrics = { + f"atom:{stem}": {"type": "counter", "series": [_series(u, v, edges) for u, v in ep.items()]} + for stem, ep in counters.items() + } + export: dict = {"metrics": metrics} + if window is not None: + profiling = {"start_ns": window[0], "end_ns": window[1]} + export["summary"] = {"phase_time_ranges": {"profiling": profiling}} + return export + + +def _pd(offload: tuple | None = (0, 50, 50, 0), **override: tuple) -> dict: + """Interior prefill: full 1000, cached 600, offload 100, loaded 150, 5 admissions.""" + prefill = { + "prefix_cache_full_tokens": (0, 400, 600, 0), + "prefix_cache_cached_tokens": (0, 200, 400, 0), + "prefix_cache_offload_tokens": offload, + "lmcache_loaded_tokens": (0, 70, 80, 0), + "prefix_cache_requests": (0, 3, 2, 0), + } | override + decode = {"prefix_cache_full_tokens": (0, 500, 650, 0), "prefix_cache_requests": (0, 3, 2, 0)} + return _export( + { + stem: {PREFILL: values, DECODE: decode.get(stem, ZERO)} + for stem, values in prefill.items() + if values is not None + } + ) + + +def _run(export: dict, *, roles=ROLES, scope=SCOPE, deployment=PD, framework="atom-disagg"): + _, nested, warnings = compute_server_metrics( + export, + framework=framework, + records=[], + endpoint_roles=roles, + request_scope=scope, + deployment=deployment, + ) + return nested, warnings + + +def _rates(nested: dict) -> tuple: + return tuple(nested["cache"][key] for key in RATE_KEYS) + + +def test_complete_pd_observation_is_eligible_and_ignores_decode_and_load_volume() -> None: + nested, warnings = _run(_pd()) + validity = nested["cache_validity"] + + assert _rates(nested) == (0.6, 0.1, 0.1, pytest.approx(0.7)) + assert nested["cache"]["prefix_cache_queries"] == 1000 + assert nested["tokens"]["prompt_by_source"]["raw"]["atom:lmcache_loaded_tokens"] == 150 + assert validity["status"] == "VALID_COMPLETE" + assert validity["pricing_eligible"] is True + assert validity["observed_scope"] == "profiling_window_interior_timeslices" + assert validity["accounted_endpoints"] == [PREFILL] + assert validity["request_scope"]["conserved"] is True + assert warnings == [] + + +@pytest.mark.parametrize( + ("kv_offloading", "status", "cpu", "eligible"), + [ + ("dram", "PARTIAL_COMPONENT", None, False), + (None, "PARTIAL_COMPONENT", None, False), + ("none", "VALID_COMPLETE", 0.0, True), + ], +) +def test_missing_cpu_counter_is_partial_unless_no_cpu_tier_is_configured( + kv_offloading: str | None, status: str, cpu: float | None, eligible: bool +) -> None: + nested, _ = _run(_pd(offload=None), deployment={"disagg": True, "kv_offloading": kv_offloading}) + + assert nested["cache"]["gpu_cache_hit_rate"] == 0.6 + assert nested["cache"]["cpu_cache_hit_rate"] == cpu + assert nested["cache_validity"]["status"] == status + assert nested["cache_validity"]["pricing_eligible"] is eligible + + +@pytest.mark.parametrize( + ("offload", "kv_offloading"), + [(None, "none"), ((0, 0, 0, 0), "dram")], +) +def test_cached_above_full_is_refused_on_every_cpu_branch(offload, kv_offloading) -> None: + deployment = {"disagg": True, "kv_offloading": kv_offloading} + export = _pd(offload=offload, prefix_cache_cached_tokens=(0, 500, 700, 0)) + + nested, warnings = _run(export, deployment=deployment) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "INVALID_COUNTER" + assert nested["cache_validity"]["pricing_eligible"] is False + assert any("cached" in w for w in warnings) + + +def test_missing_counter_contract_zero_matches_an_explicit_zero() -> None: + no_cpu_tier = {"disagg": True, "kv_offloading": "none"} + + explicit, _ = _run(_pd(offload=(0, 0, 0, 0)), deployment=no_cpu_tier) + missing, _ = _run(_pd(offload=None), deployment=no_cpu_tier) + + assert _rates(explicit) == _rates(missing) == (0.6, 0.0, 0.0, 0.6) + assert explicit["cache_validity"]["cpu_component"] == "MEASURED" + assert missing["cache_validity"]["cpu_component"] == "NO_CPU_TIER_CONFIRMED" + assert explicit["cache_validity"]["pricing_eligible"] is True + assert missing["cache_validity"]["pricing_eligible"] is True + + +@pytest.mark.parametrize( + ("cached", "offload", "status", "rates"), + [ + ((0, 0, 0, 0), (0, 0, 0, 0), "VALID_ZERO", (0.0, 0.0, 0.0, 0.0)), + ((0, 0, 0, 0), (0, 100, 150, 0), "VALID_COMPLETE", (0.0, 0.25, 0.25, 0.25)), + ], +) +def test_zero_and_cpu_only_reuse_are_eligible_measurements(cached, offload, status, rates) -> None: + nested, _ = _run(_pd(offload=offload, prefix_cache_cached_tokens=cached)) + + assert _rates(nested) == rates + assert nested["cache_validity"]["status"] == status + assert nested["cache_validity"]["pricing_eligible"] is True + + +@pytest.mark.parametrize( + ("roles", "deployment", "reason"), + [ + (None, PD, "no launcher endpoint roles"), + (None, {"disagg": None}, "no launcher endpoint roles"), + ({DECODE: "decode"}, PD, f"unmapped=['{PREFILL}']"), + ({**ROLES, "http://10.0.0.3:6100/metrics": "prefill"}, PD, "missing=['http://10.0.0.3"), + ({PREFILL: "decode", DECODE: "decode"}, PD, "unmapped=[], missing=[]"), + ], +) +def test_unresolved_roles_publish_no_rates(roles, deployment, reason) -> None: + nested, warnings = _run(_pd(), roles=roles, deployment=deployment) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "ROLE_OR_COVERAGE_UNRESOLVED" + assert nested["cache_validity"]["pricing_eligible"] is False + assert any(reason in w for w in warnings) + + +def test_unified_fallback_needs_a_declared_non_disagg_single_endpoint() -> None: + single = _export({stem: {PREFILL: v} for stem, v in _pd_values().items()}) + two = _export({stem: {PREFILL: v, DECODE: v} for stem, v in _pd_values().items()}) + unified = {"disagg": False, "kv_offloading": "dram"} + + one, _ = _run(single, roles=None, deployment=unified, framework="atom") + many, _ = _run(two, roles=None, deployment=unified, framework="atom") + + assert one["cache_validity"]["roles_source"] == "single_endpoint_non_disagg" + assert one["cache_validity"]["pricing_eligible"] is True + assert many["cache_validity"]["status"] == "ROLE_OR_COVERAGE_UNRESOLVED" + + +def _pd_values() -> dict[str, tuple]: + return { + "prefix_cache_full_tokens": (0, 400, 600, 0), + "prefix_cache_cached_tokens": (0, 200, 400, 0), + "prefix_cache_offload_tokens": (0, 50, 50, 0), + "prefix_cache_requests": (0, 3, 2, 0), + } + + +@pytest.mark.parametrize( + ("override", "status", "reason"), + [ + ({"prefix_cache_cached_tokens": (0, -1, 400, 0)}, "WINDOW_UNRESOLVED", "invalid"), + ({"prefix_cache_full_tokens": (0, math.nan, 600, 0)}, "WINDOW_UNRESOLVED", "invalid"), + ( + {"prefix_cache_full_tokens": ZERO, "prefix_cache_cached_tokens": ZERO}, + "INVALID_COUNTER", + "no prompt tokens", + ), + ({"prefix_cache_cached_tokens": (0, 400, 550, 0)}, "INVALID_COUNTER", "exceeds full"), + ], +) +def test_invalid_counters_are_refused_not_clamped(override, status, reason) -> None: + nested, warnings = _run(_pd(**override)) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == status + assert any(reason in w for w in warnings) + + +def test_missing_timeslices_never_fall_back_to_stats_total() -> None: + export = _pd() + for entry in export["metrics"].values(): + for series in entry["series"]: + del series["timeslices"] + + nested, _ = _run(export) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "WINDOW_UNRESOLVED" + cumulative = nested["cache_validity"]["diagnostics"]["cumulative_stats_total"] + assert cumulative["atom:prefix_cache_full_tokens"] == 1000 + + +def test_missing_profiling_window_is_unresolved() -> None: + export = _pd() + del export["summary"] + + nested, _ = _run(export) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "WINDOW_UNRESOLVED" + + +# Window (1_200, 4_000): (1_000, 1_400) straddles the start, (3_600, 4_200) the end. +STRADDLE = ((0, 1_000), (1_000, 1_400), (1_400, 2_600), (2_600, 3_600), (3_600, 4_200)) + + +def _unified(full: tuple, cached: tuple, warmup: dict | None = None) -> dict: + export = _export( + { + "prefix_cache_full_tokens": {PREFILL: full}, + "prefix_cache_cached_tokens": {PREFILL: cached}, + "prefix_cache_offload_tokens": {PREFILL: (0, 0, 0, 0, 0)}, + "prefix_cache_requests": {PREFILL: (0, 0, 3, 2, 0)}, + }, + edges=STRADDLE, + window=(1_200, 4_000), + ) + if warmup is not None: + export["warmup_metrics"] = { + f"atom:{stem}": {"series": [{"endpoint_url": PREFILL, "stats": {"total": total}}]} + for stem, total in warmup.items() + } + return export + + +@pytest.mark.parametrize( + ("full", "cached", "warmup", "status"), + [ + ((0, 0, 400, 600, 0), (0, 0, 200, 400, 0), None, "VALID_COMPLETE"), + ((0, 0, 400, 600, 9), (0, 0, 200, 400, 0), None, "WINDOW_UNRESOLVED"), + ((300, 90, 400, 600, 0), (100, 30, 200, 400, 0), None, "WINDOW_UNRESOLVED"), + ( + (300, 90, 400, 600, 0), + (100, 30, 200, 400, 0), + {"prefix_cache_full_tokens": 390, "prefix_cache_cached_tokens": 130}, + "VALID_COMPLETE", + ), + ( + (300, 90, 400, 600, 0), + (100, 30, 200, 400, 0), + {"prefix_cache_full_tokens": 300, "prefix_cache_cached_tokens": 100}, + "WINDOW_UNRESOLVED", + ), + ], +) +def test_boundary_slice_activity_must_be_proven_warmup_only(full, cached, warmup, status) -> None: + unified = {"disagg": False, "kv_offloading": "dram"} + nested, _ = _run(_unified(full, cached, warmup), roles=None, deployment=unified) + + assert nested["cache_validity"]["status"] == status + assert nested["cache_validity"]["pricing_eligible"] is (status == "VALID_COMPLETE") + assert nested["cache"]["gpu_cache_hit_rate"] == 0.6 + + +def test_sampling_gap_inside_the_window_is_unresolved() -> None: + gappy = ((0, 1_000), (1_000, 2_000), (2_200, 4_000), (4_000, 5_000)) + export = _export( + {stem: {PREFILL: values} for stem, values in _pd_values().items()}, edges=gappy + ) + + nested, warnings = _run(export, roles=None, deployment={"disagg": False}) + + assert nested["cache_validity"]["status"] == "WINDOW_UNRESOLVED" + assert any("sampling gap" in w for w in warnings) + + +def test_duplicate_alias_series_for_one_endpoint_is_invalid() -> None: + export = _pd() + export["metrics"]["atom_prefix_cache_cached_tokens_total"] = export["metrics"][ + "atom:prefix_cache_cached_tokens" + ] + + nested, _ = _run(export) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "INVALID_COUNTER" + + +@pytest.mark.parametrize( + "scope", + [ + {**SCOPE, "profiling_requests": 4, "profiling_prompt_tokens": 900}, + {**SCOPE, "error_records": 1}, + {**SCOPE, "profiling_prompt_tokens": None}, + None, + ], +) +def test_request_scope_mismatch_keeps_window_telemetry_but_blocks_pricing(scope) -> None: + nested, _ = _run(_pd(), scope=scope) + + assert _rates(nested) == (0.6, 0.1, 0.1, pytest.approx(0.7)) + assert nested["cache_validity"]["status"] == "REQUEST_SCOPE_UNRESOLVED" + assert nested["cache_validity"]["pricing_eligible"] is False + + +def test_unrelated_router_endpoint_is_not_a_missing_prefill() -> None: + export = _pd() + router = "http://10.0.0.9:8000/metrics" + export["metrics"]["atomesh:http_requests_total"] = { + "type": "counter", + "series": [_series(router, (0, 5, 5, 0))], + } + + nested, _ = _run(export, roles={**ROLES, router: "router"}) + + assert nested["cache_validity"]["status"] == "VALID_COMPLETE" + + +@pytest.mark.parametrize( + ("env", "expected"), + [ + ( + {"SRT_PREFILL_ENDPOINTS": "10.0.0.1:6100", "SRT_DECODE_ENDPOINTS": "10.0.0.2:6100,"}, + ROLES, + ), + ( + {"SRT_AGG_ENDPOINTS": "a:1,b:1", "SRT_PREFILL_ENDPOINTS": "c:1"}, + {"http://a:1/metrics": "unified", "http://b:1/metrics": "unified"}, + ), + ], +) +def test_endpoint_roles_cover_launcher_scrape_urls( + env: dict[str, str], expected: dict[str, str] +) -> None: + roles = endpoint_roles_from_env(env) + + assert roles == expected + assert set(roles) == set(_server_metrics_urls(env)) + + +@pytest.mark.parametrize( + "env", + [ + {"AIPERF_SERVER_METRICS_URLS": "http://x/metrics", "SRT_PREFILL_ENDPOINTS": "a:1"}, + {"SRTCTL_FRONTEND_TYPE": "dynamo", "SRT_PREFILL_ENDPOINTS": "a:1"}, + {"SRT_PREFILL_ENDPOINTS": "a:1", "SRT_DECODE_ENDPOINTS": "a:1"}, + {}, + ], +) +def test_endpoint_roles_unknown_without_launcher_metadata(env: dict[str, str]) -> None: + assert endpoint_roles_from_env(env) is None + + +def test_processor_entry_point_carries_validity_through_the_serializer(tmp_path: Path) -> None: + result_dir = _write_fixture(tmp_path) + artifacts = result_dir / "aiperf_artifacts" + (artifacts / "server_metrics_export.json").write_text(json.dumps(_pd())) + jsonl = artifacts / "profile_export.jsonl" + records = [json.loads(line) for line in jsonl.read_text().splitlines() if line.strip()] + for record in records: + record["metrics"]["usage_prompt_tokens"] = {"value": 1000 // len(records), "unit": "tokens"} + jsonl.write_text("".join(json.dumps(record) + "\n" for record in records)) + disagg_env = { + "FRAMEWORK": "atom-disagg", + "IS_MULTINODE": "true", + "DISAGG": "true", + **{ + f"{side}_{key}": value + for side in ("PREFILL", "DECODE") + for key, value in { + "NUM_WORKERS": "1", + "TP": "8", + "PP_SIZE": "1", + "DCP_SIZE": "1", + "PCP_SIZE": "1", + "EP": "1", + "DP_ATTN": "false", + "HARDWARE": "mi355x", + }.items() + }, + } + + agg = _run_processor( + result_dir, + tmp_path / "out", + env_overrides={ + **disagg_env, + "SRT_PREFILL_ENDPOINTS": "10.0.0.1:6100", + "SRT_DECODE_ENDPOINTS": "10.0.0.2:6100", + }, + ) + unresolved = _run_processor(result_dir, tmp_path / "unresolved", env_overrides=disagg_env) + + _assert_stable_server_metrics_schema(agg) + assert len(records) == 5 + assert _rates(agg["server_metrics"]) == (0.6, 0.1, 0.1, 0.7) + assert agg["server_metrics"]["cache_validity"]["status"] == "VALID_COMPLETE" + assert agg["server_metrics"]["cache_validity"]["pricing_eligible"] is True + assert _rates(unresolved["server_metrics"]) == NO_RATES + assert unresolved["server_metrics"]["cache_validity"]["pricing_eligible"] is False + assert ( + agg["server_metrics"]["tokens"]["prompt_total"] + == (unresolved["server_metrics"]["tokens"]["prompt_total"]) + ) diff --git a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py index d60a9813d2..f0a48a4297 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py @@ -79,6 +79,7 @@ "kv_offload", "tokens", "sources", + "cache_validity", } SERVER_CACHE_KEYS = { "gpu_cache_hit_rate", @@ -341,8 +342,7 @@ def _run_processor( timeout=30, ) assert proc.returncode == 0, ( - f"processor exited {proc.returncode}\n" - f"stdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" + f"processor exited {proc.returncode}\nstdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" ) out = output_dir / "agg_test.json" assert out.exists(), f"missing output {out}; stdout:\n{proc.stdout}" @@ -398,27 +398,54 @@ def test_processor_omits_component_metadata_when_absent(tmp_path: Path): assert "kv_p2p_transfer" not in agg -@pytest.mark.parametrize("parser", [ - _optional_component_metadata, _optional_kv_offload_backend_metadata, -]) +@pytest.mark.parametrize( + "parser", + [ + _optional_component_metadata, + _optional_kv_offload_backend_metadata, + ], +) @pytest.mark.parametrize("raw", [None, "", "null"]) def test_optional_metadata_accepts_unset_values(parser, raw): env = {} if raw is None else {"TEST_METADATA": raw} assert parser(env, "TEST_METADATA") is None -@pytest.mark.parametrize(("parser", "raw", "message"), [ - (_optional_component_metadata, "{", "must contain valid JSON"), - (_optional_component_metadata, "[]", "must contain exactly 'name' and 'version'"), - (_optional_component_metadata, '{"name":"router"}', "must contain exactly 'name' and 'version'"), - (_optional_component_metadata, '{"name":"router","version":0}', "name and version must be non-empty strings"), - (_optional_kv_offload_backend_metadata, "{", "must contain valid JSON"), - (_optional_kv_offload_backend_metadata, "[]", "may contain only 'name' and 'version'"), - (_optional_kv_offload_backend_metadata, '{"name":"cache","extra":1}', "may contain only 'name' and 'version'"), - (_optional_kv_offload_backend_metadata, "{}", "must contain 'name' and optional 'version'"), - (_optional_kv_offload_backend_metadata, '{"version":"1"}', "must contain 'name' and optional 'version'"), - (_optional_kv_offload_backend_metadata, '{"name":"cache","version":""}', "values must be non-empty strings"), -]) +@pytest.mark.parametrize( + ("parser", "raw", "message"), + [ + (_optional_component_metadata, "{", "must contain valid JSON"), + (_optional_component_metadata, "[]", "must contain exactly 'name' and 'version'"), + ( + _optional_component_metadata, + '{"name":"router"}', + "must contain exactly 'name' and 'version'", + ), + ( + _optional_component_metadata, + '{"name":"router","version":0}', + "name and version must be non-empty strings", + ), + (_optional_kv_offload_backend_metadata, "{", "must contain valid JSON"), + (_optional_kv_offload_backend_metadata, "[]", "may contain only 'name' and 'version'"), + ( + _optional_kv_offload_backend_metadata, + '{"name":"cache","extra":1}', + "may contain only 'name' and 'version'", + ), + (_optional_kv_offload_backend_metadata, "{}", "must contain 'name' and optional 'version'"), + ( + _optional_kv_offload_backend_metadata, + '{"version":"1"}', + "must contain 'name' and optional 'version'", + ), + ( + _optional_kv_offload_backend_metadata, + '{"name":"cache","version":""}', + "values must be non-empty strings", + ), + ], +) def test_optional_metadata_preserves_cli_errors(parser, raw, message): with pytest.raises(SystemExit) as error: parser({"TEST_METADATA": raw}, "TEST_METADATA") @@ -807,20 +834,36 @@ def test_gpu_shape_defaults_empty_parallelism_to_one(env): assert _gpu_shape(env) == ({"pp": 1, "dcp_size": 1, "pcp_size": 1}, 1, 1, 1, "false") -@pytest.mark.parametrize("decode_workers,expected_gpus,expected_decode", [ - ("3", 78, (2, 11, 1, 9, 5)), - ("0", 48, (0, 0, 1, 1, 1)), -]) +@pytest.mark.parametrize( + "decode_workers,expected_gpus,expected_decode", + [ + ("3", 78, (2, 11, 1, 9, 5)), + ("0", 48, (0, 0, 1, 1, 1)), + ], +) def test_gpu_shape_counts_workers_and_normalizes_absent_decode( - decode_workers, expected_gpus, expected_decode, + decode_workers, + expected_gpus, + expected_decode, ): env = { - "IS_MULTINODE": "true", "PREFILL_NUM_WORKERS": "2", "PREFILL_TP": "3", - "PREFILL_PP_SIZE": "2", "PREFILL_PCP_SIZE": "4", "PREFILL_DCP_SIZE": "7", - "PREFILL_EP": "8", "PREFILL_DP_ATTN": "YES", "DECODE_NUM_WORKERS": decode_workers, - "DECODE_TP": "2", "DECODE_EP": "11", "DECODE_PP_SIZE": "1", - "DECODE_DCP_SIZE": "9", "DECODE_PCP_SIZE": "5", "DECODE_DP_ATTN": "false", - "PREFILL_GPUS": "999", "DECODE_GPUS": "999", + "IS_MULTINODE": "true", + "PREFILL_NUM_WORKERS": "2", + "PREFILL_TP": "3", + "PREFILL_PP_SIZE": "2", + "PREFILL_PCP_SIZE": "4", + "PREFILL_DCP_SIZE": "7", + "PREFILL_EP": "8", + "PREFILL_DP_ATTN": "YES", + "DECODE_NUM_WORKERS": decode_workers, + "DECODE_TP": "2", + "DECODE_EP": "11", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "9", + "DECODE_PCP_SIZE": "5", + "DECODE_DP_ATTN": "false", + "PREFILL_GPUS": "999", + "DECODE_GPUS": "999", } fields, num_gpus, tp, ep, attention = _gpu_shape(env) # Two 24-GPU prefill workers plus either three 10-GPU decode workers or @@ -828,30 +871,47 @@ def test_gpu_shape_counts_workers_and_normalizes_absent_decode( assert num_gpus == expected_gpus assert fields["num_prefill_gpu"] == 48 assert fields["num_decode_gpu"] == (30 if decode_workers == "3" else 0) - assert tuple(fields[key] for key in ("decode_tp", "decode_ep", "decode_pp", - "decode_dcp_size", "decode_pcp_size")) == expected_decode + assert ( + tuple( + fields[key] + for key in ("decode_tp", "decode_ep", "decode_pp", "decode_dcp_size", "decode_pcp_size") + ) + == expected_decode + ) assert (tp, ep, attention) == ((5, 11, "true") if decode_workers == "3" else (3, 8, "true")) assert fields["prefill_dp_attention"] == "YES" assert fields["decode_dp_attention"] == "false" -@pytest.mark.parametrize("env,error_type,message", [ - ({"IS_MULTINODE": "true", "TP": "bad"}, ValueError, "invalid literal for int"), - ({"IS_MULTINODE": "true", "PREFILL_PP_SIZE": "0", "PREFILL_HARDWARE": "gpu"}, - SystemExit, "Multinode PP, DCP, and PCP sizes must be positive integers."), - ({"IS_MULTINODE": "true", "DECODE_NUM_WORKERS": "0", "DECODE_PCP_SIZE": "0"}, - SystemExit, "Multinode PP, DCP, and PCP sizes must be positive integers."), -]) +@pytest.mark.parametrize( + "env,error_type,message", + [ + ({"IS_MULTINODE": "true", "TP": "bad"}, ValueError, "invalid literal for int"), + ( + {"IS_MULTINODE": "true", "PREFILL_PP_SIZE": "0", "PREFILL_HARDWARE": "gpu"}, + SystemExit, + "Multinode PP, DCP, and PCP sizes must be positive integers.", + ), + ( + {"IS_MULTINODE": "true", "DECODE_NUM_WORKERS": "0", "DECODE_PCP_SIZE": "0"}, + SystemExit, + "Multinode PP, DCP, and PCP sizes must be positive integers.", + ), + ], +) def test_gpu_shape_preserves_parsing_and_validation_order(env, error_type, message): with pytest.raises(error_type, match=message): _gpu_shape(env) -@pytest.mark.parametrize("error,category", [ - ({"type": "HTTPStatusError", "message": "500 server error"}, "HTTPStatusError"), - (" \t\r\n", "unknown"), - ({"message": " \t\r\n"}, "unknown"), -]) +@pytest.mark.parametrize( + "error,category", + [ + ({"type": "HTTPStatusError", "message": "500 server error"}, "HTTPStatusError"), + (" \t\r\n", "unknown"), + ({"message": " \t\r\n"}, "unknown"), + ], +) def test_processor_surfaces_request_accounting(tmp_path: Path, error, category): result_dir = tmp_path / "results" artifact = result_dir / "aiperf_artifacts" @@ -1351,9 +1411,7 @@ def test_processor_normalizes_sglang_server_metrics(tmp_path: Path, framework: s }, "sglang:realtime_tokens": { "type": "counter", - "series": [ - {"labels": {"mode": "prefill_compute"}, "stats": {"total": 500.0}} - ], + "series": [{"labels": {"mode": "prefill_compute"}, "stats": {"total": 500.0}}], }, } } @@ -1398,9 +1456,7 @@ def test_processor_normalizes_sglang_server_metrics(tmp_path: Path, framework: s if framework == "dynamo-sglang": assert agg["server_metrics"]["kv_cache"]["gpu_total_tokens"] is None assert agg["kv_cache_pool_tokens"] is None - assert any( - "rank replicas are not normalized" in warning for warning in agg["warnings"] - ) + assert any("rank replicas are not normalized" in warning for warning in agg["warnings"]) def test_processor_normalizes_trtllm_server_metrics(tmp_path: Path): @@ -1689,9 +1745,7 @@ def test_processor_uses_aiperf_theoretical_cache_metric(tmp_path: Path): agg = json.loads((tmp_path / "out" / "agg_test.json").read_text()) assert agg["request_metrics"]["cache"]["theoretical_cache_hit_rate"] == pytest.approx(0.25) # output_tokens_expected populated from trace metadata (5 records: A turns 0,1,2 + B turns 0,1) - assert agg["request_metrics"]["tokens"]["output_expected"]["mean"] == pytest.approx( - 55.0 - ) + assert agg["request_metrics"]["tokens"]["output_expected"]["mean"] == pytest.approx(55.0) @pytest.mark.parametrize("cache_state", ["matching", "missing", "ambiguous", "no_identity"]) @@ -1760,29 +1814,61 @@ def test_processor_supports_per_run_subdir_layout(tmp_path: Path): def test_builder_uses_explicit_inputs_without_environment_or_file_access(monkeypatch): records = [ - _make_record(conv_id="root", turn_index=0, isl=100, osl=50, - ttft_ms=30, e2e_ms=1000, itl_ms=10, - start_ns=1_000_000_000, end_ns=2_000_000_000), - _make_record(conv_id="root::sa:child", turn_index=1, isl=200, osl=100, - ttft_ms=50, e2e_ms=2000, itl_ms=20, - start_ns=2_000_000_000, end_ns=4_000_000_000), + _make_record( + conv_id="root", + turn_index=0, + isl=100, + osl=50, + ttft_ms=30, + e2e_ms=1000, + itl_ms=10, + start_ns=1_000_000_000, + end_ns=2_000_000_000, + ), + _make_record( + conv_id="root::sa:child", + turn_index=1, + isl=200, + osl=100, + ttft_ms=50, + e2e_ms=2000, + itl_ms=20, + start_ns=2_000_000_000, + end_ns=4_000_000_000, + ), ] aggregate = {"metadata": {"dataset": {"hf_dataset_name": "example/traces"}}} - server_metrics = {"metrics": {"vllm:prefix_cache_hits": { - "series": [{"stats": {"total": 10}}], - }, "vllm:prefix_cache_queries": {"series": [{"stats": {"total": 40}}]}}} - traces = [{"id": "root", "requests": [ - {"type": "n", "out": 11}, {"type": "tool", "out": 999}, - {"type": "s", "output_length": 29}, - ]}] + server_metrics = { + "metrics": { + "vllm:prefix_cache_hits": { + "series": [{"stats": {"total": 10}}], + }, + "vllm:prefix_cache_queries": {"series": [{"stats": {"total": 40}}]}, + } + } + traces = [ + { + "id": "root", + "requests": [ + {"type": "n", "out": 11}, + {"type": "tool", "out": 999}, + {"type": "s", "output_length": 29}, + ], + } + ] logs = [ "INFO (EngineCore_DP0 pid=10) GPU KV cache size: 800 tokens\n" "INFO (EngineCore_DP0 pid=10) GPU KV cache size: 800 tokens", "INFO (EngineCore_DP0 pid=11) GPU KV cache size: 1,200 tokens", ] - accounting = {"records_total": 4, "records_profiled": 2, "records_dropped_total": 2, - "records_warmup_dropped": 1, "records_error_dropped": 1, - "error_categories": {"Timeout": 1}} + accounting = { + "records_total": 4, + "records_profiled": 2, + "records_dropped_total": 2, + "records_warmup_dropped": 1, + "records_error_dropped": 1, + "error_categories": {"Timeout": 1}, + } env = {"KV_OFFLOADING": "none", "TP": "4", "FRAMEWORK": "vllm"} before = deepcopy((records, aggregate, server_metrics, traces, logs, accounting, env)) @@ -1797,14 +1883,21 @@ class NoEnvironment(dict): isolated.setattr("builtins.open", forbidden) isolated.setattr(Path, "open", forbidden) result = build_result( - records, aggregate, server_metrics, MappingProxyType(env), - request_accounting=accounting, traces=iter(traces), server_logs=iter(logs), + records, + aggregate, + server_metrics, + MappingProxyType(env), + request_accounting=accounting, + traces=iter(traces), + server_logs=iter(logs), ) second = build_result(records, aggregate, {}, {**env, "TP": "2"}) # 450 tokens over three seconds, divided among four GPUs. assert result["request_metrics"]["throughput"]["per_gpu"] == { - "total_tput_tps": 37.5, "output_tput_tps": 12.5, "input_tput_tps": 25.0, + "total_tput_tps": 37.5, + "output_tput_tps": 12.5, + "input_tput_tps": 25.0, } assert second["request_metrics"]["throughput"]["per_gpu"]["total_tput_tps"] == 75.0 assert result["request_metrics"]["tokens"]["output_expected"]["mean"] == 20 @@ -1817,18 +1910,23 @@ class NoEnvironment(dict): assert (records, aggregate, server_metrics, traces, logs, accounting, env) == before -@pytest.mark.parametrize("framework,metric_names", [ - ("atom", ["atom:prompt_tokens"]), - ("trtllm", ["trtllm_kv_cache_max_blocks"]), - ("dynamo-sglang", ["sglang:num_requests_total"]), -]) +@pytest.mark.parametrize( + "framework,metric_names", + [ + ("atom", ["atom:prompt_tokens"]), + ("trtllm", ["trtllm_kv_cache_max_blocks"]), + ("dynamo-sglang", ["sglang:num_requests_total"]), + ], +) def test_builder_does_not_consume_logs_when_backend_does_not_use_them(framework, metric_names): def unavailable_logs(): raise OSError("unreadable optional log") yield # Make failure occur on consumption, as with a file iterator. result = build_result( - [], {}, {"metrics": {name: {"series": []} for name in metric_names}}, + [], + {}, + {"metrics": {name: {"series": []} for name in metric_names}}, {"KV_OFFLOADING": "none", "FRAMEWORK": framework}, server_logs=unavailable_logs(), ) @@ -1840,30 +1938,47 @@ def unavailable_logs(): ] -@pytest.mark.parametrize("override,message", [ - ({"KV_OFFLOADING": ""}, "Missing required environment variable: KV_OFFLOADING"), - ({"PP_SIZE": "0"}, "PP_SIZE, DCP_SIZE, and PCP_SIZE must be positive integers."), - ({"ROUTER_METADATA": "{"}, "ROUTER_METADATA must contain valid JSON"), - ({}, "invalid literal for int() with base 10: 'bad-output'"), -]) +@pytest.mark.parametrize( + "override,message", + [ + ({"KV_OFFLOADING": ""}, "Missing required environment variable: KV_OFFLOADING"), + ({"PP_SIZE": "0"}, "PP_SIZE, DCP_SIZE, and PCP_SIZE must be positive integers."), + ({"ROUTER_METADATA": "{"}, "ROUTER_METADATA must contain valid JSON"), + ({}, "invalid literal for int() with base 10: 'bad-output'"), + ], +) def test_processor_preserves_validation_before_optional_trace_reads(tmp_path, override, message): result_dir = _write_fixture(tmp_path) cache = tmp_path / "hf" snapshot = cache / "datasets--semianalysisai--cc-traces-weka-062126" / "snapshots" / "one" snapshot.mkdir(parents=True) - (snapshot / "traces.json").write_text(json.dumps({ - "id": "trace-A", "requests": [{"type": "n", "out": "bad-output"}], - })) + (snapshot / "traces.json").write_text( + json.dumps( + { + "id": "trace-A", + "requests": [{"type": "n", "out": "bad-output"}], + } + ) + ) out = tmp_path / "out" env = { - "PATH": os.environ.get("PATH", ""), "PYTHONPATH": str(REPO_ROOT), - "RESULT_DIR": str(result_dir), "AGENTIC_OUTPUT_DIR": str(out), - "RESULT_FILENAME": "agg_test", "KV_OFFLOADING": "none", - "FRAMEWORK": "vllm", "HF_HUB_CACHE": str(cache), **override, + "PATH": os.environ.get("PATH", ""), + "PYTHONPATH": str(REPO_ROOT), + "RESULT_DIR": str(result_dir), + "AGENTIC_OUTPUT_DIR": str(out), + "RESULT_FILENAME": "agg_test", + "KV_OFFLOADING": "none", + "FRAMEWORK": "vllm", + "HF_HUB_CACHE": str(cache), + **override, } proc = subprocess.run( [sys.executable, "-m", "infx.results.agentic.process_agentic_result"], - cwd=REPO_ROOT, env=env, text=True, capture_output=True, timeout=30, + cwd=REPO_ROOT, + env=env, + text=True, + capture_output=True, + timeout=30, ) assert proc.returncode == 1 assert proc.stderr.rstrip().endswith(message) @@ -1880,9 +1995,14 @@ def test_processor_trace_cache_precedence_and_last_duplicate(tmp_path, cache_env trace = {"id": "trace-A", "requests": [{"type": "n", "out": 17}] * 3} # Bad JSON is skipped; later JSON overrides earlier JSONL for this trace. (snapshot / "a.jsonl").write_text("{bad json\n" + json.dumps(trace) + "\n") - (snapshot / "z.json").write_text(json.dumps({ - **trace, "requests": [{"type": "s", "output_length": 23}] * 3, - })) + (snapshot / "z.json").write_text( + json.dumps( + { + **trace, + "requests": [{"type": "s", "output_length": 23}] * 3, + } + ) + ) (snapshot / "zz.json").write_text(json.dumps({"id": "trace-A", "requests": []})) env = {"HF_HUB_CACHE": "", "HUGGINGFACE_HUB_CACHE": "", "HF_HOME": ""} env[cache_env] = str(cache) @@ -1903,13 +2023,20 @@ def test_processor_trace_cache_precedence_and_last_duplicate(tmp_path, cache_env assert expected["std"] == 0 -@pytest.mark.parametrize("offsets,expected_mean,expected_p95,windows", [ - ([0, 500_000_000, 1_000_000_000, 1_500_000_000, 2_000_000_000], 2, 2, 2), - ([0, 1_000_000_000, 1_000_000_000, 2_000_000_000], 1.5, 1.95, 2), - ([0, 250_000_000, 500_000_000], 6, None, 0), -]) -def test_qps_windows_exclude_right_boundary_and_preserve_duplicates(offsets, expected_mean, expected_p95, windows): - records = [{"metadata": {"request_end_ns": 1_000_000_000 + offset}} for offset in reversed(offsets)] +@pytest.mark.parametrize( + "offsets,expected_mean,expected_p95,windows", + [ + ([0, 500_000_000, 1_000_000_000, 1_500_000_000, 2_000_000_000], 2, 2, 2), + ([0, 1_000_000_000, 1_000_000_000, 2_000_000_000], 1.5, 1.95, 2), + ([0, 250_000_000, 500_000_000], 6, None, 0), + ], +) +def test_qps_windows_exclude_right_boundary_and_preserve_duplicates( + offsets, expected_mean, expected_p95, windows +): + records = [ + {"metadata": {"request_end_ns": 1_000_000_000 + offset}} for offset in reversed(offsets) + ] flat, nested = compute_qps_stats(records) assert nested["samples"] == windows assert flat["mean_qps"] == pytest.approx(expected_mean) @@ -1921,10 +2048,11 @@ def test_qps_windows_exclude_right_boundary_and_preserve_duplicates(offsets, exp @pytest.mark.parametrize("invalid_turn", [-1, -3, 2]) def test_expected_output_tokens_ignore_out_of_range_trace_turns(invalid_turn): - records = [{"metadata": {"conversation_id": "root", "turn_index": index}} - for index in (0, 1, invalid_turn)] - traces = [{"id": "root", "requests": [{"type": "n", "out": 11}, - {"type": "s", "out": 29}]}] + records = [ + {"metadata": {"conversation_id": "root", "turn_index": index}} + for index in (0, 1, invalid_turn) + ] + traces = [{"id": "root", "requests": [{"type": "n", "out": 11}, {"type": "s", "out": 29}]}] _, metrics = compute_request_metrics(records, traces=traces) assert metrics["tokens"]["output_expected"]["mean"] == 20 assert metrics["tokens"]["output_expected"]["std"] == 9 From 0fda3f1e3a4f6edfa072b10d6d3e39fd10922488 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Wed, 14 Oct 2026 05:48:31 +0000 Subject: [PATCH 02/13] fix(agentx): gate ATOM cache pricing on profiling errors only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit records_error_dropped counts failed requests in every phase, and the ATOM request-scope check refused pricing whenever it was nonzero. A failed warmup request therefore invalidated a profiling window whose admissions and prompt tokens conserved exactly. - load_records_with_accounting adds records_profiling_error_dropped for errors outside warmup; records without a phase still count as profiling. records_error_dropped stays the all-phase total. - request_scope_from_records passes the profiling count as error_records and falls back to the all-phase total for accounting without the split. - A processor-entry regression covers clean, warmup-only, profiling, both-phase and legacy missing-phase errors. Review: SemiAnalysisAI/InferenceX#3964 (andyluo7) records_error_dropped 统计所有阶段的失败请求，而 ATOM 请求范围检查只要它非零就拒绝计价，因此一次 warmup 失败也会让请求数和 prompt token 完全守恒的 profiling 窗口失去计价资格。load_records_with_accounting 新增 records_profiling_error_dropped，只统计 warmup 之外的错误，缺少 phase 的记录仍按 profiling 计；records_error_dropped 保持全阶段总数。request_scope_from_records 以 profiling 错误数作为 error_records，没有该字段的旧记账回退为全阶段总数。新增经处理器入口的回归测试，覆盖正常、仅 warmup 错误、profiling 错误、两个阶段都有错误，以及缺少 phase 的旧记录。 --- .../infx/results/agentic/__init__.py | 1 + .../infx/results/agentic/artifacts.py | 3 + .../infx/results/agentic/server_metrics.py | 7 +- .../agentic/test_atom_cache_accounting.py | 111 +++++++++++++----- .../agentic/test_process_agentic_result.py | 2 + 5 files changed, 92 insertions(+), 32 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/__init__.py b/inferencex-e2e/infx/results/agentic/__init__.py index ba84e85324..a378329a28 100644 --- a/inferencex-e2e/infx/results/agentic/__init__.py +++ b/inferencex-e2e/infx/results/agentic/__init__.py @@ -172,6 +172,7 @@ def build_result( "records_dropped_total": 0, "records_warmup_dropped": 0, "records_error_dropped": 0, + "records_profiling_error_dropped": 0, "error_categories": {}, } diff --git a/inferencex-e2e/infx/results/agentic/artifacts.py b/inferencex-e2e/infx/results/agentic/artifacts.py index a35ee829ae..ef072d274a 100644 --- a/inferencex-e2e/infx/results/agentic/artifacts.py +++ b/inferencex-e2e/infx/results/agentic/artifacts.py @@ -45,6 +45,7 @@ def load_records_with_accounting( "records_dropped_total": 0, "records_warmup_dropped": 0, "records_error_dropped": 0, + "records_profiling_error_dropped": 0, "error_categories": {}, } with open(path) as f: @@ -61,6 +62,8 @@ def load_records_with_accounting( accounting["records_warmup_dropped"] += 1 if error: accounting["records_error_dropped"] += 1 + if not is_warmup: + accounting["records_profiling_error_dropped"] += 1 category = _error_category(error) categories = accounting["error_categories"] categories[category] = categories.get(category, 0) + 1 diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index 311aa264d6..46cbc17684 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -21,13 +21,16 @@ def request_scope_from_records( """Client-side profiling cohort in server token units (``usage_prompt_tokens``). ``profiling_prompt_tokens`` is None when any successful record lacks usage, so a partial - sum is never compared with server counters. + sum is never compared with server counters. ``error_records`` counts failed profiling + requests; accounting without that split falls back to the all-phase error total. """ prompt = extract_per_record_ints(records, "usage_prompt_tokens") return { "profiling_requests": len(records), "profiling_prompt_tokens": sum(prompt) if len(prompt) == len(records) else None, - "error_records": request_accounting.get("records_error_dropped"), + "error_records": request_accounting.get( + "records_profiling_error_dropped", request_accounting.get("records_error_dropped") + ), "cancelled_records": sum( 1 for record in records if (record.get("metadata") or {}).get("was_cancelled") ), diff --git a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py index 90f67ee35c..b82c25f1d3 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py @@ -389,7 +389,29 @@ def test_endpoint_roles_unknown_without_launcher_metadata(env: dict[str, str]) - assert endpoint_roles_from_env(env) is None -def test_processor_entry_point_carries_validity_through_the_serializer(tmp_path: Path) -> None: +DISAGG_ENV = { + "FRAMEWORK": "atom-disagg", + "IS_MULTINODE": "true", + "DISAGG": "true", + **{ + f"{side}_{key}": value + for side in ("PREFILL", "DECODE") + for key, value in { + "NUM_WORKERS": "1", + "TP": "8", + "PP_SIZE": "1", + "DCP_SIZE": "1", + "PCP_SIZE": "1", + "EP": "1", + "DP_ATTN": "false", + "HARDWARE": "mi355x", + }.items() + }, +} +ROLE_ENV = {"SRT_PREFILL_ENDPOINTS": "10.0.0.1:6100", "SRT_DECODE_ENDPOINTS": "10.0.0.2:6100"} + + +def _atom_result_dir(tmp_path: Path) -> tuple[Path, list[dict]]: result_dir = _write_fixture(tmp_path) artifacts = result_dir / "aiperf_artifacts" (artifacts / "server_metrics_export.json").write_text(json.dumps(_pd())) @@ -397,37 +419,25 @@ def test_processor_entry_point_carries_validity_through_the_serializer(tmp_path: records = [json.loads(line) for line in jsonl.read_text().splitlines() if line.strip()] for record in records: record["metrics"]["usage_prompt_tokens"] = {"value": 1000 // len(records), "unit": "tokens"} + return result_dir, records + + +def _write_records(result_dir: Path, records: list[dict]) -> None: + jsonl = result_dir / "aiperf_artifacts" / "profile_export.jsonl" jsonl.write_text("".join(json.dumps(record) + "\n" for record in records)) - disagg_env = { - "FRAMEWORK": "atom-disagg", - "IS_MULTINODE": "true", - "DISAGG": "true", - **{ - f"{side}_{key}": value - for side in ("PREFILL", "DECODE") - for key, value in { - "NUM_WORKERS": "1", - "TP": "8", - "PP_SIZE": "1", - "DCP_SIZE": "1", - "PCP_SIZE": "1", - "EP": "1", - "DP_ATTN": "false", - "HARDWARE": "mi355x", - }.items() - }, - } - agg = _run_processor( - result_dir, - tmp_path / "out", - env_overrides={ - **disagg_env, - "SRT_PREFILL_ENDPOINTS": "10.0.0.1:6100", - "SRT_DECODE_ENDPOINTS": "10.0.0.2:6100", - }, - ) - unresolved = _run_processor(result_dir, tmp_path / "unresolved", env_overrides=disagg_env) + +def _failed(phase: str | None) -> dict: + metadata = {} if phase is None else {"benchmark_phase": phase} + return {"metadata": metadata, "error": {"type": "Timeout", "message": "request timed out"}} + + +def test_processor_entry_point_carries_validity_through_the_serializer(tmp_path: Path) -> None: + result_dir, records = _atom_result_dir(tmp_path) + _write_records(result_dir, records) + + agg = _run_processor(result_dir, tmp_path / "out", env_overrides={**DISAGG_ENV, **ROLE_ENV}) + unresolved = _run_processor(result_dir, tmp_path / "unresolved", env_overrides=DISAGG_ENV) _assert_stable_server_metrics_schema(agg) assert len(records) == 5 @@ -440,3 +450,44 @@ def test_processor_entry_point_carries_validity_through_the_serializer(tmp_path: agg["server_metrics"]["tokens"]["prompt_total"] == (unresolved["server_metrics"]["tokens"]["prompt_total"]) ) + + +@pytest.mark.parametrize( + ("failed_phases", "legacy", "status", "profiling_errors"), + [ + ((), False, "VALID_COMPLETE", 0), + (("warmup",), False, "VALID_COMPLETE", 0), + (("profiling",), False, "REQUEST_SCOPE_UNRESOLVED", 1), + (("warmup", "profiling"), False, "REQUEST_SCOPE_UNRESOLVED", 1), + ((), True, "VALID_COMPLETE", 0), + ((None,), True, "REQUEST_SCOPE_UNRESOLVED", 1), + ], + ids=["clean", "warmup-error", "profiling-error", "both-phases", "legacy-clean", "legacy-error"], +) +def test_only_profiling_errors_from_the_loader_block_pricing( + tmp_path: Path, + failed_phases: tuple[str | None, ...], + legacy: bool, + status: str, + profiling_errors: int, +) -> None: + result_dir, records = _atom_result_dir(tmp_path) + if legacy: + for record in records: + del record["metadata"]["benchmark_phase"] + _write_records(result_dir, [_failed(phase) for phase in failed_phases] + records) + + agg = _run_processor(result_dir, tmp_path / "out", env_overrides={**DISAGG_ENV, **ROLE_ENV}) + validity = agg["server_metrics"]["cache_validity"] + accounting = agg["request_accounting"] + + assert _rates(agg["server_metrics"]) == (0.6, 0.1, 0.1, 0.7) + assert validity["status"] == status + assert validity["pricing_eligible"] is (status == "VALID_COMPLETE") + assert validity["request_scope"]["error_records"] == profiling_errors + assert accounting["records_profiled"] == 5 + assert accounting["records_error_dropped"] == len(failed_phases) + assert accounting["records_profiling_error_dropped"] == profiling_errors + assert accounting["records_warmup_dropped"] == failed_phases.count("warmup") + expected_categories = {"Timeout": len(failed_phases)} if failed_phases else {} + assert accounting["error_categories"] == expected_categories diff --git a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py index 020a5ba3fe..e420a19f5e 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py @@ -67,6 +67,7 @@ "records_dropped_total", "records_warmup_dropped", "records_error_dropped", + "records_profiling_error_dropped", "error_categories", } @@ -955,6 +956,7 @@ def test_processor_surfaces_request_accounting(tmp_path: Path, error, category): "records_dropped_total": 2, "records_warmup_dropped": 1, "records_error_dropped": 1, + "records_profiling_error_dropped": 1, "error_categories": {category: 1}, } assert agg["server_metrics"]["tokens"]["requests_completed"] == 1 From 0842d829685bd5d797588e4f82fe8be129d014aa Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Wed, 14 Oct 2026 05:49:01 +0000 Subject: [PATCH 03/13] style(agentx): trim this PR's comments to the AGENTS.md comment rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops a comment that restated the control flow and a descriptive block above a test fixture, and shortens the decode re-count note to one line. Comment-only. 按 AGENTS.md 的注释规则精简本 PR 自己的注释：删除一条复述控制流的注释和一段测试夹具上方的描述性注释，并把 decode 重复计数的说明压缩为一行。仅改注释。 --- inferencex-e2e/infx/results/agentic/backends/atom.py | 4 +--- inferencex-e2e/infx/tests/bench/test_agentic_command.py | 3 --- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/backends/atom.py b/inferencex-e2e/infx/results/agentic/backends/atom.py index 6e83c0f0d3..681d502f7c 100644 --- a/inferencex-e2e/infx/results/agentic/backends/atom.py +++ b/inferencex-e2e/infx/results/agentic/backends/atom.py @@ -10,8 +10,7 @@ from .base import ServerMetricsBackend, counter_int -# Prefix reuse happens where prompts are prefilled. A P/D decode worker re-counts the same -# prompts in prefix_cache_full_tokens on first admission without serving any reuse. +# P/D decode workers re-count prefilled prompts in prefix_cache_full_tokens without reuse. ACCOUNTED_ROLES = frozenset({"prefill", "unified"}) FULL = "prefix_cache_full_tokens" CACHED = "prefix_cache_cached_tokens" @@ -235,7 +234,6 @@ def refuse(status: str, reason: str) -> dict[str, Any]: offload = 0.0 else: validity["cpu_component"] = "MISSING" - # The invariants hold on every CPU branch, including the contract zero set just above. if not math.isfinite(full) or full <= 0: return refuse("INVALID_COUNTER", "no prompt tokens in the accounting window") if not math.isfinite(cached) or not 0 <= cached <= full: diff --git a/inferencex-e2e/infx/tests/bench/test_agentic_command.py b/inferencex-e2e/infx/tests/bench/test_agentic_command.py index 7fb317fdd2..38c4b5eec5 100644 --- a/inferencex-e2e/infx/tests/bench/test_agentic_command.py +++ b/inferencex-e2e/infx/tests/bench/test_agentic_command.py @@ -389,9 +389,6 @@ def frontend(method: str, path: str) -> tuple[int, object]: assert (aggregate["conc"], aggregate["num_requests_total"]) == (8, 1) -# The replay writes ATOM P/D counters for exactly the endpoints the launcher told it to scrape. -# The prefill endpoint is the one the servers were started with (SRT_PREFILL_ENDPOINTS), not a -# position in argv. FAKE_ATOM_AIPERF_SCRIPT = ( FAKE_AIPERF_SCRIPT + r""" From b10af712931157bf82e9d9d80c7a756ccb3169b2 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Fri, 16 Oct 2026 05:36:30 +0000 Subject: [PATCH 04/13] fix(agentx): phase-validity guard and counter-based TRT-LLM cache rate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TRT-LLM adapter averaged the per-worker trtllm_kv_cache_hit_rate gauges, decode workers included at 0, and skipped the counters because AIPerf 0.12 exports them without _total. On GB300 dynamo-trt run 38198820297 it published 0.401; the prefill counters give 0.970 once the warmup increment is removed. No adapter could say whether a published rate covers the profiling window. - phase_validity.py: per-series window validity for every adapter, from summary.phase_time_ranges, the timeslices and warmup_metrics, with no fixed time tolerance. - TRT-LLM: the combined rate comes from the counters per counting domain (prefill or combined role) and is written to overall_cache_hit_rate only as interval telemetry (CONDITIONAL_INTERVAL). Raw counts, the raw ratio and the reasons stay in validity.cache_rate; gpu_cache_hit_rate stays null and exact eligibility stays false. - cache_validity carries one core for every adapter: status_class, published, pricing_eligible and basis. ATOM keeps its own fields. - This branch's adapter context reaches ATOM as adapter_context, and apply_validity runs after populate for every adapter. Refs: SemiAnalysisAI/InferenceX#3964 TRT-LLM 适配器把各 worker 的 trtllm_kv_cache_hit_rate gauge 做不加权平均（decode 的 0 也算在内），又因为 AIPerf 0.12 导出的计数器名不带 _total 而跳过了计数器：GB300 dynamo-trt 运行 38198820297 发布 0.401，去掉 warmup 增量后 prefill 计数器得到 0.970。新增 phase_validity.py，为所有适配器给出逐序列的窗口有效性；TRT 改用计数器按计数域计算 combined 命中率，只作为区间遥测（CONDITIONAL_INTERVAL）写入 overall，原始计数、原始比值和原因保留在 validity.cache_rate。cache_validity 为所有适配器提供统一核心（status_class、published、pricing_eligible、basis），ATOM 保留原有字段。 --- .../infx/results/agentic/__init__.py | 1 + .../infx/results/agentic/backends/base.py | 28 +- .../infx/results/agentic/backends/trtllm.py | 408 ++++++++++++--- .../infx/results/agentic/cache_validity.py | 72 +++ .../infx/results/agentic/phase_validity.py | 382 ++++++++++++++ .../infx/results/agentic/server_metrics.py | 28 +- .../agentic/test_atom_cache_accounting.py | 15 + .../agentic/test_cache_validity_core.py | 68 +++ .../agentic/test_process_agentic_result.py | 11 +- .../agentic/test_server_metrics_closeout.py | 473 ++++++++++++++++++ .../test_server_metrics_phase_validity.py | 411 +++++++++++++++ 11 files changed, 1809 insertions(+), 88 deletions(-) create mode 100644 inferencex-e2e/infx/results/agentic/cache_validity.py create mode 100644 inferencex-e2e/infx/results/agentic/phase_validity.py create mode 100644 inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py create mode 100644 inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py create mode 100644 inferencex-e2e/infx/tests/results/agentic/test_server_metrics_phase_validity.py diff --git a/inferencex-e2e/infx/results/agentic/__init__.py b/inferencex-e2e/infx/results/agentic/__init__.py index a378329a28..d67fa674a2 100644 --- a/inferencex-e2e/infx/results/agentic/__init__.py +++ b/inferencex-e2e/infx/results/agentic/__init__.py @@ -225,6 +225,7 @@ def build_result( endpoint_roles=endpoint_roles_from_env(env), request_scope=request_scope_from_records(records, request_accounting), deployment=deployment_from_env(env), + disaggregated=agg["disagg"] if env.get("DISAGG") not in (None, "") else None, ) if "total_tput_tps" in request_flat and num_gpus > 0: diff --git a/inferencex-e2e/infx/results/agentic/backends/base.py b/inferencex-e2e/infx/results/agentic/backends/base.py index 370897dd92..da6e68b463 100644 --- a/inferencex-e2e/infx/results/agentic/backends/base.py +++ b/inferencex-e2e/infx/results/agentic/backends/base.py @@ -3,7 +3,10 @@ from __future__ import annotations from collections.abc import Iterable -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from infx.results.agentic.phase_validity import PhaseContext SERVER_CACHE_FLAT_FIELDS = { "server_gpu_cache_hit_rate": None, @@ -45,6 +48,15 @@ def gpu_kv_capacity_tokens( ) -> int | None: return None + def apply_validity( + self, + metrics: dict[str, dict[str, Any]], # noqa: ARG002 + flat: dict[str, Any], # noqa: ARG002 + nested: dict[str, Any], # noqa: ARG002 + context: PhaseContext, # noqa: ARG002 + ) -> None: + return None + def empty_server_metrics(present: bool, metric_count: int) -> dict[str, Any]: return { @@ -95,6 +107,20 @@ def empty_server_metrics(present: bool, metric_count: int) -> dict[str, Any]: }, }, "sources": [], + "validity": { + "phase_windows": None, + "endpoints": {"configured": 0, "unscraped": []}, + "deployment": {"disaggregated": None}, + "series_by_status": {}, + "metrics_by_status": {}, + "cache_rate": { + "status": "NOT_EVALUATED", + "published": False, + "eligible_exact_actual": False, + "reasons": ["adapter_does_not_assess_cache_rate"], + }, + "diagnostics": {}, + }, } diff --git a/inferencex-e2e/infx/results/agentic/backends/trtllm.py b/inferencex-e2e/infx/results/agentic/backends/trtllm.py index aba19a03c6..ec85c879c2 100644 --- a/inferencex-e2e/infx/results/agentic/backends/trtllm.py +++ b/inferencex-e2e/infx/results/agentic/backends/trtllm.py @@ -2,19 +2,44 @@ from __future__ import annotations -from collections.abc import Iterable +import math +import statistics +from collections.abc import Callable, Iterable from typing import Any from infx.results.agentic.common import ( gauge_stat, label_value, + metric_series, normalize_fraction, rate, + series_stat, sum_stat, ) +from infx.results.agentic.phase_validity import ( + BOUNDARY_UNCERTAIN, + CONDITIONAL_INTERVAL, + PARTIAL_COVERAGE, + REJECTED, + UNKNOWN, + VALID, + PhaseContext, + classify_series, + resolve_aliases, + series_key, + worst, +) from .base import ServerMetricsBackend, counter_int +PROMPT_TOKENS = "trtllm_prompt_tokens" +CACHED_TOKENS = "trtllm_prompt_cached_tokens" +GENERATION_TOKENS = "trtllm_generation_tokens" +# Server counters are interval totals: the producer cannot bind them to the +# billed request set or check them against the expected worker count. +UNBOUND_REASONS = ("request_scope_not_bound", "expected_instances_not_verified") +KNOWN_ROLES = ("prefill", "decode", "combined") + class TrtllmBackend(ServerMetricsBackend): name = "trtllm" @@ -31,33 +56,16 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: - prompt_total = _first_counter_total( - metrics, - ["dynamo_frontend_input_sequence_tokens", "trtllm_prompt_tokens_total"], - ) - generation_total = _first_counter_total( - metrics, - ["dynamo_frontend_output_tokens", "trtllm_generation_tokens_total"], - ) - cached_tokens = sum_stat( - metrics, - "trtllm_prompt_cached_tokens_total", - preferred_keys=("total", "sum", "max", "avg"), - ) + prompt_total = _first_total(metrics, "dynamo_frontend_input_sequence_tokens", PROMPT_TOKENS) + generation_total = _first_total(metrics, "dynamo_frontend_output_tokens", GENERATION_TOKENS) + diagnostic = _hit_rate_gauge_diagnostic(metrics) + if diagnostic is not None: + nested["validity"]["diagnostics"]["trtllm_kv_cache_hit_rate"] = diagnostic flat["total_prompt_tokens"] = counter_int(prompt_total) flat["total_generation_tokens"] = counter_int(generation_total) - flat["server_gpu_cache_hit_rate"] = normalize_fraction( - gauge_stat( - metrics, - "trtllm_kv_cache_hit_rate", - preferred_keys=("avg", "max", "total"), - combine="avg", - ) - ) - if flat["server_gpu_cache_hit_rate"] is None: - flat["server_gpu_cache_hit_rate"] = rate(cached_tokens, prompt_total) - flat["server_overall_cache_hit_rate"] = flat["server_gpu_cache_hit_rate"] + flat["server_gpu_cache_hit_rate"] = None + flat["server_overall_cache_hit_rate"] = None flat["gpu_kv_cache_usage_pct"] = normalize_fraction( gauge_stat( @@ -75,28 +83,14 @@ def populate( combine="max", ) ) - flat["kv_offload_bytes_gpu_to_cpu"] = sum_stat( - metrics, - "trtllm_kv_cache_offload_bytes_total", - preferred_keys=("total", "sum", "max", "avg"), - ) - flat["kv_offload_bytes_cpu_to_gpu"] = sum_stat( - metrics, - "trtllm_kv_cache_onboard_bytes_total", - preferred_keys=("total", "sum", "max", "avg"), - ) - - computed_tokens = None - if prompt_total is not None and cached_tokens is not None: - computed_tokens = max(prompt_total - cached_tokens, 0.0) + flat["kv_offload_bytes_gpu_to_cpu"] = _alias_total(metrics, "trtllm_kv_cache_offload_bytes") + flat["kv_offload_bytes_cpu_to_gpu"] = _alias_total(metrics, "trtllm_kv_cache_onboard_bytes") nested["cache"].update( { - "gpu_cache_hit_rate": flat["server_gpu_cache_hit_rate"], - "overall_cache_hit_rate": flat["server_overall_cache_hit_rate"], - "cached_tokens_by_source": {"device": cached_tokens} - if cached_tokens is not None - else {}, + "gpu_cache_hit_rate": None, + "overall_cache_hit_rate": None, + "cached_tokens_by_source": {}, } ) nested["kv_cache"].update( @@ -116,15 +110,38 @@ def populate( "prompt_total": flat["total_prompt_tokens"], "generation_total": flat["total_generation_tokens"], "prompt_by_source": { - "gpu_cache_hit": cached_tokens, + "gpu_cache_hit": None, "cpu_or_external_cache_hit": None, - "computed": computed_tokens, - "raw": {"device": cached_tokens} if cached_tokens is not None else {}, + "computed": None, + "raw": {}, }, } ) nested["sources"] = _trtllm_sources(metrics) + def apply_validity( + self, + metrics: dict[str, dict[str, Any]], + flat: dict[str, Any], + nested: dict[str, Any], + context: PhaseContext, + ) -> None: + reuse = _reuse_rate(metrics, context) + nested["validity"]["cache_rate"] = reuse + if not reuse["published"]: + return + cached, prompt = reuse["cached_tokens"], reuse["prompt_tokens"] + flat["server_overall_cache_hit_rate"] = reuse["raw_rate"] + nested["cache"].update( + { + "overall_cache_hit_rate": reuse["raw_rate"], + "cached_tokens_by_source": {"combined": cached}, + } + ) + nested["tokens"]["prompt_by_source"].update( + {"computed": prompt - cached, "raw": {"combined": cached}} + ) + def gpu_kv_capacity_tokens( self, metrics: dict[str, dict[str, Any]], @@ -147,24 +164,239 @@ def gpu_kv_capacity_tokens( return counter_int(max_blocks * tokens_per_block) -def _first_counter_total( +def _reuse_rate(metrics: dict[str, dict[str, Any]], context: PhaseContext) -> dict[str, Any]: + """Cached/prompt counters in one deployment scope, classified and never corrected. + + In P/D deployments decode workers report every transferred prompt token as + cached, so only prefill series measure reuse; a unified deployment needs an + authoritative mode or explicit combined labels. Numerator and denominator + are paired per endpoint and label set and must satisfy 0 <= cached <= + prompt in every pair. The cached counter can include host-tier reuse, so + the result is a combined-tier rate. + """ + findings: list[tuple[str, str]] = [] + if context.windows is None: + findings.append((UNKNOWN, "phase_window_missing")) + resolved: dict[str, list[dict[str, Any]]] = {} + conflicts: list[dict[str, Any]] = [] + for metric in (PROMPT_TOKENS, CACHED_TOKENS): + resolved[metric], metric_conflicts = resolve_aliases(metrics, metric) + conflicts.extend(metric_conflicts) + findings.extend((REJECTED, conflict["kind"]) for conflict in conflicts) + + endpoint_roles = _endpoint_role_sets(metrics) + labelled = [] + for metric, series_list in resolved.items(): + for series in series_list: + role = _series_role(series, endpoint_roles) + labelled.append((metric, series, role if role in KNOWN_ROLES else "unknown")) + if any(len(roles) > 1 for roles in endpoint_roles.values()): + findings.append((UNKNOWN, "role_conflict")) + present = {role for _, _, role in labelled} + scope, scope_findings = _deployment_scope(present, context.disaggregated) + findings.extend(scope_findings) + + in_scope: dict[str, list[dict[str, Any]]] = {PROMPT_TOKENS: [], CACHED_TOKENS: []} + excluded: list[dict[str, Any]] = [] + unresolved: list[dict[str, Any]] = [] + for metric, series, role in labelled: + entry = { + "metric": metric, + "endpoint_url": str(series.get("endpoint_url", "")), + "role": role, + } + if role == "unknown" and context.disaggregated is False: + role = "combined" + if scope is not None and role == scope: + record = classify_series( + series, + "counter", + context.windows, + has_reference=context.has_reference(metric, series), + warmup_total=context.warmup_total(metric, series), + ) + in_scope[metric].append( + {**entry, "role": role, "key": series_key(metric, series), **record} + ) + findings.extend((record["status"], code) for code in record["reasons"]) + elif scope == "prefill" and role == "decode": + excluded.append({**entry, "reason": "decode_role_excluded"}) + elif role == "unknown": + unresolved.append(entry) + if unresolved and scope is not None: + findings.append((UNKNOWN, "role_unknown")) + + domains = _pair_domains(in_scope, findings) if scope is not None else [] + if context.unscraped is None: + findings.append((UNKNOWN, "scrape_coverage_unknown")) + elif context.unscraped: + findings.append((PARTIAL_COVERAGE, "endpoint_not_scraped")) + + prompt = _finite_sum(r["total"] for r in in_scope[PROMPT_TOKENS]) + cached = _finite_sum(r["total"] for r in in_scope[CACHED_TOKENS]) + raw_rate = rate(cached, prompt) + status = worst(status for status, _ in findings) + if status == VALID and raw_rate is None: + status = UNKNOWN + findings.append((UNKNOWN, "zero_denominator")) + published = status == VALID + if published: + status = CONDITIONAL_INTERVAL + for records in in_scope.values(): + for record in records: + record.pop("key") + return { + "status": status, + "published": published, + "eligible_exact_actual": False, + "role_scope": scope or "unresolved", + "deployment_disaggregated": context.disaggregated, + "tier": "combined", + "numerator": CACHED_TOKENS, + "denominator": PROMPT_TOKENS, + "cached_tokens": cached, + "prompt_tokens": prompt, + "raw_rate": raw_rate, + "reasons": list(dict.fromkeys([code for _, code in findings] + list(UNBOUND_REASONS))), + "series": in_scope[PROMPT_TOKENS] + in_scope[CACHED_TOKENS], + "domains": domains, + "excluded": excluded, + "unresolved": unresolved, + "alias_conflicts": conflicts, + } + + +def _deployment_scope( + present: set[str], disaggregated: bool | None +) -> tuple[str | None, list[tuple[str, str]]]: + if disaggregated is True: + findings = [(UNKNOWN, "role_conflicts_with_deployment")] if "combined" in present else [] + if "prefill" in present: + return "prefill", findings + return None, [*findings, (UNKNOWN, "no_prefill_scope")] + if disaggregated is False: + if present & {"prefill", "decode"}: + return None, [(UNKNOWN, "role_conflicts_with_deployment")] + return "combined", [] + if "prefill" in present: + if "combined" in present: + return None, [(UNKNOWN, "role_conflict")] + return "prefill", [] + if "combined" in present and "decode" not in present: + return "combined", [] + if "decode" in present: + return None, [(UNKNOWN, "decode_only_no_prefill_scope")] + return None, [(UNKNOWN, "deployment_mode_unknown")] + + +def _pair_domains( + in_scope: dict[str, list[dict[str, Any]]], findings: list[tuple[str, str]] +) -> list[dict[str, Any]]: + prompt = {r["key"][1:]: r for r in in_scope[PROMPT_TOKENS]} + cached = {r["key"][1:]: r for r in in_scope[CACHED_TOKENS]} + if not prompt and not cached: + findings.append((UNKNOWN, "counter_series_missing")) + return [] + if set(prompt) != set(cached): + findings.append((PARTIAL_COVERAGE, "missing_pair")) + domains = [] + for key in sorted(set(prompt) & set(cached)): + f, k = prompt[key], cached[key] + supports = [(r["first_slice_offset_s"], r["last_slice_offset_s"]) for r in (f, k)] + windows_match = None + if None not in supports[0] + supports[1]: + windows_match = supports[0] == supports[1] + if not windows_match: + findings.append((BOUNDARY_UNCERTAIN, "window_support_mismatch")) + within = None + if _finite(f["total"]) and _finite(k["total"]): + within = 0 <= k["total"] <= f["total"] + if not within: + findings.append((REJECTED, "cached_exceeds_prompt")) + domains.append( + { + "endpoint_url": key[0], + "labels": dict(key[1]), + "prompt": f["total"], + "cached": k["total"], + "cached_within_prompt": within, + "windows_match": windows_match, + } + ) + return domains + + +def _hit_rate_gauge_diagnostic(metrics: dict[str, dict[str, Any]]) -> dict[str, Any] | None: + roles = _endpoint_role_sets(metrics) + values: list[float] = [] + by_role: dict[str, list[float]] = {} + for series in metric_series(metrics, "trtllm_kv_cache_hit_rate"): + value = normalize_fraction(series_stat(series, ("avg", "max", "total"))) + if value is None: + continue + values.append(value) + by_role.setdefault(_series_role(series, roles), []).append(value) + if not values: + return None + return { + "status": "GAUGE_DERIVED", + "pricing_eligible": False, + "all_series_avg": statistics.mean(values), + "by_role": {role: statistics.mean(v) for role, v in sorted(by_role.items())}, + } + + +def _first_total( + metrics: dict[str, dict[str, Any]], frontend_metric: str, trtllm_metric: str +) -> float | None: + frontend = sum_stat(metrics, frontend_metric, preferred_keys=("total", "sum", "max", "avg")) + if frontend is not None: + return frontend + return _alias_total(metrics, trtllm_metric) + + +def _alias_total( metrics: dict[str, dict[str, Any]], - metric_names: list[str], + base: str, + series_filter: Callable[[dict[str, Any]], bool] | None = None, ) -> float | None: - for metric_name in metric_names: - value = sum_stat( - metrics, - metric_name, - preferred_keys=("total", "sum", "max", "avg"), - ) - if value is not None: - return value - return None + series_list, conflicts = resolve_aliases(metrics, base) + if any( + series_filter is None or series_filter({"endpoint_url": c["endpoint_url"]}) + for c in conflicts + ): + return None + values = [ + series_stat(series, ("total", "sum", "max", "avg")) + for series in series_list + if series_filter is None or series_filter(series) + ] + values = [value for value in values if value is not None] + return sum(values) if values else None -def _trtllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: - endpoints: set[str] = set() - roles: dict[str, str] = {} +def _finite(value: float | None) -> bool: + return value is not None and math.isfinite(value) + + +def _finite_sum(values: Iterable[float | None]) -> float | None: + total = 0.0 + found = False + for value in values: + if not _finite(value): + return None + total += value + found = True + return total if found else None + + +def _label_role(series: dict[str, Any]) -> str | None: + mode = label_value(series, "disaggregation_mode") or label_value(series, "dynamo_component") + return _normalize_role(mode) if mode else None + + +def _endpoint_role_sets(metrics: dict[str, dict[str, Any]]) -> dict[str, set[str]]: + roles: dict[str, set[str]] = {} for entry in metrics.values(): if not isinstance(entry, dict): continue @@ -174,13 +406,35 @@ def _trtllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: for series in series_list: if not isinstance(series, dict) or not series.get("endpoint_url"): continue - endpoint = str(series["endpoint_url"]) - endpoints.add(endpoint) - mode = label_value(series, "disaggregation_mode") or label_value( - series, "dynamo_component" - ) - if mode: - roles[endpoint] = _normalize_role(mode) + role = _label_role(series) + if role: + roles.setdefault(str(series["endpoint_url"]), set()).add(role) + return roles + + +def _series_role(series: dict[str, Any], roles: dict[str, set[str]]) -> str: + own = _label_role(series) + if own: + return own + endpoint_roles = roles.get(str(series.get("endpoint_url", "")), set()) + return next(iter(endpoint_roles)) if len(endpoint_roles) == 1 else "unknown" + + +def _trtllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + endpoints: set[str] = set() + for entry in metrics.values(): + if not isinstance(entry, dict): + continue + series_list = entry.get("series") + if not isinstance(series_list, list): + continue + for series in series_list: + if isinstance(series, dict) and series.get("endpoint_url"): + endpoints.add(str(series["endpoint_url"])) + roles = { + endpoint: next(iter(found)) if len(found) == 1 else "unknown" + for endpoint, found in _endpoint_role_sets(metrics).items() + } sources: list[dict[str, Any]] = [] for endpoint in sorted(endpoints): @@ -188,21 +442,9 @@ def _trtllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: def series_filter(series: dict, endpoint: str = endpoint) -> bool: return str(series.get("endpoint_url", "")) == endpoint - prompt_tokens = sum_stat( - metrics, - "trtllm_prompt_tokens_total", - series_filter=series_filter, - ) - generation_tokens = sum_stat( - metrics, - "trtllm_generation_tokens_total", - series_filter=series_filter, - ) - cached_tokens = sum_stat( - metrics, - "trtllm_prompt_cached_tokens_total", - series_filter=series_filter, - ) + prompt_tokens = _alias_total(metrics, PROMPT_TOKENS, series_filter) + generation_tokens = _alias_total(metrics, GENERATION_TOKENS, series_filter) + cached_tokens = _alias_total(metrics, CACHED_TOKENS, series_filter) hit_rate = normalize_fraction( gauge_stat( metrics, diff --git a/inferencex-e2e/infx/results/agentic/cache_validity.py b/inferencex-e2e/infx/results/agentic/cache_validity.py new file mode 100644 index 0000000000..446cd013e2 --- /dev/null +++ b/inferencex-e2e/infx/results/agentic/cache_validity.py @@ -0,0 +1,72 @@ +"""Consumer-facing cache validity core, filled for every adapter. + +``status`` keeps the adapter's own state and ``status_class`` maps it onto one vocabulary, so a +consumer can apply a single rule to every producer. ``published`` says whether any standard cache +rate was written, ``pricing_eligible`` whether it may be priced as an exact actual observation, and +``basis`` whether the written rates are disjoint tiers or one combined total. +""" + +from __future__ import annotations + +from typing import Any + +CORE_SCHEMA = "infx-cache-validity-core/1" +STATUS_CLASSES = ( + "VALID_COMPLETE", + "VALID_ZERO", + "VALID_INTERVAL", + "PARTIAL", + "UNRESOLVED", + "CONTAMINATED", + "INVALID", + "MISSING", + "NOT_EVALUATED", +) +_STATUS_CLASS = { + "VALID_COMPLETE": "VALID_COMPLETE", + "VALID_ZERO": "VALID_ZERO", + "PARTIAL_COMPONENT": "PARTIAL", + "WINDOW_UNRESOLVED": "UNRESOLVED", + "ROLE_OR_COVERAGE_UNRESOLVED": "UNRESOLVED", + "REQUEST_SCOPE_UNRESOLVED": "UNRESOLVED", + "MISSING_OBSERVATION": "MISSING", + "INVALID_COUNTER": "INVALID", + "CONDITIONAL_INTERVAL": "VALID_INTERVAL", + "UNKNOWN": "UNRESOLVED", + "BOUNDARY_RESIDUAL": "UNRESOLVED", + "BOUNDARY_UNCERTAIN": "UNRESOLVED", + "PARTIAL_COVERAGE": "PARTIAL", + "PHASE_CONTAMINATED": "CONTAMINATED", + "REJECTED": "INVALID", + "NOT_EVALUATED": "NOT_EVALUATED", +} +_TIER_RATES = ("gpu_cache_hit_rate", "cpu_cache_hit_rate", "external_cache_hit_rate") + + +def summarize_cache_validity(nested: dict[str, Any]) -> dict[str, Any]: + cache = nested["cache"] + tiers = any(cache.get(key) is not None for key in _TIER_RATES) + combined = cache.get("overall_cache_hit_rate") is not None + core = { + "core_schema": CORE_SCHEMA, + "published": tiers or combined, + "basis": "tiers" if tiers else "combined" if combined else None, + } + detail = nested.get("cache_validity") + if isinstance(detail, dict): + return { + **detail, + **core, + "status_class": _STATUS_CLASS.get(detail.get("status"), "UNRESOLVED"), + } + rate = nested["validity"]["cache_rate"] + present = bool(nested.get("present")) + status = rate.get("status") if present else "MISSING_OBSERVATION" + return { + "schema": CORE_SCHEMA, + "status": status, + "status_class": _STATUS_CLASS.get(status, "UNRESOLVED"), + "pricing_eligible": present and bool(rate.get("eligible_exact_actual")), + "reasons": list(rate.get("reasons") or []) if present else ["server_metrics_missing"], + **core, + } diff --git a/inferencex-e2e/infx/results/agentic/phase_validity.py b/inferencex-e2e/infx/results/agentic/phase_validity.py new file mode 100644 index 0000000000..ee424b604d --- /dev/null +++ b/inferencex-e2e/infx/results/agentic/phase_validity.py @@ -0,0 +1,382 @@ +"""Phase validity of AIPerf server-metric counter and histogram series. + +AIPerf exports each series' profiling delta (``stats``) next to 1 s timeslices +whose window runs from the first to the last PROFILING-tagged sample of that +series. ``stats`` minus the timeslice sum is the increment between the reference +sample and the first PROFILING sample. Series are classified here and never +corrected: the exporter clamps negative deltas, so the values cannot be repaired +from the export alone, and no time tolerance is used to treat activity at or +outside the phase boundary as profiling traffic. +""" + +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from .common import to_float + +VALID = "VALID" +UNKNOWN = "UNKNOWN" +BOUNDARY_RESIDUAL = "BOUNDARY_RESIDUAL" +BOUNDARY_UNCERTAIN = "BOUNDARY_UNCERTAIN" +PARTIAL_COVERAGE = "PARTIAL_COVERAGE" +PHASE_CONTAMINATED = "PHASE_CONTAMINATED" +REJECTED = "REJECTED" +_SEVERITY = { + VALID: 0, + UNKNOWN: 1, + BOUNDARY_RESIDUAL: 2, + BOUNDARY_UNCERTAIN: 3, + PARTIAL_COVERAGE: 4, + PHASE_CONTAMINATED: 5, + REJECTED: 6, +} +# Rate built only from VALID series: an interval measurement, not a request-bound actual. +CONDITIONAL_INTERVAL = "CONDITIONAL_INTERVAL" + +NS_PER_S = 1_000_000_000 +COUNT_TOLERANCE = 0.5 +SUM_RELATIVE_TOLERANCE = 1e-6 + + +@dataclass(frozen=True) +class PhaseWindows: + profiling: tuple[int, int] + warmup: tuple[int, int] | None + + def as_dict(self) -> dict[str, list[int] | None]: + return { + "profiling": list(self.profiling), + "warmup": list(self.warmup) if self.warmup else None, + } + + +@dataclass(frozen=True) +class PhaseContext: + """What a backend needs to judge its own series; kept out of the published result. + + ``unscraped`` is None when the export does not say which endpoints were + configured and scraped. ``disaggregated`` is None when no authoritative + deployment mode was supplied. + """ + + windows: PhaseWindows | None + warmup_totals: Mapping[tuple, float | None] = field(default_factory=dict) + unscraped: tuple[str, ...] | None = () + disaggregated: bool | None = None + + def has_reference(self, metric: str, series: Mapping[str, Any]) -> bool: + return series_key(metric, series) in self.warmup_totals + + def warmup_total(self, metric: str, series: Mapping[str, Any]) -> float | None: + return self.warmup_totals.get(series_key(metric, series)) + + +def base_name(metric: str) -> str: + return metric.removesuffix("_total") + + +def series_key(metric: str, series: Mapping[str, Any]) -> tuple: + labels = series.get("labels") if isinstance(series.get("labels"), Mapping) else {} + return ( + base_name(metric), + str(series.get("endpoint_url", "")), + tuple(sorted((str(k), str(v)) for k, v in labels.items())), + ) + + +def phase_windows(server_metrics: Any) -> PhaseWindows | None: + summary = server_metrics.get("summary") if isinstance(server_metrics, Mapping) else None + ranges = summary.get("phase_time_ranges") if isinstance(summary, Mapping) else None + if not isinstance(ranges, Mapping): + return None + profiling = _time_range(ranges.get("profiling")) + if profiling is None: + return None + return PhaseWindows(profiling, _time_range(ranges.get("warmup"))) + + +def warmup_totals(server_metrics: Any) -> dict[tuple, float | None]: + """Warmup-phase delta of every series exported under ``warmup_metrics``. + + Presence means the series has a sample before its first PROFILING sample; a + zero total means no increment was counted during warmup. + """ + warmup = server_metrics.get("warmup_metrics") if isinstance(server_metrics, Mapping) else None + totals: dict[tuple, float | None] = {} + for name, entry in warmup.items() if isinstance(warmup, Mapping) else (): + if not isinstance(entry, Mapping): + continue + key_name = "count" if entry.get("type") == "histogram" else "total" + series_list = entry.get("series") + for series in series_list if isinstance(series_list, list) else (): + if isinstance(series, Mapping): + stats = series.get("stats") if isinstance(series.get("stats"), Mapping) else {} + totals[series_key(name, series)] = to_float(stats.get(key_name)) + return totals + + +def export_context(server_metrics: Any, windows: PhaseWindows | None) -> dict[str, Any]: + summary = server_metrics.get("summary") if isinstance(server_metrics, Mapping) else None + summary = summary if isinstance(summary, Mapping) else {} + configured = summary.get("endpoints_configured") + successful = summary.get("endpoints_successful") + if isinstance(configured, list) and isinstance(successful, list): + scraped = {str(url) for url in successful} + endpoints = { + "configured": len(configured), + "unscraped": [str(url) for url in configured if str(url) not in scraped], + } + else: + endpoints = {"configured": None, "unscraped": None} + return {"phase_windows": windows.as_dict() if windows else None, "endpoints": endpoints} + + +def worst(statuses: Iterable[str]) -> str: + return max(statuses, key=_SEVERITY.__getitem__, default=VALID) + + +def classify_series( + series: Mapping[str, Any], + metric_type: str, + windows: PhaseWindows | None, + *, + has_reference: bool = False, + warmup_total: float | None = None, +) -> dict[str, Any]: + histogram = metric_type == "histogram" + keys = ("sum", "count") if histogram else ("total",) + stats = series.get("stats") + stats = stats if isinstance(stats, Mapping) else {} + totals = {key: to_float(stats.get(key)) for key in keys} + raw_slices = series.get("timeslices") + slices = ( + [s for s in raw_slices if isinstance(s, Mapping)] if isinstance(raw_slices, list) else [] + ) + findings: list[tuple[str, str]] = [] + record: dict[str, Any] = { + "endpoint_url": series.get("endpoint_url"), + "total": totals[keys[0]], + "slice_sum": None, + "residual": None, + "first_slice_offset_s": None, + "last_slice_offset_s": None, + } + + for value in totals.values(): + if value is None: + findings.append((UNKNOWN, "missing_total")) + elif not math.isfinite(value): + findings.append((REJECTED, "nonfinite_total")) + elif value < 0: + findings.append((REJECTED, "negative_total")) + if windows is None: + findings.append((UNKNOWN, "phase_window_missing")) + if not slices: + findings.append((UNKNOWN, "no_timeslices")) + else: + findings.extend( + _slice_findings(slices, windows, record, histogram, has_reference, warmup_total) + ) + for key in keys: + slice_sum = _slice_sum(slices, key, histogram) + if slice_sum is None: + findings.append((REJECTED, "negative_or_nonfinite_slice")) + continue + total = totals[key] + if key == keys[0]: + record["slice_sum"] = slice_sum + if total is None or not math.isfinite(total): + continue + residual = total - slice_sum + if key == keys[0]: + record["residual"] = residual + tolerance = COUNT_TOLERANCE + if key == "sum": + tolerance = max(COUNT_TOLERANCE, SUM_RELATIVE_TOLERANCE * abs(total)) + if residual > tolerance: + findings.append((BOUNDARY_RESIDUAL, "total_exceeds_slice_sum")) + elif residual < -tolerance: + findings.append((REJECTED, "slice_sum_exceeds_total")) + + record["status"] = worst(status for status, _ in findings) + record["reasons"] = list(dict.fromkeys(code for _, code in findings)) + return record + + +def summarize( + metrics: Mapping[str, Any], + windows: PhaseWindows | None, + warmup: Mapping[tuple, float | None] | None = None, +) -> dict[str, Any]: + warmup = warmup or {} + series_by_status: Counter[str] = Counter() + metrics_by_status: dict[str, list[str]] = {} + for name in sorted(metrics): + entry = metrics[name] + if not isinstance(entry, Mapping) or entry.get("type") not in ("counter", "histogram"): + continue + series_list = entry.get("series") + if not isinstance(series_list, list): + continue + statuses = [ + classify_series( + series, + entry["type"], + windows, + has_reference=series_key(name, series) in warmup, + warmup_total=warmup.get(series_key(name, series)), + )["status"] + for series in series_list + if isinstance(series, Mapping) + ] + series_by_status.update(statuses) + status = worst(statuses) + if statuses and status != VALID: + metrics_by_status.setdefault(status, []).append(name) + return { + "series_by_status": dict(series_by_status), + "metrics_by_status": metrics_by_status, + } + + +def resolve_aliases( + metrics: Mapping[str, Any], base: str +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Series of ``base`` and its Prometheus ``_total`` alias, one per endpoint and label set. + + Cross-name aliases collapse only when they are the same observation (equal + ``stats`` and ``timeslices``); any other overlap is a conflict, whatever + the order. Repeated series under one name are reported as duplicates. Both + are dropped so they are never summed. + """ + groups: dict[tuple, list[tuple[str, Mapping[str, Any]]]] = {} + for name in (base, f"{base}_total"): + entry = metrics.get(name) + series_list = entry.get("series") if isinstance(entry, Mapping) else None + for series in series_list if isinstance(series_list, list) else []: + if isinstance(series, Mapping): + groups.setdefault(series_key(name, series), []).append((name, series)) + chosen: list[dict[str, Any]] = [] + conflicts: list[dict[str, Any]] = [] + for key, items in groups.items(): + names = [name for name, _ in items] + if len(items) == 1: + chosen.append(items[0][1]) + continue + kind = "duplicate_series" if len(set(names)) < len(names) else "alias_conflict" + first = items[0][1] + if kind == "alias_conflict" and all(_same_observation(first, s) for _, s in items[1:]): + chosen.append(first) + continue + conflicts.append( + {"metric": base, "endpoint_url": key[1], "names": sorted(names), "kind": kind} + ) + return chosen, conflicts + + +def _time_range(value: Any) -> tuple[int, int] | None: + if not isinstance(value, Mapping): + return None + start, end = value.get("start_ns"), value.get("end_ns") + if not isinstance(start, int) or not isinstance(end, int) or start >= end: + return None + return start, end + + +def _slice_findings( + slices: list[Mapping[str, Any]], + windows: PhaseWindows | None, + record: dict[str, Any], + histogram: bool, + has_reference: bool, + warmup_total: float | None, +) -> list[tuple[str, str]]: + bounds = [(s.get("start_ns"), s.get("end_ns")) for s in slices] + if not all(isinstance(a, int) and isinstance(b, int) for a, b in bounds): + return [(REJECTED, "slice_bounds_missing")] + findings: list[tuple[str, str]] = [] + if any(end <= start for start, end in bounds): + findings.append((REJECTED, "slice_nonpositive_duration")) + if any(bounds[i][0] < bounds[i - 1][1] for i in range(1, len(bounds))): + findings.append((REJECTED, "slice_overlap_or_backward")) + if any(bounds[i][0] > bounds[i - 1][1] for i in range(1, len(bounds))): + findings.append((REJECTED, "slice_gap")) + if findings or windows is None: + return findings + + start, end = windows.profiling + first, last = bounds[0][0], bounds[-1][1] + step = max(b - a for a, b in bounds) + activity = [_slice_activity(s, histogram) for s in slices] + before = sum(v for (a, b), v in zip(bounds, activity, strict=True) if b <= start) + straddle_start = sum(v for (a, b), v in zip(bounds, activity, strict=True) if a < start < b) + straddle_end = sum(v for (a, b), v in zip(bounds, activity, strict=True) if a < end < b) + after = sum(v for (a, b), v in zip(bounds, activity, strict=True) if a >= end) + record.update( + first_slice_offset_s=(first - start) / NS_PER_S, + last_slice_offset_s=(last - end) / NS_PER_S, + activity_before_profiling=before, + activity_straddling_start=straddle_start, + activity_straddling_end=straddle_end, + activity_after_profiling=after, + ) + # A window that opens before the phase holds PROFILING-tagged samples from + # before it. Increments between the last of those and the first in-phase + # sample land in whichever slice holds that later sample, so slice position + # cannot clear them; only a zero warmup-phase total can. + if first < start: + if before > 0 or (straddle_start > 0 and first <= start - step): + findings.append((PHASE_CONTAMINATED, "activity_before_profiling")) + elif warmup_total != 0 and first <= start - step: + findings.append((PHASE_CONTAMINATED, "window_opens_before_profiling")) + elif straddle_start > 0: + findings.append((BOUNDARY_UNCERTAIN, "activity_straddles_profiling_start")) + elif warmup_total != 0: + findings.append((BOUNDARY_UNCERTAIN, "window_opens_before_profiling")) + if after > 0: + findings.append((PHASE_CONTAMINATED, "activity_after_profiling")) + elif straddle_end > 0: + findings.append((BOUNDARY_UNCERTAIN, "activity_straddles_profiling_end")) + # A late first slice is covered only if a reference sample exists, so that + # stats.total spans the gap and the residual check sees any increment in it. + if first > start and not has_reference: + findings.append((PARTIAL_COVERAGE, "window_starts_after_profiling")) + if last < end: + findings.append((PARTIAL_COVERAGE, "window_ends_before_profiling")) + return findings + + +def _slice_activity(item: Mapping[str, Any], histogram: bool) -> float: + value = to_float(item.get("count" if histogram else "total")) + return value if value is not None and math.isfinite(value) and value > 0 else 0.0 + + +def _slice_sum(slices: list[Mapping[str, Any]], key: str, histogram: bool) -> float | None: + total = 0.0 + for item in slices: + if histogram: + count = to_float(item.get("count")) + value = count if key == "count" else _product(count, to_float(item.get("avg"))) + else: + value = to_float(item.get("total")) + if value is None or not math.isfinite(value) or value < 0: + return None + total += value + return total + + +def _product(count: float | None, avg: float | None) -> float | None: + if count is None or avg is None: + return None + return count * avg + + +def _same_observation(left: Mapping[str, Any], right: Mapping[str, Any]) -> bool: + return left.get("stats") == right.get("stats") and left.get("timeslices") == right.get( + "timeslices" + ) diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index 46cbc17684..c40c9cf84b 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -11,7 +11,15 @@ apply_profile_totals, empty_server_metrics, ) +from .cache_validity import summarize_cache_validity from .common import index_server_metrics +from .phase_validity import ( + PhaseContext, + export_context, + phase_windows, + summarize, + warmup_totals, +) from .request_metrics import extract_per_record_ints @@ -89,11 +97,24 @@ def compute_server_metrics( endpoint_roles: Mapping[str, str] | None = None, request_scope: Mapping[str, Any] | None = None, deployment: Mapping[str, Any] | None = None, + disaggregated: bool | None = None, ) -> tuple[dict[str, Any], dict[str, Any], list[str]]: metrics = index_server_metrics(server_metrics) flat = SERVER_CACHE_FLAT_FIELDS.copy() warnings: list[str] = [] nested = empty_server_metrics(bool(metrics), len(metrics)) + windows = phase_windows(server_metrics) + exported = export_context(server_metrics, windows) + unscraped = exported["endpoints"]["unscraped"] + context = PhaseContext( + windows=windows, + warmup_totals=warmup_totals(server_metrics), + unscraped=None if unscraped is None else tuple(unscraped), + disaggregated=disaggregated, + ) + nested["validity"].update(exported) + nested["validity"]["deployment"] = {"disaggregated": disaggregated} + nested["validity"].update(summarize(metrics, windows, context.warmup_totals)) adapter = detect_backend(metrics, framework) if not metrics: @@ -106,6 +127,7 @@ def compute_server_metrics( nested["tokens"]["prompt_total"] = flat["total_prompt_tokens"] nested["tokens"]["generation_total"] = flat["total_generation_tokens"] nested["tokens"]["requests_completed"] = flat["total_requests_completed"] + nested["cache_validity"] = summarize_cache_validity(nested) return flat, nested, warnings if adapter is None: @@ -117,7 +139,7 @@ def compute_server_metrics( nested["adapter"] = adapter.name if getattr(adapter, "accepts_context", False): warmup = server_metrics.get("warmup_metrics") - context = { + adapter_context = { "framework": framework, "endpoint_roles": endpoint_roles, "profiling_window": _profiling_window(server_metrics), @@ -126,9 +148,10 @@ def compute_server_metrics( **dict(deployment or {}), "warnings": warnings, } - adapter.populate(metrics, flat, nested, context=context) + adapter.populate(metrics, flat, nested, context=adapter_context) else: adapter.populate(metrics, flat, nested) + adapter.apply_validity(metrics, flat, nested, context) if framework.lower() == "dynamo-sglang" and adapter.name == "sglang": nested["kv_cache"]["gpu_total_tokens"] = None @@ -147,4 +170,5 @@ def compute_server_metrics( nested["tokens"]["generation_total"] = flat["total_generation_tokens"] nested["tokens"]["requests_completed"] = flat["total_requests_completed"] + nested["cache_validity"] = summarize_cache_validity(nested) return flat, nested, warnings diff --git a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py index b82c25f1d3..cad34cbbee 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py @@ -491,3 +491,18 @@ def test_only_profiling_errors_from_the_loader_block_pricing( assert accounting["records_warmup_dropped"] == failed_phases.count("warmup") expected_categories = {"Timeout": len(failed_phases)} if failed_phases else {} assert accounting["error_categories"] == expected_categories + + +def test_cache_validity_core_classifies_atom_states() -> None: + complete, _ = _run(_pd()) + partial, _ = _run(_pd(offload=None), deployment={"disagg": True, "kv_offloading": "dram"}) + refused, _ = _run(_pd(prefix_cache_cached_tokens=(0, 700, 600, 0))) + + assert complete["cache_validity"]["status_class"] == "VALID_COMPLETE" + assert complete["cache_validity"]["published"] is True + assert complete["cache_validity"]["basis"] == "tiers" + assert partial["cache_validity"]["status_class"] == "PARTIAL" + assert partial["cache_validity"]["published"] is True + assert partial["cache_validity"]["pricing_eligible"] is False + assert refused["cache_validity"]["status_class"] == "INVALID" + assert refused["cache_validity"]["published"] is False diff --git a/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py b/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py new file mode 100644 index 0000000000..421a194156 --- /dev/null +++ b/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py @@ -0,0 +1,68 @@ +"""The cache validity core is filled for every adapter, so one consumer rule covers every producer.""" + +from __future__ import annotations + +from infx.results.agentic.cache_validity import STATUS_CLASSES +from infx.results.agentic.server_metrics import compute_server_metrics +from infx.tests.results.agentic.test_server_metrics_closeout import _pair +from infx.tests.results.agentic.test_server_metrics_closeout import _run as _run_trt + + +def _counter(value: float) -> dict: + return { + "type": "counter", + "series": [ + { + "endpoint_url": "http://10.0.0.5:8000/metrics", + "labels": {}, + "stats": {"total": value}, + } + ], + } + + +class TestCacheValidityCore: + def test_trt_interval_telemetry_is_published_but_not_exact(self) -> None: + core = _run_trt(_pair())["cache_validity"] + + assert core["status"] == "CONDITIONAL_INTERVAL" + assert core["status_class"] == "VALID_INTERVAL" + assert core["published"] is True + assert core["pricing_eligible"] is False + assert core["basis"] == "combined" + + def test_trt_rejected_rate_is_not_published(self) -> None: + core = _run_trt(_pair(cached=1200.0))["cache_validity"] + + assert core["status_class"] == "INVALID" + assert core["published"] is False + assert core["basis"] is None + assert "cached_exceeds_prompt" in core["reasons"] + + def test_adapter_without_assessment_is_not_evaluated_on_its_tiers(self) -> None: + export = { + "metrics": { + "vllm:prefix_cache_hits": _counter(600.0), + "vllm:prefix_cache_queries": _counter(1000.0), + } + } + _, nested, _ = compute_server_metrics(export, framework="vllm", records=[]) + core = nested["cache_validity"] + + assert core["status_class"] == "NOT_EVALUATED" + assert core["published"] is True + assert core["pricing_eligible"] is False + assert core["basis"] == "tiers" + + def test_missing_server_metrics_is_missing(self) -> None: + _, nested, _ = compute_server_metrics({}, framework="vllm", records=[]) + core = nested["cache_validity"] + + assert core["status_class"] == "MISSING" + assert core["published"] is False + assert core["reasons"] == ["server_metrics_missing"] + + def test_every_class_is_from_the_shared_vocabulary(self) -> None: + for export in (_pair(), _pair(cached=1200.0), {}): + _, nested, _ = compute_server_metrics(export, framework="dynamo-trt", records=[]) + assert nested["cache_validity"]["status_class"] in STATUS_CLASSES diff --git a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py index e420a19f5e..664495f3e7 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py @@ -81,6 +81,7 @@ "tokens", "sources", "cache_validity", + "validity", } SERVER_CACHE_KEYS = { "gpu_cache_hit_rate", @@ -1560,8 +1561,14 @@ def test_processor_normalizes_trtllm_server_metrics(tmp_path: Path): _assert_stable_server_metrics_schema(agg) assert agg["server_metrics"]["tokens"]["prompt_total"] == 1000 assert agg["server_metrics"]["tokens"]["generation_total"] == 200 - assert agg["server_metrics"]["cache"]["gpu_cache_hit_rate"] == pytest.approx(0.5) - assert agg["server_metrics"]["cache"]["overall_cache_hit_rate"] == pytest.approx(0.5) + assert agg["server_metrics"]["cache"]["gpu_cache_hit_rate"] is None + assert agg["server_metrics"]["cache"]["overall_cache_hit_rate"] is None + validity = agg["server_metrics"]["validity"] + assert validity["cache_rate"]["status"] == "UNKNOWN" + assert "phase_window_missing" in validity["cache_rate"]["reasons"] + gauge = validity["diagnostics"]["trtllm_kv_cache_hit_rate"] + assert gauge["all_series_avg"] == pytest.approx(0.5) + assert gauge["by_role"] == pytest.approx({"prefill": 0.4, "decode": 0.6}) assert agg["server_metrics"]["kv_cache"]["gpu_usage_pct"] == pytest.approx(0.8) assert agg["server_metrics"]["kv_cache"]["cpu_usage_pct"] == pytest.approx(0.25) assert agg["server_metrics"]["kv_cache"]["gpu_total_tokens"] == 19_200 diff --git a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py new file mode 100644 index 0000000000..f0bc433310 --- /dev/null +++ b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py @@ -0,0 +1,473 @@ +"""Pre-release closeout of the TRT reuse-rate guard (X1-C1R R1-R4). + +Each case is a synthetic AIPerf 0.12 export: counter names without ``_total``, +1 s timeslices and ``summary.phase_time_ranges``. A standard cache rate may be +published only as interval telemetry (``CONDITIONAL_INTERVAL``); anything the +export cannot explain keeps its raw counts and reasons but publishes no rate. +""" + +from __future__ import annotations + +import copy + +import pytest + +from infx.results.agentic.server_metrics import compute_server_metrics + +NS = 1_000_000_000 +START = 1_000 * NS +END = START + 60 * NS +PRE = ["http://10.0.0.2:7500/metrics", "http://10.0.0.3:7501/metrics"] +DEC = "http://10.0.0.4:7502/metrics" +UNLABELLED = "http://10.0.0.9:7509/metrics" + + +def _slices(total: float, start_ns: int = START, end_ns: int = END) -> list[dict]: + bounds = list(range(start_ns, end_ns, NS)) + per = total / len(bounds) + return [ + {"start_ns": s, "end_ns": min(s + NS, end_ns), "total": per, "rate": per} + for s in bounds + ] + + +def _series( + endpoint: str, + role: str | None, + total: float, + *, + start_ns: int = START, + end_ns: int = END, + labels: dict | None = None, + slices: list[dict] | None = None, +) -> dict: + series_labels = {"model_name": "m", **(labels or {})} + if role is not None: + series_labels["dynamo_component"] = role + return { + "endpoint_url": endpoint, + "labels": series_labels, + "stats": {"total": total}, + "timeslices": slices if slices is not None else _slices(total, start_ns, end_ns), + } + + +def _export(prompt: list[dict], cached: list[dict], *, endpoints: bool = True) -> dict: + summary: dict = { + "phase_time_ranges": { + "profiling": {"start_ns": START, "end_ns": END}, + "warmup": {"start_ns": START - 120 * NS, "end_ns": START}, + } + } + if endpoints: + urls = sorted({s["endpoint_url"] for s in prompt + cached}) + summary["endpoints_configured"] = urls + summary["endpoints_successful"] = list(urls) + return { + "summary": summary, + "metrics": { + "trtllm_prompt_tokens": {"type": "counter", "series": prompt}, + "trtllm_prompt_cached_tokens": {"type": "counter", "series": cached}, + }, + } + + +def _with_warmup(export: dict, total: float) -> dict: + """Add the same series to ``warmup_metrics`` with the given warmup-phase total.""" + export["warmup_metrics"] = { + name: { + "type": "counter", + "series": [ + _series( + s["endpoint_url"], + s["labels"].get("dynamo_component"), + total, + start_ns=START - 120 * NS, + end_ns=START, + labels={k: v for k, v in s["labels"].items() if k != "dynamo_component"}, + ) + for s in entry["series"] + ], + } + for name, entry in export["metrics"].items() + } + return export + + +def _run(export: dict, **kwargs) -> dict: + _, nested, _ = compute_server_metrics( + export, framework="dynamo-trt", records=[], **kwargs + ) + return nested + + +def _rate(nested: dict) -> dict: + return nested["validity"]["cache_rate"] + + +def _pair(prompt: float = 1000.0, cached: float = 900.0, **kwargs) -> dict: + return _export( + [_series(PRE[0], "prefill", prompt, **kwargs)], + [_series(PRE[0], "prefill", cached, **kwargs)], + ) + + +class TestR1CachedWithinPrompt: + def test_cached_above_prompt_is_rejected_and_not_published(self) -> None: + nested = _run(_pair(cached=1200.0)) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert nested["tokens"]["prompt_by_source"]["computed"] is None + assert _rate(nested)["status"] == "REJECTED" + assert "cached_exceeds_prompt" in _rate(nested)["reasons"] + assert _rate(nested)["raw_rate"] == pytest.approx(1.2) + + def test_one_domain_above_prompt_is_not_offset_by_the_sum(self) -> None: + export = _export( + [_series(PRE[0], "prefill", 1000.0), _series(PRE[1], "prefill", 3000.0)], + [_series(PRE[0], "prefill", 1200.0), _series(PRE[1], "prefill", 1000.0)], + ) + + nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert "cached_exceeds_prompt" in _rate(nested)["reasons"] + + def test_cached_in_a_domain_without_prompt_is_rejected(self) -> None: + export = _export( + [_series(PRE[0], "prefill", 1000.0), _series(PRE[1], "prefill", 0.0)], + [_series(PRE[0], "prefill", 900.0), _series(PRE[1], "prefill", 5.0)], + ) + + nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert "cached_exceeds_prompt" in _rate(nested)["reasons"] + + @pytest.mark.parametrize(("cached", "expected"), [(0.0, 0.0), (1000.0, 1.0)]) + def test_zero_and_full_reuse_stay_publishable(self, cached, expected) -> None: + nested = _run(_pair(cached=cached)) + + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(expected) + assert _rate(nested)["status"] == "CONDITIONAL_INTERVAL" + assert nested["tokens"]["prompt_by_source"]["computed"] == pytest.approx( + 1000.0 - cached + ) + + def test_idle_prefill_domain_does_not_block(self) -> None: + export = _export( + [_series(PRE[0], "prefill", 1000.0), _series(PRE[1], "prefill", 0.0)], + [_series(PRE[0], "prefill", 900.0), _series(PRE[1], "prefill", 0.0)], + ) + + nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) + + +class TestR2RoleScopeAndPairing: + def test_decode_only_is_not_treated_as_combined(self) -> None: + export = _export( + [_series(DEC, "backend", 1000.0)], [_series(DEC, "backend", 1000.0)] + ) + + nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _rate(nested)["role_scope"] != "combined" + assert "decode_only_no_prefill_scope" in _rate(nested)["reasons"] + + def test_unknown_role_beside_prefill_blocks_instead_of_being_excluded(self) -> None: + export = _export( + [_series(PRE[0], "prefill", 1000.0), _series(UNLABELLED, None, 1000.0)], + [_series(PRE[0], "prefill", 900.0), _series(UNLABELLED, None, 0.0)], + ) + + nested = _run(export) + + rate = _rate(nested) + assert nested["cache"]["overall_cache_hit_rate"] is None + assert "role_unknown" in rate["reasons"] + assert {s["endpoint_url"] for s in rate["unresolved"]} == {UNLABELLED} + assert UNLABELLED not in {s["endpoint_url"] for s in rate["excluded"]} + + def test_unlabelled_series_need_an_authoritative_deployment_mode(self) -> None: + export = _export( + [_series(PRE[0], None, 1000.0)], [_series(PRE[0], None, 900.0)] + ) + + unknown = _run(copy.deepcopy(export)) + unified = _run(copy.deepcopy(export), disaggregated=False) + + assert unknown["cache"]["overall_cache_hit_rate"] is None + assert "deployment_mode_unknown" in _rate(unknown)["reasons"] + assert unified["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) + assert _rate(unified)["role_scope"] == "combined" + + def test_disaggregated_deployment_without_prefill_series_publishes_nothing( + self, + ) -> None: + export = _export( + [_series(PRE[0], None, 1000.0)], [_series(PRE[0], None, 900.0)] + ) + + nested = _run(export, disaggregated=True) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert "no_prefill_scope" in _rate(nested)["reasons"] + + def test_role_labels_contradicting_the_deployment_block(self) -> None: + nested = _run(_pair(), disaggregated=False) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert "role_conflicts_with_deployment" in _rate(nested)["reasons"] + + def test_same_endpoint_with_different_rank_is_not_a_pair(self) -> None: + export = _export( + [_series(PRE[0], "prefill", 1000.0, labels={"rank": "0"})], + [_series(PRE[0], "prefill", 900.0, labels={"rank": "1"})], + ) + + nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert "missing_pair" in _rate(nested)["reasons"] + + def test_instance_order_does_not_change_the_result(self) -> None: + prompt = [ + _series(PRE[0], "prefill", 1000.0), + _series(PRE[1], "prefill", 3000.0), + _series(DEC, "backend", 4000.0), + ] + cached = [ + _series(PRE[0], "prefill", 900.0), + _series(PRE[1], "prefill", 2100.0), + _series(DEC, "backend", 4000.0), + ] + + forward = _run(_export(prompt, cached)) + backward = _run(_export(prompt[::-1], cached[::-1])) + + assert forward["cache"] == backward["cache"] + assert forward["cache"]["overall_cache_hit_rate"] == pytest.approx(0.75) + assert _rate(forward)["status"] == _rate(backward)["status"] + + def test_missing_scrape_metadata_is_not_full_coverage(self) -> None: + export = _export( + [_series(PRE[0], "prefill", 1000.0)], + [_series(PRE[0], "prefill", 900.0)], + endpoints=False, + ) + + nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert "scrape_coverage_unknown" in _rate(nested)["reasons"] + + +class TestR3AliasObservations: + def test_equal_total_aliases_with_different_windows_conflict_in_either_order( + self, + ) -> None: + clean = _series(PRE[0], "prefill", 1000.0) + early = _series(PRE[0], "prefill", 1000.0, start_ns=START - 120 * NS) + outcomes = [] + for first, second in ((clean, early), (early, clean)): + export = _export([copy.deepcopy(first)], [_series(PRE[0], "prefill", 900.0)]) + export["metrics"]["trtllm_prompt_tokens_total"] = { + "type": "counter", + "series": [copy.deepcopy(second)], + } + nested = _run(export) + outcomes.append( + ( + nested["cache"]["overall_cache_hit_rate"], + _rate(nested)["status"], + "alias_conflict" in _rate(nested)["reasons"], + ) + ) + + assert outcomes[0] == outcomes[1] == (None, "REJECTED", True) + + def test_identical_alias_observations_count_once(self) -> None: + export = _pair() + export["metrics"]["trtllm_prompt_tokens_total"] = copy.deepcopy( + export["metrics"]["trtllm_prompt_tokens"] + ) + + nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) + assert _rate(nested)["prompt_tokens"] == pytest.approx(1000.0) + + def test_duplicate_series_under_one_name_is_not_an_alias(self) -> None: + export = _export( + [_series(PRE[0], "prefill", 1000.0), _series(PRE[0], "prefill", 1000.0)], + [_series(PRE[0], "prefill", 900.0)], + ) + + nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert "duplicate_series" in _rate(nested)["reasons"] + assert "alias_conflict" not in _rate(nested)["reasons"] + + +class TestR4TimeWindow: + def test_activity_after_profiling_end_is_not_valid(self) -> None: + nested = _run(_pair(end_ns=END + 100 * NS)) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _rate(nested)["status"] == "PHASE_CONTAMINATED" + assert "activity_after_profiling" in _rate(nested)["reasons"] + + def test_nonpositive_duration_slice_is_rejected(self) -> None: + def backward(total: float) -> list[dict]: + return [ + {"start_ns": START, "end_ns": START - NS, "total": total / 2}, + {"start_ns": START - NS, "end_ns": END, "total": total / 2}, + ] + + export = _export( + [_series(PRE[0], "prefill", 1000.0, slices=backward(1000.0))], + [_series(PRE[0], "prefill", 900.0, slices=backward(900.0))], + ) + + nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _rate(nested)["status"] == "REJECTED" + assert "slice_nonpositive_duration" in _rate(nested)["reasons"] + + def test_activity_one_second_before_profiling_is_not_valid(self) -> None: + nested = _run(_pair(start_ns=START - NS)) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _rate(nested)["status"] == "PHASE_CONTAMINATED" + assert "activity_before_profiling" in _rate(nested)["reasons"] + + def test_activity_in_a_slice_straddling_the_start_is_uncertain(self) -> None: + nested = _run(_pair(start_ns=START - NS // 2)) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _rate(nested)["status"] == "BOUNDARY_UNCERTAIN" + assert "activity_straddles_profiling_start" in _rate(nested)["reasons"] + + def test_idle_slices_outside_the_window_need_zero_warmup_evidence(self) -> None: + def padded(total: float) -> list[dict]: + inside = _slices(total) + before = [ + {"start_ns": START - k * NS, "end_ns": START - (k - 1) * NS, "total": 0.0} + for k in (2, 1) + ] + after = [ + {"start_ns": END + k * NS, "end_ns": END + (k + 1) * NS, "total": 0.0} + for k in (0, 1) + ] + return before + inside + after + + export = _export( + [_series(PRE[0], "prefill", 1000.0, slices=padded(1000.0))], + [_series(PRE[0], "prefill", 900.0, slices=padded(900.0))], + ) + with_zero_warmup = _with_warmup(copy.deepcopy(export), 0.0) + + without = _run(export) + idle_warmup = _run(with_zero_warmup) + + assert without["cache"]["overall_cache_hit_rate"] is None + assert "window_opens_before_profiling" in _rate(without)["reasons"] + assert idle_warmup["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) + assert _rate(idle_warmup)["status"] == "CONDITIONAL_INTERVAL" + + def test_warmup_jump_landing_after_the_start_is_not_hidden(self) -> None: + def early_with_jump(profiling_total: float, warmup_total: float) -> list[dict]: + flat = [ + {"start_ns": START - k * NS, "end_ns": START - (k - 1) * NS, "total": 0.0} + for k in range(120, 0, -1) + ] + inside = _slices(profiling_total) + inside[1]["total"] += warmup_total + return flat + inside + + export = _export( + [_series(PRE[0], "prefill", 1500.0, slices=early_with_jump(1000.0, 500.0))], + [_series(PRE[0], "prefill", 1350.0, slices=early_with_jump(900.0, 450.0))], + ) + + nested = _run(_with_warmup(export, 500.0)) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _rate(nested)["status"] == "PHASE_CONTAMINATED" + assert "window_opens_before_profiling" in _rate(nested)["reasons"] + + def test_window_opening_within_one_slice_needs_zero_warmup_evidence(self) -> None: + def early_by_half(total: float) -> list[dict]: + first = {"start_ns": START - NS // 2, "end_ns": START + NS // 2, "total": 0.0} + rest = _slices(total, START + NS // 2, END) + return [first] + rest + + export = _export( + [_series(PRE[0], "prefill", 1000.0, slices=early_by_half(1000.0))], + [_series(PRE[0], "prefill", 900.0, slices=early_by_half(900.0))], + ) + + busy_warmup = _run(_with_warmup(copy.deepcopy(export), 50.0)) + idle_warmup = _run(_with_warmup(copy.deepcopy(export), 0.0)) + + assert busy_warmup["cache"]["overall_cache_hit_rate"] is None + assert _rate(busy_warmup)["status"] == "BOUNDARY_UNCERTAIN" + assert "window_opens_before_profiling" in _rate(busy_warmup)["reasons"] + assert idle_warmup["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) + + def test_late_start_needs_warmup_reference_evidence(self) -> None: + late = _pair(start_ns=START + NS // 5) + with_reference = _with_warmup(copy.deepcopy(late), 50.0) + + without = _run(late) + with_ref = _run(with_reference) + + assert without["cache"]["overall_cache_hit_rate"] is None + assert "window_starts_after_profiling" in _rate(without)["reasons"] + assert with_ref["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) + + def test_numerator_and_denominator_need_the_same_window(self) -> None: + export = _export( + [_series(PRE[0], "prefill", 1000.0)], + [_series(PRE[0], "prefill", 900.0, end_ns=END - 30 * NS)], + ) + + nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert {"window_ends_before_profiling", "window_support_mismatch"} <= set( + _rate(nested)["reasons"] + ) + + +class TestTelemetryVersusEligibility: + @pytest.mark.parametrize( + ("export", "status", "published"), + [ + (_pair(), "CONDITIONAL_INTERVAL", True), + (_pair(cached=1200.0), "REJECTED", False), + ], + ) + def test_publication_follows_status_and_exact_stays_false( + self, export, status, published + ) -> None: + nested = _run(copy.deepcopy(export)) + + rate = _rate(nested) + assert (rate["status"], rate["published"]) == (status, published) + assert rate["eligible_exact_actual"] is False + assert rate["cached_tokens"] is not None and rate["prompt_tokens"] is not None + + def test_unknown_window_is_reported_separately(self) -> None: + export = _pair() + del export["summary"]["phase_time_ranges"] + + rate = _rate(_run(export)) + + assert (rate["status"], rate["published"]) == ("UNKNOWN", False) + assert "phase_window_missing" in rate["reasons"] diff --git a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_phase_validity.py b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_phase_validity.py new file mode 100644 index 0000000000..8392f558e4 --- /dev/null +++ b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_phase_validity.py @@ -0,0 +1,411 @@ +"""Phase validity of AIPerf server-metric counters and the TensorRT-LLM reuse rate. + +Fixtures follow the AIPerf 0.12 ``server_metrics_export.json`` shape: counter +names without the Prometheus ``_total`` suffix, 1 s timeslices per series and +``summary.phase_time_ranges``. The P/D topology mirrors the Dynamo TRT runs +(``prefill`` and ``backend`` components); decode workers report every +transferred prompt token as cached. +""" + +from __future__ import annotations + +import copy +import math + +import pytest + +from infx.results.agentic.server_metrics import compute_server_metrics + +NS = 1_000_000_000 +P0 = 1_000 * NS +PROFILING = (P0, P0 + 60 * NS) +WARMUP = (P0 - 120 * NS, P0 + int(0.3 * NS)) +FRONTEND = "http://10.0.0.1:8000/metrics" +PREFILL = ["http://10.0.0.2:7500/metrics", "http://10.0.0.3:7501/metrics"] +DECODE = ["http://10.0.0.4:7502/metrics", "http://10.0.0.5:7503/metrics"] +PROMPT = {PREFILL[0]: 1000.0, PREFILL[1]: 2000.0, DECODE[0]: 1000.0, DECODE[1]: 2000.0} +CACHED = {PREFILL[0]: 900.0, PREFILL[1]: 1800.0, DECODE[0]: 1000.0, DECODE[1]: 2000.0} + + +def _role(endpoint: str) -> str: + return "prefill" if endpoint in PREFILL else "backend" + + +def _slices(start_s: float, end_s: float, total: float) -> list[dict]: + count = int(round(end_s - start_s)) + first = P0 + round(start_s * NS) + per = total / count + return [ + { + "start_ns": first + i * NS, + "end_ns": first + (i + 1) * NS, + "total": per, + "rate": per, + } + for i in range(count) + ] + + +def _counter( + endpoint: str, + total: float, + *, + start_s: float = 0.0, + end_s: float = 60.0, + slice_sum: float | None = None, + timeslices: bool = True, +) -> dict: + series = { + "endpoint_url": endpoint, + "labels": {"dynamo_component": _role(endpoint), "model_name": "m"}, + "stats": {"total": total}, + } + if timeslices: + series["timeslices"] = _slices( + start_s, end_s, total if slice_sum is None else slice_sum + ) + return series + + +def _histogram(endpoint: str, total: float, count: int, *, start_s: float = 0.0): + slices = [] + items = _slices(start_s, 60.0, total) + per_count = count / len(items) + for item in items: + slices.append( + { + "start_ns": item["start_ns"], + "end_ns": item["end_ns"], + "count": per_count, + "avg": item["total"] / per_count, + } + ) + return { + "endpoint_url": endpoint, + "labels": {"dynamo_namespace": "dynamo"}, + "stats": {"sum": total, "count": count}, + "timeslices": slices, + } + + +def _export( + *, + names: tuple[str, ...] = ("",), + prompt: dict[str, float] | None = None, + cached: dict[str, float] | None = None, + gauge: bool = False, + phase_time_ranges: bool = True, +) -> dict: + prompt = PROMPT if prompt is None else prompt + cached = CACHED if cached is None else cached + metrics: dict = { + "dynamo_frontend_input_sequence_tokens": { + "type": "histogram", + "series": [_histogram(FRONTEND, 3000.0, 60)], + } + } + for suffix in names: + metrics[f"trtllm_prompt_tokens{suffix}"] = { + "type": "counter", + "series": [_counter(e, v) for e, v in prompt.items()], + } + metrics[f"trtllm_prompt_cached_tokens{suffix}"] = { + "type": "counter", + "series": [_counter(e, v) for e, v in cached.items()], + } + if gauge: + metrics["trtllm_kv_cache_hit_rate"] = { + "type": "gauge", + "series": [ + { + "endpoint_url": e, + "labels": {"dynamo_component": _role(e)}, + "stats": {"avg": 0.93 if e in PREFILL else 0.0}, + } + for e in PREFILL + DECODE + ], + } + endpoints = [FRONTEND, *PREFILL, *DECODE] + summary: dict = { + "endpoints_configured": endpoints, + "endpoints_successful": list(endpoints), + } + if phase_time_ranges: + summary["phase_time_ranges"] = { + "profiling": {"start_ns": PROFILING[0], "end_ns": PROFILING[1]}, + "warmup": {"start_ns": WARMUP[0], "end_ns": WARMUP[1]}, + } + return {"summary": summary, "metrics": metrics} + + +def _run(export: dict, framework: str = "dynamo-trt") -> tuple[dict, dict]: + flat, nested, _ = compute_server_metrics(export, framework=framework, records=[]) + return flat, nested + + +def _cache_rate(nested: dict) -> dict: + return nested["validity"]["cache_rate"] + + +class TestTrtCounterNames: + def test_stripped_counter_names_feed_the_prefill_reuse_rate(self) -> None: + _, nested = _run(_export()) + + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) + assert _cache_rate(nested)["status"] == "CONDITIONAL_INTERVAL" + assert _cache_rate(nested)["role_scope"] == "prefill" + + def test_native_total_names_are_still_accepted(self) -> None: + _, nested = _run(_export(names=("_total",))) + + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) + + def test_alias_pairs_with_equal_values_count_once(self) -> None: + _, nested = _run(_export(names=("", "_total"))) + + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) + assert _cache_rate(nested)["cached_tokens"] == pytest.approx(2700.0) + assert _cache_rate(nested)["prompt_tokens"] == pytest.approx(3000.0) + + def test_alias_conflict_rejects_the_rate(self) -> None: + export = _export(names=("", "_total")) + native = export["metrics"]["trtllm_prompt_cached_tokens_total"]["series"][0] + native["stats"]["total"] = 950.0 + + _, nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _cache_rate(nested)["status"] == "REJECTED" + assert "alias_conflict" in _cache_rate(nested)["reasons"] + + +class TestTrtRoleAndTier: + def test_decode_transfer_is_excluded_and_reported(self) -> None: + _, nested = _run(_export()) + + excluded = { + (item["metric"], item["endpoint_url"], item["reason"]) + for item in _cache_rate(nested)["excluded"] + } + assert excluded == { + (metric, endpoint, "decode_role_excluded") + for metric in ("trtllm_prompt_tokens", "trtllm_prompt_cached_tokens") + for endpoint in DECODE + } + + def test_role_mixed_gauge_is_diagnostic_only(self) -> None: + _, nested = _run(_export(gauge=True)) + + assert nested["cache"]["gpu_cache_hit_rate"] is None + diagnostic = nested["validity"]["diagnostics"]["trtllm_kv_cache_hit_rate"] + assert diagnostic["status"] == "GAUGE_DERIVED" + assert diagnostic["all_series_avg"] == pytest.approx(0.465) + assert diagnostic["by_role"] == pytest.approx({"prefill": 0.93, "decode": 0.0}) + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) + + def test_combined_cached_is_not_presented_as_gpu_tier(self) -> None: + _, nested = _run(_export()) + + assert nested["cache"]["cached_tokens_by_source"] == {"combined": 2700.0} + assert _cache_rate(nested)["tier"] == "combined" + prompt_by_source = nested["tokens"]["prompt_by_source"] + assert prompt_by_source["gpu_cache_hit"] is None + assert prompt_by_source["computed"] == pytest.approx(300.0) + assert prompt_by_source["raw"] == {"combined": 2700.0} + + def test_rate_is_never_marked_exact_without_request_binding(self) -> None: + _, nested = _run(_export()) + + assert _cache_rate(nested)["eligible_exact_actual"] is False + assert {"request_scope_not_bound", "expected_instances_not_verified"} <= set( + _cache_rate(nested)["reasons"] + ) + + +class TestTrtPhaseValidity: + def test_contaminated_cached_series_blocks_rate_and_keeps_telemetry(self) -> None: + export = _export() + export["metrics"]["trtllm_prompt_cached_tokens"]["series"][0] = _counter( + PREFILL[0], 4900.0, start_s=-120.0 + ) + + _, nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert nested["cache"]["cached_tokens_by_source"] == {} + rate = _cache_rate(nested) + assert rate["status"] == "REJECTED" + assert {"activity_before_profiling", "cached_exceeds_prompt"} <= set(rate["reasons"]) + assert rate["cached_tokens"] == pytest.approx(6700.0) + assert rate["raw_rate"] == pytest.approx(6700.0 / 3000.0) + contaminated = [s for s in rate["series"] if s["status"] == "PHASE_CONTAMINATED"] + assert [(s["endpoint_url"], s["reasons"]) for s in contaminated] == [ + (PREFILL[0], ["activity_before_profiling"]) + ] + assert contaminated[0]["first_slice_offset_s"] == pytest.approx(-120.0) + + def test_total_above_slice_sum_is_a_boundary_residual(self) -> None: + export = _export() + export["metrics"]["trtllm_prompt_tokens"]["series"][0] = _counter( + PREFILL[0], 1098.0, slice_sum=1000.0 + ) + + _, nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + rate = _cache_rate(nested) + assert rate["status"] == "BOUNDARY_RESIDUAL" + residual = [s for s in rate["series"] if s["status"] == "BOUNDARY_RESIDUAL"] + assert residual[0]["residual"] == pytest.approx(98.0) + assert residual[0]["reasons"] == ["total_exceeds_slice_sum"] + + def test_slice_sum_above_total_is_a_suspected_reset(self) -> None: + export = _export() + export["metrics"]["trtllm_prompt_cached_tokens"]["series"][1] = _counter( + PREFILL[1], 0.0, slice_sum=1800.0 + ) + + _, nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _cache_rate(nested)["status"] == "REJECTED" + assert "slice_sum_exceeds_total" in _cache_rate(nested)["reasons"] + + @pytest.mark.parametrize( + ("total", "reason"), + [(-5.0, "negative_total"), (math.nan, "nonfinite_total"), (math.inf, "nonfinite_total")], + ) + def test_negative_and_nonfinite_counters_are_rejected(self, total, reason) -> None: + export = _export() + series = export["metrics"]["trtllm_prompt_cached_tokens"]["series"][0] + series["stats"]["total"] = total + + _, nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _cache_rate(nested)["status"] == "REJECTED" + assert reason in _cache_rate(nested)["reasons"] + + def test_missing_counter_total_is_unknown_not_rejected(self) -> None: + export = _export() + del export["metrics"]["trtllm_prompt_tokens"]["series"][1]["stats"]["total"] + + _, nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _cache_rate(nested)["status"] == "UNKNOWN" + assert "missing_total" in _cache_rate(nested)["reasons"] + + def test_missing_phase_window_is_unknown(self) -> None: + _, nested = _run(_export(phase_time_ranges=False)) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _cache_rate(nested)["status"] == "UNKNOWN" + assert "phase_window_missing" in _cache_rate(nested)["reasons"] + assert nested["validity"]["phase_windows"] is None + + def test_series_without_timeslices_is_unknown(self) -> None: + export = _export() + export["metrics"]["trtllm_prompt_tokens"]["series"][1] = _counter( + PREFILL[1], 2000.0, timeslices=False + ) + + _, nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _cache_rate(nested)["status"] == "UNKNOWN" + assert "no_timeslices" in _cache_rate(nested)["reasons"] + + def test_late_created_series_is_partial_coverage(self) -> None: + export = _export() + export["metrics"]["trtllm_prompt_tokens"]["series"][1] = _counter( + PREFILL[1], 2000.0, start_s=40.0 + ) + + _, nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _cache_rate(nested)["status"] == "PARTIAL_COVERAGE" + assert "window_starts_after_profiling" in _cache_rate(nested)["reasons"] + + def test_prefill_endpoint_missing_its_cached_series_is_partial_coverage(self) -> None: + cached = {k: v for k, v in CACHED.items() if k != PREFILL[1]} + + _, nested = _run(_export(cached=cached)) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _cache_rate(nested)["status"] == "PARTIAL_COVERAGE" + assert "missing_pair" in _cache_rate(nested)["reasons"] + + def test_unscraped_configured_endpoint_is_partial_coverage(self) -> None: + prompt = {k: v for k, v in PROMPT.items() if k != PREFILL[1]} + cached = {k: v for k, v in CACHED.items() if k != PREFILL[1]} + export = _export(prompt=prompt, cached=cached) + export["summary"]["endpoints_successful"].remove(PREFILL[1]) + + _, nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] is None + assert _cache_rate(nested)["status"] == "PARTIAL_COVERAGE" + assert "endpoint_not_scraped" in _cache_rate(nested)["reasons"] + + +class TestSharedPhaseSummary: + def test_histograms_pass_through_the_same_phase_check(self) -> None: + export = _export() + export["metrics"]["dynamo_frontend_cached_tokens"] = { + "type": "histogram", + "series": [_histogram(FRONTEND, 2700.0, 60, start_s=-119.9)], + } + gapped = _histogram(FRONTEND, 3000.0, 60) + del gapped["timeslices"][30] + export["metrics"]["dynamo_frontend_output_tokens"] = { + "type": "histogram", + "series": [gapped], + } + + _, nested = _run(export) + + by_status = nested["validity"]["metrics_by_status"] + assert "dynamo_frontend_cached_tokens" in by_status["PHASE_CONTAMINATED"] + assert "dynamo_frontend_output_tokens" in by_status["REJECTED"] + assert "dynamo_frontend_input_sequence_tokens" not in { + name for names in by_status.values() for name in names + } + + def test_empty_histogram_is_unknown_not_rejected(self) -> None: + export = _export() + export["metrics"]["dynamo_component_router_shared_cache_hit_rate"] = { + "type": "histogram", + "series": [{"endpoint_url": FRONTEND, "labels": {}, "stats": {"count": 0}}], + } + + _, nested = _run(export) + + by_status = nested["validity"]["metrics_by_status"] + assert by_status == {"UNKNOWN": ["dynamo_component_router_shared_cache_hit_rate"]} + + def test_other_backends_keep_published_values_and_are_not_evaluated(self) -> None: + def vllm_export(with_phase_data: bool) -> dict: + hits = _counter(PREFILL[0], 900.0) + queries = _counter(PREFILL[0], 1000.0) + if not with_phase_data: + hits.pop("timeslices") + queries.pop("timeslices") + export = { + "metrics": { + "vllm:prefix_cache_hits": {"type": "counter", "series": [hits]}, + "vllm:prefix_cache_queries": {"type": "counter", "series": [queries]}, + } + } + if with_phase_data: + export["summary"] = copy.deepcopy(_export()["summary"]) + return export + + _, with_phase = _run(vllm_export(True), framework="vllm") + _, without_phase = _run(vllm_export(False), framework="vllm") + + assert with_phase["cache"] == without_phase["cache"] + assert with_phase["cache"]["gpu_cache_hit_rate"] == pytest.approx(0.9) + assert with_phase["validity"]["cache_rate"]["status"] == "NOT_EVALUATED" + assert with_phase["validity"]["cache_rate"]["eligible_exact_actual"] is False From 52c34f6d0a60aef1f45e520ae58a71a421084f98 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Thu, 22 Oct 2026 16:55:58 +0000 Subject: [PATCH 05/13] test(results): remove duplicate validity cases and history labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 移除重复覆盖和历史代号，保留更强的窗口及别名测试。 --- .../agentic/test_cache_validity_core.py | 6 ---- .../agentic/test_server_metrics_closeout.py | 36 +++---------------- 2 files changed, 5 insertions(+), 37 deletions(-) diff --git a/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py b/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py index 50875755bb..d71302a608 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py @@ -2,7 +2,6 @@ from __future__ import annotations -from infx.results.agentic.cache_validity import STATUS_CLASSES from infx.results.agentic.server_metrics import compute_server_metrics from infx.tests.results.agentic.test_atom_cache_validity import _export as _atom_export from infx.tests.results.agentic.test_atom_cache_validity import _run as _run_atom @@ -83,8 +82,3 @@ def test_missing_server_metrics_is_missing(self) -> None: assert core["status_class"] == "MISSING" assert core["published"] is False assert core["reasons"] == ["server_metrics_missing"] - - def test_every_class_is_from_the_shared_vocabulary(self) -> None: - for export in (_pair(), _pair(cached=1200.0), {}): - _, nested, _ = compute_server_metrics(export, framework="dynamo-trt", records=[]) - assert nested["cache_validity"]["status_class"] in STATUS_CLASSES diff --git a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py index f0bc433310..5601c05dec 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py @@ -1,10 +1,4 @@ -"""Pre-release closeout of the TRT reuse-rate guard (X1-C1R R1-R4). - -Each case is a synthetic AIPerf 0.12 export: counter names without ``_total``, -1 s timeslices and ``summary.phase_time_ranges``. A standard cache rate may be -published only as interval telemetry (``CONDITIONAL_INTERVAL``); anything the -export cannot explain keeps its raw counts and reasons but publishes no rate. -""" +"""TRT reuse rates require paired counters and profiling-window evidence.""" from __future__ import annotations @@ -112,7 +106,7 @@ def _pair(prompt: float = 1000.0, cached: float = 900.0, **kwargs) -> dict: ) -class TestR1CachedWithinPrompt: +class TestPairedCounterBounds: def test_cached_above_prompt_is_rejected_and_not_published(self) -> None: nested = _run(_pair(cached=1200.0)) @@ -165,7 +159,7 @@ def test_idle_prefill_domain_does_not_block(self) -> None: assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) -class TestR2RoleScopeAndPairing: +class TestDeploymentRolesAndCounterPairs: def test_decode_only_is_not_treated_as_combined(self) -> None: export = _export( [_series(DEC, "backend", 1000.0)], [_series(DEC, "backend", 1000.0)] @@ -265,7 +259,7 @@ def test_missing_scrape_metadata_is_not_full_coverage(self) -> None: assert "scrape_coverage_unknown" in _rate(nested)["reasons"] -class TestR3AliasObservations: +class TestCounterAliases: def test_equal_total_aliases_with_different_windows_conflict_in_either_order( self, ) -> None: @@ -289,17 +283,6 @@ def test_equal_total_aliases_with_different_windows_conflict_in_either_order( assert outcomes[0] == outcomes[1] == (None, "REJECTED", True) - def test_identical_alias_observations_count_once(self) -> None: - export = _pair() - export["metrics"]["trtllm_prompt_tokens_total"] = copy.deepcopy( - export["metrics"]["trtllm_prompt_tokens"] - ) - - nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) - assert _rate(nested)["prompt_tokens"] == pytest.approx(1000.0) - def test_duplicate_series_under_one_name_is_not_an_alias(self) -> None: export = _export( [_series(PRE[0], "prefill", 1000.0), _series(PRE[0], "prefill", 1000.0)], @@ -313,7 +296,7 @@ def test_duplicate_series_under_one_name_is_not_an_alias(self) -> None: assert "alias_conflict" not in _rate(nested)["reasons"] -class TestR4TimeWindow: +class TestProfilingWindowEvidence: def test_activity_after_profiling_end_is_not_valid(self) -> None: nested = _run(_pair(end_ns=END + 100 * NS)) @@ -462,12 +445,3 @@ def test_publication_follows_status_and_exact_stays_false( assert (rate["status"], rate["published"]) == (status, published) assert rate["eligible_exact_actual"] is False assert rate["cached_tokens"] is not None and rate["prompt_tokens"] is not None - - def test_unknown_window_is_reported_separately(self) -> None: - export = _pair() - del export["summary"]["phase_time_ranges"] - - rate = _rate(_run(export)) - - assert (rate["status"], rate["published"]) == ("UNKNOWN", False) - assert "phase_window_missing" in rate["reasons"] From 51fe82a0723ab3be495566b11b27ee52885c9c4d Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Thu, 22 Oct 2026 20:32:39 +0000 Subject: [PATCH 06/13] fix(agentx): align prompt totals and validate TRT source telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 统一 P/D 的成功 profiling 输入总量，修复 TRT 聚合角色判断及 source 命中率校验，保留阶段诊断和条件区间合同。 --- .../infx/results/agentic/backends/base.py | 2 +- .../infx/results/agentic/backends/trtllm.py | 109 +++++++++++------- .../infx/results/agentic/cache_validity.py | 8 -- .../infx/results/agentic/phase_validity.py | 11 -- .../infx/results/agentic/server_metrics.py | 44 ++++++- .../agentic/test_cache_validity_core.py | 47 +++++++- .../agentic/test_server_metrics_closeout.py | 39 ++++++- .../test_server_metrics_phase_validity.py | 9 -- 8 files changed, 194 insertions(+), 75 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/backends/base.py b/inferencex-e2e/infx/results/agentic/backends/base.py index f453d59ef4..192a53d7d6 100644 --- a/inferencex-e2e/infx/results/agentic/backends/base.py +++ b/inferencex-e2e/infx/results/agentic/backends/base.py @@ -151,6 +151,6 @@ def _record_token_sum(records: list[dict[str, Any]], metric_name: str) -> int | try: total += int(value) found = True - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError): continue return total if found else None diff --git a/inferencex-e2e/infx/results/agentic/backends/trtllm.py b/inferencex-e2e/infx/results/agentic/backends/trtllm.py index ec85c879c2..7c66924f5c 100644 --- a/inferencex-e2e/infx/results/agentic/backends/trtllm.py +++ b/inferencex-e2e/infx/results/agentic/backends/trtllm.py @@ -35,8 +35,7 @@ PROMPT_TOKENS = "trtllm_prompt_tokens" CACHED_TOKENS = "trtllm_prompt_cached_tokens" GENERATION_TOKENS = "trtllm_generation_tokens" -# Server counters are interval totals: the producer cannot bind them to the -# billed request set or check them against the expected worker count. +# Interval counters do not identify the billed cohort or prove expected worker coverage. UNBOUND_REASONS = ("request_scope_not_bound", "expected_instances_not_verified") KNOWN_ROLES = ("prefill", "decode", "combined") @@ -56,7 +55,9 @@ def populate( flat: dict[str, Any], nested: dict[str, Any], ) -> None: - prompt_total = _first_total(metrics, "dynamo_frontend_input_sequence_tokens", PROMPT_TOKENS) + prompt_total = sum_stat( + metrics, "dynamo_frontend_input_sequence_tokens", preferred_keys=("total", "sum") + ) generation_total = _first_total(metrics, "dynamo_frontend_output_tokens", GENERATION_TOKENS) diagnostic = _hit_rate_gauge_diagnostic(metrics) if diagnostic is not None: @@ -128,9 +129,12 @@ def apply_validity( ) -> None: reuse = _reuse_rate(metrics, context) nested["validity"]["cache_rate"] = reuse + nested["sources"] = _trtllm_sources(metrics, reuse=reuse) if not reuse["published"]: return cached, prompt = reuse["cached_tokens"], reuse["prompt_tokens"] + if flat["total_prompt_tokens"] is None: + flat["total_prompt_tokens"] = nested["tokens"]["prompt_total"] = counter_int(prompt) flat["server_overall_cache_hit_rate"] = reuse["raw_rate"] nested["cache"].update( { @@ -139,7 +143,10 @@ def apply_validity( } ) nested["tokens"]["prompt_by_source"].update( - {"computed": prompt - cached, "raw": {"combined": cached}} + { + "computed": prompt - cached if nested["tokens"]["prompt_total"] == prompt else None, + "raw": {"combined": cached}, + } ) def gpu_kv_capacity_tokens( @@ -165,15 +172,7 @@ def gpu_kv_capacity_tokens( def _reuse_rate(metrics: dict[str, dict[str, Any]], context: PhaseContext) -> dict[str, Any]: - """Cached/prompt counters in one deployment scope, classified and never corrected. - - In P/D deployments decode workers report every transferred prompt token as - cached, so only prefill series measure reuse; a unified deployment needs an - authoritative mode or explicit combined labels. Numerator and denominator - are paired per endpoint and label set and must satisfy 0 <= cached <= - prompt in every pair. The cached counter can include host-tier reuse, so - the result is a combined-tier rate. - """ + # Decode transfers are not reuse; pair prefill counters within each endpoint and label domain. findings: list[tuple[str, str]] = [] if context.windows is None: findings.append((UNKNOWN, "phase_window_missing")) @@ -182,19 +181,28 @@ def _reuse_rate(metrics: dict[str, dict[str, Any]], context: PhaseContext) -> di for metric in (PROMPT_TOKENS, CACHED_TOKENS): resolved[metric], metric_conflicts = resolve_aliases(metrics, metric) conflicts.extend(metric_conflicts) - findings.extend((REJECTED, conflict["kind"]) for conflict in conflicts) - - endpoint_roles = _endpoint_role_sets(metrics) + endpoint_roles = _endpoint_role_sets(metrics, context.disaggregated) labelled = [] for metric, series_list in resolved.items(): for series in series_list: - role = _series_role(series, endpoint_roles) + role = _series_role(series, endpoint_roles, context.disaggregated) labelled.append((metric, series, role if role in KNOWN_ROLES else "unknown")) - if any(len(roles) > 1 for roles in endpoint_roles.values()): - findings.append((UNKNOWN, "role_conflict")) present = {role for _, _, role in labelled} scope, scope_findings = _deployment_scope(present, context.disaggregated) findings.extend(scope_findings) + scoped_endpoints = { + str(series.get("endpoint_url", "")) + for _, series, role in labelled + if role == scope or (role == "unknown" and context.disaggregated is False) + } + if any(len(endpoint_roles.get(url, set())) > 1 for url in scoped_endpoints): + findings.append((UNKNOWN, "role_conflict")) + findings.extend( + (REJECTED, conflict["kind"]) + for conflict in conflicts + if conflict["endpoint_url"] in scoped_endpoints + or _series_role(conflict, endpoint_roles, context.disaggregated) != "decode" + ) in_scope: dict[str, list[dict[str, Any]]] = {PROMPT_TOKENS: [], CACHED_TOKENS: []} excluded: list[dict[str, Any]] = [] @@ -390,37 +398,51 @@ def _finite_sum(values: Iterable[float | None]) -> float | None: return total if found else None -def _label_role(series: dict[str, Any]) -> str | None: - mode = label_value(series, "disaggregation_mode") or label_value(series, "dynamo_component") - return _normalize_role(mode) if mode else None +def _label_role(series: dict[str, Any], disaggregated: bool | None = None) -> str | None: + mode = label_value(series, "disaggregation_mode") + if mode: + role = _normalize_role(mode) + return role if role in KNOWN_ROLES else "unknown" + component = label_value(series, "dynamo_component") + if component == "backend" and disaggregated is False: + return "combined" + role = _normalize_role(component) if component else None + return role if role in KNOWN_ROLES else None -def _endpoint_role_sets(metrics: dict[str, dict[str, Any]]) -> dict[str, set[str]]: - roles: dict[str, set[str]] = {} +def _endpoint_role_sets( + metrics: dict[str, dict[str, Any]], disaggregated: bool | None = None +) -> dict[str, set[str]]: + explicit: dict[str, set[str]] = {} + inferred: dict[str, set[str]] = {} for entry in metrics.values(): if not isinstance(entry, dict): continue series_list = entry.get("series") - if not isinstance(series_list, list): - continue - for series in series_list: + for series in series_list if isinstance(series_list, list) else []: if not isinstance(series, dict) or not series.get("endpoint_url"): continue - role = _label_role(series) - if role: - roles.setdefault(str(series["endpoint_url"]), set()).add(role) - return roles + role = _label_role(series, disaggregated) + if role in KNOWN_ROLES: + target = explicit if label_value(series, "disaggregation_mode") else inferred + target.setdefault(str(series["endpoint_url"]), set()).add(role) + return {**inferred, **explicit} -def _series_role(series: dict[str, Any], roles: dict[str, set[str]]) -> str: - own = _label_role(series) - if own: - return own +def _series_role( + series: dict[str, Any], roles: dict[str, set[str]], disaggregated: bool | None = None +) -> str: endpoint_roles = roles.get(str(series.get("endpoint_url", "")), set()) - return next(iter(endpoint_roles)) if len(endpoint_roles) == 1 else "unknown" + if label_value(series, "disaggregation_mode"): + return _label_role(series, disaggregated) or "unknown" + if len(endpoint_roles) == 1: + return next(iter(endpoint_roles)) + return _label_role(series, disaggregated) or "unknown" -def _trtllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: +def _trtllm_sources( + metrics: dict[str, dict[str, Any]], *, reuse: dict[str, Any] | None = None +) -> list[dict[str, Any]]: endpoints: set[str] = set() for entry in metrics.values(): if not isinstance(entry, dict): @@ -433,7 +455,9 @@ def _trtllm_sources(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: endpoints.add(str(series["endpoint_url"])) roles = { endpoint: next(iter(found)) if len(found) == 1 else "unknown" - for endpoint, found in _endpoint_role_sets(metrics).items() + for endpoint, found in _endpoint_role_sets( + metrics, reuse.get("deployment_disaggregated") if reuse else None + ).items() } sources: list[dict[str, Any]] = [] @@ -444,7 +468,6 @@ def series_filter(series: dict, endpoint: str = endpoint) -> bool: prompt_tokens = _alias_total(metrics, PROMPT_TOKENS, series_filter) generation_tokens = _alias_total(metrics, GENERATION_TOKENS, series_filter) - cached_tokens = _alias_total(metrics, CACHED_TOKENS, series_filter) hit_rate = normalize_fraction( gauge_stat( metrics, @@ -453,8 +476,12 @@ def series_filter(series: dict, endpoint: str = endpoint) -> bool: series_filter=series_filter, ) ) - if hit_rate is None: - hit_rate = rate(cached_tokens, prompt_tokens) + if hit_rate is None and reuse is not None and reuse["published"]: + domains = [d for d in reuse["domains"] if d["endpoint_url"] == endpoint] + if domains: + hit_rate = rate( + sum(d["cached"] for d in domains), sum(d["prompt"] for d in domains) + ) sources.append( { "id": f"{roles.get(endpoint, 'worker')}|{endpoint}", diff --git a/inferencex-e2e/infx/results/agentic/cache_validity.py b/inferencex-e2e/infx/results/agentic/cache_validity.py index 446cd013e2..ba76bbb2b0 100644 --- a/inferencex-e2e/infx/results/agentic/cache_validity.py +++ b/inferencex-e2e/infx/results/agentic/cache_validity.py @@ -1,11 +1,3 @@ -"""Consumer-facing cache validity core, filled for every adapter. - -``status`` keeps the adapter's own state and ``status_class`` maps it onto one vocabulary, so a -consumer can apply a single rule to every producer. ``published`` says whether any standard cache -rate was written, ``pricing_eligible`` whether it may be priced as an exact actual observation, and -``basis`` whether the written rates are disjoint tiers or one combined total. -""" - from __future__ import annotations from typing import Any diff --git a/inferencex-e2e/infx/results/agentic/phase_validity.py b/inferencex-e2e/infx/results/agentic/phase_validity.py index ee424b604d..a9eaf79731 100644 --- a/inferencex-e2e/infx/results/agentic/phase_validity.py +++ b/inferencex-e2e/infx/results/agentic/phase_validity.py @@ -1,14 +1,3 @@ -"""Phase validity of AIPerf server-metric counter and histogram series. - -AIPerf exports each series' profiling delta (``stats``) next to 1 s timeslices -whose window runs from the first to the last PROFILING-tagged sample of that -series. ``stats`` minus the timeslice sum is the increment between the reference -sample and the first PROFILING sample. Series are classified here and never -corrected: the exporter clamps negative deltas, so the values cannot be repaired -from the export alone, and no time tolerance is used to treat activity at or -outside the phase boundary as profiling traffic. -""" - from __future__ import annotations import math diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index 82315df8f4..9b6acbc419 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -13,7 +13,7 @@ empty_server_metrics, ) from .cache_validity import summarize_cache_validity -from .common import index_server_metrics +from .common import index_server_metrics, sum_stat from .phase_validity import PhaseContext, export_context, phase_windows, summarize, warmup_totals @@ -89,6 +89,42 @@ def endpoint_roles_from_env(env: Mapping[str, str]) -> dict[str, str] | None: return roles or None +def _disaggregated_prompt_total( + metrics: dict[str, dict[str, Any]], records: list[dict[str, Any]], nested: dict[str, Any] +) -> int | None: + values = [] + for record in records: + metric = record.get("metrics", {}).get("input_sequence_length") + value = metric.get("value") if isinstance(metric, dict) else metric + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + or int(value) != value + ): + break + values.append(int(value)) + if records and len(values) == len(records): + return sum(values) + frontend = sum_stat( + metrics, "dynamo_frontend_input_sequence_tokens", preferred_keys=("total", "sum") + ) + if ( + frontend is not None + and math.isfinite(frontend) + and frontend >= 0 + and int(frontend) == frontend + ): + return int(frontend) + reuse = nested["validity"]["cache_rate"] + if reuse.get("published") and reuse.get("role_scope") == "prefill": + prompt = reuse.get("prompt_tokens") + if isinstance(prompt, (int, float)) and math.isfinite(prompt) and prompt >= 0: + return round(prompt) + return None + + def compute_server_metrics( server_metrics: dict[str, Any], *, @@ -183,5 +219,11 @@ def compute_server_metrics( nested["tokens"]["generation_total"] = flat["total_generation_tokens"] nested["tokens"]["requests_completed"] = flat["total_requests_completed"] + if disaggregated is True and adapter.name != "atom": + prompt = _disaggregated_prompt_total(metrics, records, nested) + if prompt != nested["tokens"]["prompt_total"]: + nested["tokens"]["prompt_by_source"]["computed"] = None + flat["total_prompt_tokens"] = nested["tokens"]["prompt_total"] = prompt + nested["cache_validity"] = summarize_cache_validity(nested) return flat, nested, warnings diff --git a/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py b/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py index d71302a608..ff085ca639 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py @@ -1,7 +1,7 @@ -"""The cache validity core is filled for every adapter, so one consumer rule covers every producer.""" - from __future__ import annotations +import pytest + from infx.results.agentic.server_metrics import compute_server_metrics from infx.tests.results.agentic.test_atom_cache_validity import _export as _atom_export from infx.tests.results.agentic.test_atom_cache_validity import _run as _run_atom @@ -82,3 +82,46 @@ def test_missing_server_metrics_is_missing(self) -> None: assert core["status_class"] == "MISSING" assert core["published"] is False assert core["reasons"] == ["server_metrics_missing"] + + +class TestDisaggregatedPromptTotals: + @pytest.mark.parametrize( + "framework, metric", + [ + ("vllm", "vllm:prompt_tokens"), + ("sglang", "sglang:prompt_tokens"), + ("trtllm", "trtllm_prompt_tokens"), + ], + ) + def test_complete_profile_records_replace_duplicate_worker_prompts(self, framework, metric): + export = {"metrics": {metric: _counter(2000.0)}} + records = [{"metrics": {"input_sequence_length": {"value": n}}} for n in (800, 200)] + flat, nested, _ = compute_server_metrics( + export, framework=framework, records=records, disaggregated=True + ) + + assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 1000 + assert nested["cache_validity"]["pricing_eligible"] is False + + @pytest.mark.parametrize("value", [None, True, -1, 1.5, float("inf")]) + def test_partial_or_invalid_profile_records_do_not_supply_a_total(self, value): + records = [{"metrics": {"input_sequence_length": n}} for n in (800, value)] + flat, nested, _ = compute_server_metrics( + {"metrics": {"vllm:prompt_tokens": _counter(2000.0)}}, + framework="vllm", + records=records, + disaggregated=True, + ) + + assert flat["total_prompt_tokens"] is None + assert nested["tokens"]["prompt_total"] is None + + def test_frontend_total_remains_available_without_client_usage(self): + flat, nested, _ = compute_server_metrics( + {"metrics": {"dynamo_frontend_input_sequence_tokens": _counter(1234.0)}}, + framework="dynamo-trt", + records=[], + disaggregated=True, + ) + + assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 1234 diff --git a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py index 5601c05dec..2137bb8094 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py @@ -1,5 +1,3 @@ -"""TRT reuse rates require paired counters and profiling-window evidence.""" - from __future__ import annotations import copy @@ -114,6 +112,7 @@ def test_cached_above_prompt_is_rejected_and_not_published(self) -> None: assert nested["tokens"]["prompt_by_source"]["computed"] is None assert _rate(nested)["status"] == "REJECTED" assert "cached_exceeds_prompt" in _rate(nested)["reasons"] + assert nested["sources"][0]["prefix_cache_hit_rate"] is None assert _rate(nested)["raw_rate"] == pytest.approx(1.2) def test_one_domain_above_prompt_is_not_offset_by_the_sum(self) -> None: @@ -160,6 +159,42 @@ def test_idle_prefill_domain_does_not_block(self) -> None: class TestDeploymentRolesAndCounterPairs: + @pytest.mark.parametrize("explicit_mode, expected", [(None, 0.9), ("decode", None)]) + def test_aggregated_backend_component_does_not_override_explicit_mode( + self, explicit_mode, expected + ) -> None: + labels = {"disaggregation_mode": explicit_mode} if explicit_mode else {} + export = _export( + [_series(PRE[0], "backend", 1000.0, labels=labels)], + [_series(PRE[0], "backend", 900.0, labels=labels)], + ) + nested = _run(export, disaggregated=False) + + assert nested["cache"]["overall_cache_hit_rate"] == expected + assert nested["cache_validity"]["pricing_eligible"] is False + assert nested["sources"][0]["prefix_cache_hit_rate"] == expected + + def test_non_role_components_do_not_conflict_with_explicit_worker_roles(self) -> None: + export = _export( + [ + _series(PRE[0], None, 1000.0, labels={"disaggregation_mode": "prefill"}), + _series(DEC, None, 1000.0, labels={"disaggregation_mode": "decode"}), + ], + [ + _series(PRE[0], None, 900.0, labels={"disaggregation_mode": "prefill"}), + _series(DEC, None, 1000.0, labels={"disaggregation_mode": "decode"}), + ], + ) + export["metrics"]["other_metric"] = { + "type": "gauge", + "series": [_series(PRE[0], "tensorrt_llm", 1.0), _series(DEC, "tensorrt_llm", 1.0)], + } + nested = _run(export, disaggregated=True) + + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) + assert _rate(nested)["role_scope"] == "prefill" + assert {s["endpoint_url"] for s in _rate(nested)["excluded"]} == {DEC} + def test_decode_only_is_not_treated_as_combined(self) -> None: export = _export( [_series(DEC, "backend", 1000.0)], [_series(DEC, "backend", 1000.0)] diff --git a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_phase_validity.py b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_phase_validity.py index 8392f558e4..d5cfc1194d 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_phase_validity.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_phase_validity.py @@ -1,12 +1,3 @@ -"""Phase validity of AIPerf server-metric counters and the TensorRT-LLM reuse rate. - -Fixtures follow the AIPerf 0.12 ``server_metrics_export.json`` shape: counter -names without the Prometheus ``_total`` suffix, 1 s timeslices per series and -``summary.phase_time_ranges``. The P/D topology mirrors the Dynamo TRT runs -(``prefill`` and ``backend`` components); decode workers report every -transferred prompt token as cached. -""" - from __future__ import annotations import copy From 75d40e346b45aa801112dbdd88b6dc2f35a91916 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Thu, 22 Oct 2026 20:35:11 +0000 Subject: [PATCH 07/13] fix(agentx): require complete profiling prompt totals on every export path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P/D 总量使用完整合法的客户端或前端计数，覆盖缺失 metrics 的早退路径，并拒绝部分、负数抵消、布尔和非有限输入。 --- .../infx/results/agentic/backends/base.py | 3 +- .../infx/results/agentic/server_metrics.py | 77 +++++++++++-------- .../agentic/test_cache_validity_core.py | 24 ++++++ 3 files changed, 73 insertions(+), 31 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/backends/base.py b/inferencex-e2e/infx/results/agentic/backends/base.py index 192a53d7d6..835ba2eace 100644 --- a/inferencex-e2e/infx/results/agentic/backends/base.py +++ b/inferencex-e2e/infx/results/agentic/backends/base.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math from collections.abc import Iterable from typing import TYPE_CHECKING, Any @@ -135,7 +136,7 @@ def apply_profile_totals(flat: dict[str, Any], records: list[dict[str, Any]]) -> def counter_int(value: float | None) -> int | None: - if value is None: + if value is None or not math.isfinite(value): return None return round(value) diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index 345107cda2..b53c10bc33 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -13,7 +13,7 @@ empty_server_metrics, ) from .cache_validity import summarize_cache_validity -from .common import index_server_metrics, sum_stat +from .common import index_server_metrics from .phase_validity import PhaseContext, export_context, phase_windows, summarize, warmup_totals @@ -91,42 +91,60 @@ def endpoint_roles_from_env(env: Mapping[str, str]) -> dict[str, str] | None: return roles or None +def _integer_total(values: list[Any]) -> int | None: + if not values or any( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or (isinstance(value, float) and not math.isfinite(value)) + or value < 0 + or int(value) != value + for value in values + ): + return None + return sum(int(value) for value in values) + + def _disaggregated_prompt_total( metrics: dict[str, dict[str, Any]], records: list[dict[str, Any]], nested: dict[str, Any] ) -> int | None: values = [] for record in records: metric = record.get("metrics", {}).get("input_sequence_length") - value = metric.get("value") if isinstance(metric, dict) else metric - if ( - isinstance(value, bool) - or not isinstance(value, (int, float)) - or not math.isfinite(value) - or value < 0 - or int(value) != value - ): - break - values.append(int(value)) - if records and len(values) == len(records): - return sum(values) - frontend = sum_stat( - metrics, "dynamo_frontend_input_sequence_tokens", preferred_keys=("total", "sum") - ) - if ( - frontend is not None - and math.isfinite(frontend) - and frontend >= 0 - and int(frontend) == frontend - ): - return int(frontend) + values.append(metric.get("value") if isinstance(metric, dict) else metric) + prompt = _integer_total(values) + if prompt is not None: + return prompt + frontend = metrics.get("dynamo_frontend_input_sequence_tokens") + if frontend is not None: + series = frontend.get("series") + if not isinstance(series, list): + return None + return _integer_total( + [ + item.get("stats", {}).get("total") + if isinstance(item, dict) and isinstance(item.get("stats"), dict) + else None + for item in series + ] + ) reuse = nested["validity"]["cache_rate"] if reuse.get("published") and reuse.get("role_scope") == "prefill": - prompt = reuse.get("prompt_tokens") - if isinstance(prompt, (int, float)) and math.isfinite(prompt) and prompt >= 0: - return round(prompt) + return _integer_total([reuse.get("prompt_tokens")]) return None +def _apply_disaggregated_prompt_total( + metrics: dict[str, dict[str, Any]], + records: list[dict[str, Any]], + flat: dict[str, Any], + nested: dict[str, Any], +) -> None: + prompt = _disaggregated_prompt_total(metrics, records, nested) + if prompt != nested["tokens"]["prompt_total"]: + nested["tokens"]["prompt_by_source"]["computed"] = None + flat["total_prompt_tokens"] = nested["tokens"]["prompt_total"] = prompt + + def compute_server_metrics( server_metrics: dict[str, Any], *, @@ -171,6 +189,8 @@ def compute_server_metrics( nested["tokens"]["prompt_total"] = flat["total_prompt_tokens"] nested["tokens"]["generation_total"] = flat["total_generation_tokens"] nested["tokens"]["requests_completed"] = flat["total_requests_completed"] + if disaggregated is True and (adapter is None or adapter.name != "atom"): + _apply_disaggregated_prompt_total(metrics, records, flat, nested) nested["cache_validity"] = summarize_cache_validity(nested) return flat, nested, warnings @@ -222,10 +242,7 @@ def compute_server_metrics( nested["tokens"]["requests_completed"] = flat["total_requests_completed"] if disaggregated is True and adapter.name != "atom": - prompt = _disaggregated_prompt_total(metrics, records, nested) - if prompt != nested["tokens"]["prompt_total"]: - nested["tokens"]["prompt_by_source"]["computed"] = None - flat["total_prompt_tokens"] = nested["tokens"]["prompt_total"] = prompt + _apply_disaggregated_prompt_total(metrics, records, flat, nested) nested["cache_validity"] = summarize_cache_validity(nested) return flat, nested, warnings diff --git a/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py b/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py index ff085ca639..3ea14e9db3 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py @@ -116,6 +116,30 @@ def test_partial_or_invalid_profile_records_do_not_supply_a_total(self, value): assert flat["total_prompt_tokens"] is None assert nested["tokens"]["prompt_total"] is None + def test_missing_metrics_does_not_publish_a_partial_client_total(self): + records = [{"metrics": {"input_sequence_length": 800}}, {"metrics": {}}] + flat, nested, _ = compute_server_metrics( + {}, framework="vllm", records=records, disaggregated=True + ) + + assert flat["total_prompt_tokens"] is None + assert nested["tokens"]["prompt_total"] is None + assert nested["cache_validity"]["status_class"] == "MISSING" + + @pytest.mark.parametrize("totals", [[1234, None], [-10, 1244], [True], [float("inf")]]) + def test_incomplete_or_invalid_frontend_cannot_supply_a_prompt_total(self, totals): + counter = _counter(0) + counter["series"] = [{"stats": {"total": value}} for value in totals] + flat, nested, _ = compute_server_metrics( + {"metrics": {"dynamo_frontend_input_sequence_tokens": counter}}, + framework="dynamo-trt", + records=[], + disaggregated=True, + ) + + assert flat["total_prompt_tokens"] is None + assert nested["tokens"]["prompt_total"] is None + def test_frontend_total_remains_available_without_client_usage(self): flat, nested, _ = compute_server_metrics( {"metrics": {"dynamo_frontend_input_sequence_tokens": _counter(1234.0)}}, From 1b4d4e270e153ff54b9140a5afd2730fc9be5e83 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Thu, 22 Oct 2026 20:35:49 +0000 Subject: [PATCH 08/13] fix(agentx): keep explicit invalid TRT roles unresolved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 显式非法 disaggregation_mode 不再按聚合部署回退为 combined；无标签且可信聚合模式仍可接受。 --- inferencex-e2e/infx/results/agentic/backends/trtllm.py | 6 +++++- .../tests/results/agentic/test_server_metrics_closeout.py | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/backends/trtllm.py b/inferencex-e2e/infx/results/agentic/backends/trtllm.py index 7c66924f5c..b997da5707 100644 --- a/inferencex-e2e/infx/results/agentic/backends/trtllm.py +++ b/inferencex-e2e/infx/results/agentic/backends/trtllm.py @@ -213,7 +213,11 @@ def _reuse_rate(metrics: dict[str, dict[str, Any]], context: PhaseContext) -> di "endpoint_url": str(series.get("endpoint_url", "")), "role": role, } - if role == "unknown" and context.disaggregated is False: + if ( + role == "unknown" + and context.disaggregated is False + and not label_value(series, "disaggregation_mode") + ): role = "combined" if scope is not None and role == scope: record = classify_series( diff --git a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py index 2137bb8094..f962dd9d75 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py @@ -159,7 +159,9 @@ def test_idle_prefill_domain_does_not_block(self) -> None: class TestDeploymentRolesAndCounterPairs: - @pytest.mark.parametrize("explicit_mode, expected", [(None, 0.9), ("decode", None)]) + @pytest.mark.parametrize( + "explicit_mode, expected", [(None, 0.9), ("decode", None), ("garbage", None)] + ) def test_aggregated_backend_component_does_not_override_explicit_mode( self, explicit_mode, expected ) -> None: From 362265abf2dca0b8c4da24cb52b32dd632ff81fa Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Thu, 22 Oct 2026 20:42:32 +0000 Subject: [PATCH 09/13] fix(agentx): preserve typed frontend totals and separate changed scopes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按 counter total 或 histogram sum 严格读取前端总量；客户端替换 P/D worker 范围时清除三个标准化分量并保留 raw 诊断。 --- .../infx/results/agentic/server_metrics.py | 8 ++- .../agentic/test_cache_validity_core.py | 60 +++++++++++++++++-- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index b53c10bc33..ad34632a46 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -117,11 +117,12 @@ def _disaggregated_prompt_total( frontend = metrics.get("dynamo_frontend_input_sequence_tokens") if frontend is not None: series = frontend.get("series") - if not isinstance(series, list): + key = {"counter": "total", "histogram": "sum"}.get(frontend.get("type")) + if not isinstance(series, list) or key is None: return None return _integer_total( [ - item.get("stats", {}).get("total") + item.get("stats", {}).get(key) if isinstance(item, dict) and isinstance(item.get("stats"), dict) else None for item in series @@ -141,7 +142,8 @@ def _apply_disaggregated_prompt_total( ) -> None: prompt = _disaggregated_prompt_total(metrics, records, nested) if prompt != nested["tokens"]["prompt_total"]: - nested["tokens"]["prompt_by_source"]["computed"] = None + for field in ("gpu_cache_hit", "cpu_or_external_cache_hit", "computed"): + nested["tokens"]["prompt_by_source"][field] = None flat["total_prompt_tokens"] = nested["tokens"]["prompt_total"] = prompt diff --git a/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py b/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py index 3ea14e9db3..621d7e986c 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py @@ -140,12 +140,62 @@ def test_incomplete_or_invalid_frontend_cannot_supply_a_prompt_total(self, total assert flat["total_prompt_tokens"] is None assert nested["tokens"]["prompt_total"] is None - def test_frontend_total_remains_available_without_client_usage(self): + @pytest.mark.parametrize("metric_type, key", [("counter", "total"), ("histogram", "sum")]) + def test_frontend_total_remains_available_without_client_usage(self, metric_type, key): + export = { + "metrics": { + name: {"type": metric_type, "series": [{"stats": {key: value, "count": 10}}]} + for name, value in ( + ("dynamo_frontend_input_sequence_tokens", 1000.0), + ("dynamo_frontend_cached_tokens", 800.0), + ) + } + } flat, nested, _ = compute_server_metrics( - {"metrics": {"dynamo_frontend_input_sequence_tokens": _counter(1234.0)}}, - framework="dynamo-trt", - records=[], + export, framework="dynamo-vllm", records=[], disaggregated=True + ) + + assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 1000 + assert flat["server_overall_cache_hit_rate"] == pytest.approx(0.8) + assert nested["cache"]["frontend_cache_hit_rate"] == pytest.approx(0.8) + + def test_replacing_worker_scope_clears_normalized_tiers_and_keeps_raw_diagnostics(self): + export = { + "metrics": { + "sglang:prompt_tokens": { + "type": "counter", + "series": [ + {"endpoint_url": endpoint, "stats": {"total": 1000}} + for endpoint in ("http://prefill/metrics", "http://decode/metrics") + ], + }, + "sglang:cached_tokens": { + "type": "counter", + "series": [ + { + "endpoint_url": endpoint, + "labels": {"cache_source": source}, + "stats": {"total": total}, + } + for source, total in (("device", 900), ("host", 100)) + for endpoint in ("http://prefill/metrics", "http://decode/metrics") + ], + }, + } + } + flat, nested, _ = compute_server_metrics( + export, + framework="sglang", + records=[{"metrics": {"input_sequence_length": 1000}}], disaggregated=True, ) - assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 1234 + assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 1000 + assert nested["tokens"]["prompt_by_source"] == { + "gpu_cache_hit": None, + "cpu_or_external_cache_hit": None, + "computed": None, + "raw": {"device": 1800, "host": 200}, + } + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(1.0) + assert nested["cache_validity"]["pricing_eligible"] is False From 20217c76e02c75b626cc0790a7dc1774f7aba461 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Thu, 22 Oct 2026 21:18:04 +0000 Subject: [PATCH 10/13] refactor(agentx): narrow TRT validation to profiling counter pairs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将 TRT 校验收窄为现代 profiling 配对 counter，删除通用阶段分类与历史窗口恢复，保留 P/D 总量守恒。 --- .../infx/results/agentic/backends/base.py | 19 +- .../infx/results/agentic/backends/trtllm.py | 483 +++++++---------- .../infx/results/agentic/cache_validity.py | 90 ++-- .../infx/results/agentic/phase_validity.py | 371 -------------- .../infx/results/agentic/server_metrics.py | 28 +- ...py => test_disaggregated_prompt_totals.py} | 66 --- .../agentic/test_process_agentic_result.py | 9 +- .../agentic/test_server_metrics_closeout.py | 484 ------------------ .../test_server_metrics_phase_validity.py | 402 --------------- .../agentic/test_trtllm_cache_counters.py | 207 ++++++++ 10 files changed, 458 insertions(+), 1701 deletions(-) delete mode 100644 inferencex-e2e/infx/results/agentic/phase_validity.py rename inferencex-e2e/infx/tests/results/agentic/{test_cache_validity_core.py => test_disaggregated_prompt_totals.py} (65%) delete mode 100644 inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py delete mode 100644 inferencex-e2e/infx/tests/results/agentic/test_server_metrics_phase_validity.py create mode 100644 inferencex-e2e/infx/tests/results/agentic/test_trtllm_cache_counters.py diff --git a/inferencex-e2e/infx/results/agentic/backends/base.py b/inferencex-e2e/infx/results/agentic/backends/base.py index 835ba2eace..637e629683 100644 --- a/inferencex-e2e/infx/results/agentic/backends/base.py +++ b/inferencex-e2e/infx/results/agentic/backends/base.py @@ -4,10 +4,7 @@ import math from collections.abc import Iterable -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from infx.results.agentic.phase_validity import PhaseContext +from typing import Any SERVER_CACHE_FLAT_FIELDS = { "server_gpu_cache_hit_rate": None, @@ -49,15 +46,6 @@ def gpu_kv_capacity_tokens( ) -> int | None: return None - def apply_validity( - self, - metrics: dict[str, dict[str, Any]], # noqa: ARG002 - flat: dict[str, Any], # noqa: ARG002 - nested: dict[str, Any], # noqa: ARG002 - context: PhaseContext, # noqa: ARG002 - ) -> None: - return None - def empty_server_metrics(present: bool, metric_count: int) -> dict[str, Any]: return { @@ -109,18 +97,13 @@ def empty_server_metrics(present: bool, metric_count: int) -> dict[str, Any]: }, "sources": [], "validity": { - "phase_windows": None, - "endpoints": {"configured": 0, "unscraped": []}, "deployment": {"disaggregated": None}, - "series_by_status": {}, - "metrics_by_status": {}, "cache_rate": { "status": "NOT_EVALUATED", "published": False, "eligible_exact_actual": False, "reasons": ["adapter_does_not_assess_cache_rate"], }, - "diagnostics": {}, }, } diff --git a/inferencex-e2e/infx/results/agentic/backends/trtllm.py b/inferencex-e2e/infx/results/agentic/backends/trtllm.py index b997da5707..ad3762eb22 100644 --- a/inferencex-e2e/infx/results/agentic/backends/trtllm.py +++ b/inferencex-e2e/infx/results/agentic/backends/trtllm.py @@ -3,32 +3,18 @@ from __future__ import annotations import math -import statistics from collections.abc import Callable, Iterable from typing import Any +from infx.results.agentic.cache_validity import modern_profile_error from infx.results.agentic.common import ( gauge_stat, label_value, metric_series, normalize_fraction, rate, - series_stat, sum_stat, ) -from infx.results.agentic.phase_validity import ( - BOUNDARY_UNCERTAIN, - CONDITIONAL_INTERVAL, - PARTIAL_COVERAGE, - REJECTED, - UNKNOWN, - VALID, - PhaseContext, - classify_series, - resolve_aliases, - series_key, - worst, -) from .base import ServerMetricsBackend, counter_int @@ -37,11 +23,12 @@ GENERATION_TOKENS = "trtllm_generation_tokens" # Interval counters do not identify the billed cohort or prove expected worker coverage. UNBOUND_REASONS = ("request_scope_not_bound", "expected_instances_not_verified") -KNOWN_ROLES = ("prefill", "decode", "combined") +ROLE_LABELS = {"disaggregation_mode", "dynamo_component"} class TrtllmBackend(ServerMetricsBackend): name = "trtllm" + accepts_context = True def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: metric_names = set(metrics) @@ -54,14 +41,14 @@ def populate( metrics: dict[str, dict[str, Any]], flat: dict[str, Any], nested: dict[str, Any], + *, + context: dict[str, Any] | None = None, ) -> None: + context = context or {} prompt_total = sum_stat( metrics, "dynamo_frontend_input_sequence_tokens", preferred_keys=("total", "sum") ) generation_total = _first_total(metrics, "dynamo_frontend_output_tokens", GENERATION_TOKENS) - diagnostic = _hit_rate_gauge_diagnostic(metrics) - if diagnostic is not None: - nested["validity"]["diagnostics"]["trtllm_kv_cache_hit_rate"] = diagnostic flat["total_prompt_tokens"] = counter_int(prompt_total) flat["total_generation_tokens"] = counter_int(generation_total) @@ -118,36 +105,29 @@ def populate( }, } ) - nested["sources"] = _trtllm_sources(metrics) - - def apply_validity( - self, - metrics: dict[str, dict[str, Any]], - flat: dict[str, Any], - nested: dict[str, Any], - context: PhaseContext, - ) -> None: - reuse = _reuse_rate(metrics, context) + reuse, paired = _reuse_rate(metrics, context) nested["validity"]["cache_rate"] = reuse - nested["sources"] = _trtllm_sources(metrics, reuse=reuse) - if not reuse["published"]: - return - cached, prompt = reuse["cached_tokens"], reuse["prompt_tokens"] - if flat["total_prompt_tokens"] is None: - flat["total_prompt_tokens"] = nested["tokens"]["prompt_total"] = counter_int(prompt) - flat["server_overall_cache_hit_rate"] = reuse["raw_rate"] - nested["cache"].update( - { - "overall_cache_hit_rate": reuse["raw_rate"], - "cached_tokens_by_source": {"combined": cached}, - } - ) - nested["tokens"]["prompt_by_source"].update( - { - "computed": prompt - cached if nested["tokens"]["prompt_total"] == prompt else None, - "raw": {"combined": cached}, - } - ) + nested["sources"] = _trtllm_sources(metrics, paired, context.get("disagg")) + if reuse["published"]: + prompt = reuse["prompt_tokens"] + cached = sum(item[1] for item in paired.values()) + if flat["total_prompt_tokens"] is None: + flat["total_prompt_tokens"] = nested["tokens"]["prompt_total"] = prompt + flat["server_overall_cache_hit_rate"] = cached / prompt + nested["cache"].update( + { + "overall_cache_hit_rate": cached / prompt, + "cached_tokens_by_source": {"combined": cached}, + } + ) + nested["tokens"]["prompt_by_source"].update( + { + "computed": prompt - cached + if nested["tokens"]["prompt_total"] == prompt + else None, + "raw": {"combined": cached}, + } + ) def gpu_kv_capacity_tokens( self, @@ -171,191 +151,130 @@ def gpu_kv_capacity_tokens( return counter_int(max_blocks * tokens_per_block) -def _reuse_rate(metrics: dict[str, dict[str, Any]], context: PhaseContext) -> dict[str, Any]: - # Decode transfers are not reuse; pair prefill counters within each endpoint and label domain. - findings: list[tuple[str, str]] = [] - if context.windows is None: - findings.append((UNKNOWN, "phase_window_missing")) - resolved: dict[str, list[dict[str, Any]]] = {} - conflicts: list[dict[str, Any]] = [] - for metric in (PROMPT_TOKENS, CACHED_TOKENS): - resolved[metric], metric_conflicts = resolve_aliases(metrics, metric) - conflicts.extend(metric_conflicts) - endpoint_roles = _endpoint_role_sets(metrics, context.disaggregated) - labelled = [] - for metric, series_list in resolved.items(): - for series in series_list: - role = _series_role(series, endpoint_roles, context.disaggregated) - labelled.append((metric, series, role if role in KNOWN_ROLES else "unknown")) - present = {role for _, _, role in labelled} - scope, scope_findings = _deployment_scope(present, context.disaggregated) - findings.extend(scope_findings) - scoped_endpoints = { - str(series.get("endpoint_url", "")) - for _, series, role in labelled - if role == scope or (role == "unknown" and context.disaggregated is False) - } - if any(len(endpoint_roles.get(url, set())) > 1 for url in scoped_endpoints): - findings.append((UNKNOWN, "role_conflict")) - findings.extend( - (REJECTED, conflict["kind"]) - for conflict in conflicts - if conflict["endpoint_url"] in scoped_endpoints - or _series_role(conflict, endpoint_roles, context.disaggregated) != "decode" - ) - - in_scope: dict[str, list[dict[str, Any]]] = {PROMPT_TOKENS: [], CACHED_TOKENS: []} - excluded: list[dict[str, Any]] = [] - unresolved: list[dict[str, Any]] = [] - for metric, series, role in labelled: - entry = { - "metric": metric, - "endpoint_url": str(series.get("endpoint_url", "")), - "role": role, - } - if ( - role == "unknown" - and context.disaggregated is False - and not label_value(series, "disaggregation_mode") - ): - role = "combined" - if scope is not None and role == scope: - record = classify_series( - series, - "counter", - context.windows, - has_reference=context.has_reference(metric, series), - warmup_total=context.warmup_total(metric, series), - ) - in_scope[metric].append( - {**entry, "role": role, "key": series_key(metric, series), **record} - ) - findings.extend((record["status"], code) for code in record["reasons"]) - elif scope == "prefill" and role == "decode": - excluded.append({**entry, "reason": "decode_role_excluded"}) - elif role == "unknown": - unresolved.append(entry) - if unresolved and scope is not None: - findings.append((UNKNOWN, "role_unknown")) - - domains = _pair_domains(in_scope, findings) if scope is not None else [] - if context.unscraped is None: - findings.append((UNKNOWN, "scrape_coverage_unknown")) - elif context.unscraped: - findings.append((PARTIAL_COVERAGE, "endpoint_not_scraped")) - - prompt = _finite_sum(r["total"] for r in in_scope[PROMPT_TOKENS]) - cached = _finite_sum(r["total"] for r in in_scope[CACHED_TOKENS]) - raw_rate = rate(cached, prompt) - status = worst(status for status, _ in findings) - if status == VALID and raw_rate is None: - status = UNKNOWN - findings.append((UNKNOWN, "zero_denominator")) - published = status == VALID - if published: - status = CONDITIONAL_INTERVAL - for records in in_scope.values(): - for record in records: - record.pop("key") - return { - "status": status, - "published": published, +def _reuse_rate( + metrics: dict[str, dict[str, Any]], context: dict[str, Any] +) -> tuple[dict[str, Any], dict[str, tuple[int, int]]]: + result = { + "status": "UNKNOWN", + "published": False, "eligible_exact_actual": False, - "role_scope": scope or "unresolved", - "deployment_disaggregated": context.disaggregated, - "tier": "combined", - "numerator": CACHED_TOKENS, - "denominator": PROMPT_TOKENS, - "cached_tokens": cached, - "prompt_tokens": prompt, - "raw_rate": raw_rate, - "reasons": list(dict.fromkeys([code for _, code in findings] + list(UNBOUND_REASONS))), - "series": in_scope[PROMPT_TOKENS] + in_scope[CACHED_TOKENS], - "domains": domains, - "excluded": excluded, - "unresolved": unresolved, - "alias_conflicts": conflicts, + "reasons": [], + "role_scope": None, + "prompt_tokens": None, } + disaggregated = context.get("disagg") + roles = _endpoint_role_sets(metrics, disaggregated, context.get("endpoint_roles")) + observed = { + _series_role(series, roles, disaggregated) + for name in (PROMPT_TOKENS, CACHED_TOKENS) + for series in metric_series(metrics, [name, f"{name}_total"]) + } + scope = "prefill" if disaggregated is True else "combined" if disaggregated is False else None + if scope is None: + candidates = observed - {"decode"} + scope = next(iter(candidates)) if len(candidates) == 1 else None + if ( + scope not in ("prefill", "combined") + or observed - {scope, "decode"} + or (scope == "combined" and "decode" in observed) + ): + result["reasons"] = ["role_or_deployment_unresolved"] + return result, {} + if any( + len(roles.get(str(series.get("endpoint_url", "")), set())) > 1 + for name in (PROMPT_TOKENS, CACHED_TOKENS) + for series in metric_series(metrics, [name, f"{name}_total"]) + if _series_role(series, roles, disaggregated) == scope + ): + result["reasons"] = ["role_conflict"] + return result, {} + result["role_scope"] = scope + try: + prompt = _counter_domains(metrics, PROMPT_TOKENS, roles, scope, disaggregated) + cached = _counter_domains(metrics, CACHED_TOKENS, roles, scope, disaggregated) + except ValueError as error: + result.update(status="REJECTED", reasons=[str(error)]) + return result, {} + if not prompt or prompt.keys() != cached.keys(): + result.update(status="PARTIAL_COVERAGE", reasons=["missing_counter_pair"]) + return result, {} + error = modern_profile_error(context.get("collector_export") or {}, {key[0] for key in prompt}) + if error: + result["reasons"] = [error] + return result, {} + if any(cached[key] > value for key, value in prompt.items()): + result.update(status="REJECTED", reasons=["cached_exceeds_prompt"]) + return result, {} + total = sum(prompt.values()) + if total <= 0: + result["reasons"] = ["no_prompt_tokens"] + return result, {} + paired: dict[str, tuple[int, int]] = {} + for key, value in prompt.items(): + full, hit = paired.get(key[0], (0, 0)) + paired[key[0]] = (full + value, hit + cached[key]) + result.update( + status="CONDITIONAL_INTERVAL", + published=True, + reasons=list(UNBOUND_REASONS), + prompt_tokens=total, + ) + return result, paired -def _deployment_scope( - present: set[str], disaggregated: bool | None -) -> tuple[str | None, list[tuple[str, str]]]: - if disaggregated is True: - findings = [(UNKNOWN, "role_conflicts_with_deployment")] if "combined" in present else [] - if "prefill" in present: - return "prefill", findings - return None, [*findings, (UNKNOWN, "no_prefill_scope")] - if disaggregated is False: - if present & {"prefill", "decode"}: - return None, [(UNKNOWN, "role_conflicts_with_deployment")] - return "combined", [] - if "prefill" in present: - if "combined" in present: - return None, [(UNKNOWN, "role_conflict")] - return "prefill", [] - if "combined" in present and "decode" not in present: - return "combined", [] - if "decode" in present: - return None, [(UNKNOWN, "decode_only_no_prefill_scope")] - return None, [(UNKNOWN, "deployment_mode_unknown")] - - -def _pair_domains( - in_scope: dict[str, list[dict[str, Any]]], findings: list[tuple[str, str]] -) -> list[dict[str, Any]]: - prompt = {r["key"][1:]: r for r in in_scope[PROMPT_TOKENS]} - cached = {r["key"][1:]: r for r in in_scope[CACHED_TOKENS]} - if not prompt and not cached: - findings.append((UNKNOWN, "counter_series_missing")) - return [] - if set(prompt) != set(cached): - findings.append((PARTIAL_COVERAGE, "missing_pair")) - domains = [] - for key in sorted(set(prompt) & set(cached)): - f, k = prompt[key], cached[key] - supports = [(r["first_slice_offset_s"], r["last_slice_offset_s"]) for r in (f, k)] - windows_match = None - if None not in supports[0] + supports[1]: - windows_match = supports[0] == supports[1] - if not windows_match: - findings.append((BOUNDARY_UNCERTAIN, "window_support_mismatch")) - within = None - if _finite(f["total"]) and _finite(k["total"]): - within = 0 <= k["total"] <= f["total"] - if not within: - findings.append((REJECTED, "cached_exceeds_prompt")) - domains.append( - { - "endpoint_url": key[0], - "labels": dict(key[1]), - "prompt": f["total"], - "cached": k["total"], - "cached_within_prompt": within, - "windows_match": windows_match, - } - ) - return domains - - -def _hit_rate_gauge_diagnostic(metrics: dict[str, dict[str, Any]]) -> dict[str, Any] | None: - roles = _endpoint_role_sets(metrics) - values: list[float] = [] - by_role: dict[str, list[float]] = {} - for series in metric_series(metrics, "trtllm_kv_cache_hit_rate"): - value = normalize_fraction(series_stat(series, ("avg", "max", "total"))) - if value is None: +def _counter_domains( + metrics: dict[str, dict[str, Any]], + name: str, + roles: dict[str, set[str]], + scope: str, + disaggregated: bool | None, +) -> dict[tuple, int]: + totals: dict[tuple, int] = {} + for alias in (name, f"{name}_total"): + entry = metrics.get(alias) + if entry is None: continue - values.append(value) - by_role.setdefault(_series_role(series, roles), []).append(value) - if not values: - return None - return { - "status": "GAUGE_DERIVED", - "pricing_eligible": False, - "all_series_avg": statistics.mean(values), - "by_role": {role: statistics.mean(v) for role, v in sorted(by_role.items())}, - } + if ( + not isinstance(entry, dict) + or entry.get("type") != "counter" + or not isinstance(entry.get("series"), list) + ): + raise ValueError("invalid_counter_series") + seen = set() + for series in entry["series"]: + if not isinstance(series, dict): + raise ValueError("invalid_counter_series") + if _series_role(series, roles, disaggregated) != scope: + continue + endpoint = series.get("endpoint_url") + labels = series.get("labels") + labels = {} if labels is None else labels + if ( + not isinstance(endpoint, str) + or not endpoint + or not isinstance(labels, dict) + or any( + not isinstance(key, str) or not isinstance(value, str) + for key, value in labels.items() + ) + ): + raise ValueError("invalid_counter_domain") + key = endpoint, tuple(sorted((k, v) for k, v in labels.items() if k not in ROLE_LABELS)) + stats = series.get("stats") + value = stats.get("total") if isinstance(stats, dict) else None + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or (isinstance(value, float) and not math.isfinite(value)) + or value < 0 + or int(value) != value + ): + raise ValueError("invalid_counter_total") + if key in seen or (key in totals and totals[key] != value): + raise ValueError("counter_alias_conflict") + seen.add(key) + totals[key] = int(value) + return totals def _first_total( @@ -372,50 +291,17 @@ def _alias_total( base: str, series_filter: Callable[[dict[str, Any]], bool] | None = None, ) -> float | None: - series_list, conflicts = resolve_aliases(metrics, base) - if any( - series_filter is None or series_filter({"endpoint_url": c["endpoint_url"]}) - for c in conflicts - ): - return None - values = [ - series_stat(series, ("total", "sum", "max", "avg")) - for series in series_list - if series_filter is None or series_filter(series) - ] - values = [value for value in values if value is not None] - return sum(values) if values else None - - -def _finite(value: float | None) -> bool: - return value is not None and math.isfinite(value) - - -def _finite_sum(values: Iterable[float | None]) -> float | None: - total = 0.0 - found = False - for value in values: - if not _finite(value): - return None - total += value - found = True - return total if found else None - - -def _label_role(series: dict[str, Any], disaggregated: bool | None = None) -> str | None: - mode = label_value(series, "disaggregation_mode") - if mode: - role = _normalize_role(mode) - return role if role in KNOWN_ROLES else "unknown" - component = label_value(series, "dynamo_component") - if component == "backend" and disaggregated is False: - return "combined" - role = _normalize_role(component) if component else None - return role if role in KNOWN_ROLES else None + # Native and exporter-stripped names represent the same counter. + for name in (base, f"{base}_total"): + if name in metrics: + return sum_stat(metrics, name, series_filter=series_filter) + return None def _endpoint_role_sets( - metrics: dict[str, dict[str, Any]], disaggregated: bool | None = None + metrics: dict[str, dict[str, Any]], + disaggregated: bool | None = None, + declared: dict[str, str] | None = None, ) -> dict[str, set[str]]: explicit: dict[str, set[str]] = {} inferred: dict[str, set[str]] = {} @@ -424,28 +310,46 @@ def _endpoint_role_sets( continue series_list = entry.get("series") for series in series_list if isinstance(series_list, list) else []: - if not isinstance(series, dict) or not series.get("endpoint_url"): + if not isinstance(series, dict): continue - role = _label_role(series, disaggregated) - if role in KNOWN_ROLES: - target = explicit if label_value(series, "disaggregation_mode") else inferred - target.setdefault(str(series["endpoint_url"]), set()).add(role) - return {**inferred, **explicit} + endpoint = str(series.get("endpoint_url", "")) + labels = series.get("labels") + component = label_value(series, "dynamo_component") + if isinstance(labels, dict) and "disaggregation_mode" in labels: + mode = labels["disaggregation_mode"] + role = _normalize_role(mode) if isinstance(mode, str) and mode else "unknown" + explicit.setdefault(endpoint, set()).add(role) + elif component: + role = ( + "combined" + if component == "backend" and disaggregated is False + else _normalize_role(component) + ) + if role in ("prefill", "decode", "combined"): + inferred.setdefault(endpoint, set()).add(role) + roles = {**inferred, **explicit} + for endpoint, role in (declared or {}).items(): + roles.setdefault(endpoint, set()).add(_normalize_role(role)) + return roles def _series_role( series: dict[str, Any], roles: dict[str, set[str]], disaggregated: bool | None = None ) -> str: - endpoint_roles = roles.get(str(series.get("endpoint_url", "")), set()) - if label_value(series, "disaggregation_mode"): - return _label_role(series, disaggregated) or "unknown" - if len(endpoint_roles) == 1: - return next(iter(endpoint_roles)) - return _label_role(series, disaggregated) or "unknown" + labels = series.get("labels") + if isinstance(labels, dict) and "disaggregation_mode" in labels: + mode = labels["disaggregation_mode"] + return _normalize_role(mode) if isinstance(mode, str) and mode else "unknown" + found = roles.get(str(series.get("endpoint_url", "")), set()) + if len(found) == 1: + return next(iter(found)) + return "combined" if not found and disaggregated is False else "unknown" def _trtllm_sources( - metrics: dict[str, dict[str, Any]], *, reuse: dict[str, Any] | None = None + metrics: dict[str, dict[str, Any]], + paired: dict[str, tuple[int, int]], + disaggregated: bool | None, ) -> list[dict[str, Any]]: endpoints: set[str] = set() for entry in metrics.values(): @@ -459,9 +363,7 @@ def _trtllm_sources( endpoints.add(str(series["endpoint_url"])) roles = { endpoint: next(iter(found)) if len(found) == 1 else "unknown" - for endpoint, found in _endpoint_role_sets( - metrics, reuse.get("deployment_disaggregated") if reuse else None - ).items() + for endpoint, found in _endpoint_role_sets(metrics, disaggregated).items() } sources: list[dict[str, Any]] = [] @@ -480,12 +382,9 @@ def series_filter(series: dict, endpoint: str = endpoint) -> bool: series_filter=series_filter, ) ) - if hit_rate is None and reuse is not None and reuse["published"]: - domains = [d for d in reuse["domains"] if d["endpoint_url"] == endpoint] - if domains: - hit_rate = rate( - sum(d["cached"] for d in domains), sum(d["prompt"] for d in domains) - ) + if hit_rate is None and endpoint in paired: + prompt, cached = paired[endpoint] + hit_rate = rate(cached, prompt) sources.append( { "id": f"{roles.get(endpoint, 'worker')}|{endpoint}", @@ -511,6 +410,6 @@ def _normalize_role(mode: str) -> str: return "prefill" if lowered in ("decode", "generation", "gen", "backend"): return "decode" - if lowered in ("aggregated", "prefill_and_decode", "agg"): + if lowered in ("aggregated", "prefill_and_decode", "agg", "combined", "unified"): return "combined" return lowered diff --git a/inferencex-e2e/infx/results/agentic/cache_validity.py b/inferencex-e2e/infx/results/agentic/cache_validity.py index ba76bbb2b0..330183e0d2 100644 --- a/inferencex-e2e/infx/results/agentic/cache_validity.py +++ b/inferencex-e2e/infx/results/agentic/cache_validity.py @@ -1,64 +1,80 @@ from __future__ import annotations +from collections.abc import Iterable, Mapping from typing import Any CORE_SCHEMA = "infx-cache-validity-core/1" -STATUS_CLASSES = ( - "VALID_COMPLETE", - "VALID_ZERO", - "VALID_INTERVAL", - "PARTIAL", - "UNRESOLVED", - "CONTAMINATED", - "INVALID", - "MISSING", - "NOT_EVALUATED", -) _STATUS_CLASS = { "VALID_COMPLETE": "VALID_COMPLETE", "VALID_ZERO": "VALID_ZERO", + "CONDITIONAL_INTERVAL": "VALID_INTERVAL", "PARTIAL_COMPONENT": "PARTIAL", + "PARTIAL_COVERAGE": "PARTIAL", "WINDOW_UNRESOLVED": "UNRESOLVED", "ROLE_OR_COVERAGE_UNRESOLVED": "UNRESOLVED", "REQUEST_SCOPE_UNRESOLVED": "UNRESOLVED", "MISSING_OBSERVATION": "MISSING", "INVALID_COUNTER": "INVALID", - "CONDITIONAL_INTERVAL": "VALID_INTERVAL", - "UNKNOWN": "UNRESOLVED", - "BOUNDARY_RESIDUAL": "UNRESOLVED", - "BOUNDARY_UNCERTAIN": "UNRESOLVED", - "PARTIAL_COVERAGE": "PARTIAL", - "PHASE_CONTAMINATED": "CONTAMINATED", "REJECTED": "INVALID", + "PHASE_CONTAMINATED": "CONTAMINATED", "NOT_EVALUATED": "NOT_EVALUATED", } -_TIER_RATES = ("gpu_cache_hit_rate", "cpu_cache_hit_rate", "external_cache_hit_rate") + + +def modern_profile_error(export: Mapping[str, Any], endpoints: Iterable[str] = ()) -> str | None: + if ( + export.get("schema_version") != "1.1" + or export.get("aiperf_version") != "1.0.12" + or export.get("metrics_phase") != "profiling" + ): + return "AgentX 1.0.12 profiling export required" + summary = export.get("summary") + ranges = summary.get("phase_time_ranges") if isinstance(summary, dict) else None + window = ranges.get("profiling") if isinstance(ranges, dict) else None + if not isinstance(window, dict) or not ( + type(window.get("start_ns")) is int + and type(window.get("end_ns")) is int + and 0 <= window["start_ns"] < window["end_ns"] + ): + return "profiling time bounds missing or invalid" + for key in ("endpoints_configured", "endpoints_successful"): + covered = summary.get(key) + if ( + not isinstance(covered, list) + or not all(isinstance(url, str) for url in covered) + or not set(endpoints).issubset(covered) + ): + return "profiling endpoint coverage incomplete" + return None def summarize_cache_validity(nested: dict[str, Any]) -> dict[str, Any]: cache = nested["cache"] - tiers = any(cache.get(key) is not None for key in _TIER_RATES) + tiers = any( + cache.get(f"{tier}_cache_hit_rate") is not None for tier in ("gpu", "cpu", "external") + ) combined = cache.get("overall_cache_hit_rate") is not None - core = { - "core_schema": CORE_SCHEMA, - "published": tiers or combined, - "basis": "tiers" if tiers else "combined" if combined else None, - } detail = nested.get("cache_validity") - if isinstance(detail, dict): - return { - **detail, - **core, - "status_class": _STATUS_CLASS.get(detail.get("status"), "UNRESOLVED"), + if not isinstance(detail, dict): + rate = nested.get("validity", {}).get("cache_rate", {}) + detail = { + "status": rate.get("status", "NOT_EVALUATED"), + "pricing_eligible": bool(rate.get("eligible_exact_actual")), + "reasons": list(rate.get("reasons") or []), } - rate = nested["validity"]["cache_rate"] - present = bool(nested.get("present")) - status = rate.get("status") if present else "MISSING_OBSERVATION" - return { - "schema": CORE_SCHEMA, + if not nested.get("present"): + detail = {"status": "MISSING_OBSERVATION", "reasons": ["server_metrics_missing"]} + status = detail.get("status") + core = { + "schema": detail.get("schema", CORE_SCHEMA), + "core_schema": CORE_SCHEMA, "status": status, "status_class": _STATUS_CLASS.get(status, "UNRESOLVED"), - "pricing_eligible": present and bool(rate.get("eligible_exact_actual")), - "reasons": list(rate.get("reasons") or []) if present else ["server_metrics_missing"], - **core, + "published": tiers or combined, + "pricing_eligible": bool(detail.get("pricing_eligible")), + "basis": "tiers" if tiers else "combined" if combined else None, + "reasons": list(detail.get("reasons") or []), } + if isinstance(detail.get("request_scope"), dict): + core["request_scope"] = detail["request_scope"] + return core diff --git a/inferencex-e2e/infx/results/agentic/phase_validity.py b/inferencex-e2e/infx/results/agentic/phase_validity.py deleted file mode 100644 index a9eaf79731..0000000000 --- a/inferencex-e2e/infx/results/agentic/phase_validity.py +++ /dev/null @@ -1,371 +0,0 @@ -from __future__ import annotations - -import math -from collections import Counter -from collections.abc import Iterable, Mapping -from dataclasses import dataclass, field -from typing import Any - -from .common import to_float - -VALID = "VALID" -UNKNOWN = "UNKNOWN" -BOUNDARY_RESIDUAL = "BOUNDARY_RESIDUAL" -BOUNDARY_UNCERTAIN = "BOUNDARY_UNCERTAIN" -PARTIAL_COVERAGE = "PARTIAL_COVERAGE" -PHASE_CONTAMINATED = "PHASE_CONTAMINATED" -REJECTED = "REJECTED" -_SEVERITY = { - VALID: 0, - UNKNOWN: 1, - BOUNDARY_RESIDUAL: 2, - BOUNDARY_UNCERTAIN: 3, - PARTIAL_COVERAGE: 4, - PHASE_CONTAMINATED: 5, - REJECTED: 6, -} -# Rate built only from VALID series: an interval measurement, not a request-bound actual. -CONDITIONAL_INTERVAL = "CONDITIONAL_INTERVAL" - -NS_PER_S = 1_000_000_000 -COUNT_TOLERANCE = 0.5 -SUM_RELATIVE_TOLERANCE = 1e-6 - - -@dataclass(frozen=True) -class PhaseWindows: - profiling: tuple[int, int] - warmup: tuple[int, int] | None - - def as_dict(self) -> dict[str, list[int] | None]: - return { - "profiling": list(self.profiling), - "warmup": list(self.warmup) if self.warmup else None, - } - - -@dataclass(frozen=True) -class PhaseContext: - """What a backend needs to judge its own series; kept out of the published result. - - ``unscraped`` is None when the export does not say which endpoints were - configured and scraped. ``disaggregated`` is None when no authoritative - deployment mode was supplied. - """ - - windows: PhaseWindows | None - warmup_totals: Mapping[tuple, float | None] = field(default_factory=dict) - unscraped: tuple[str, ...] | None = () - disaggregated: bool | None = None - - def has_reference(self, metric: str, series: Mapping[str, Any]) -> bool: - return series_key(metric, series) in self.warmup_totals - - def warmup_total(self, metric: str, series: Mapping[str, Any]) -> float | None: - return self.warmup_totals.get(series_key(metric, series)) - - -def base_name(metric: str) -> str: - return metric.removesuffix("_total") - - -def series_key(metric: str, series: Mapping[str, Any]) -> tuple: - labels = series.get("labels") if isinstance(series.get("labels"), Mapping) else {} - return ( - base_name(metric), - str(series.get("endpoint_url", "")), - tuple(sorted((str(k), str(v)) for k, v in labels.items())), - ) - - -def phase_windows(server_metrics: Any) -> PhaseWindows | None: - summary = server_metrics.get("summary") if isinstance(server_metrics, Mapping) else None - ranges = summary.get("phase_time_ranges") if isinstance(summary, Mapping) else None - if not isinstance(ranges, Mapping): - return None - profiling = _time_range(ranges.get("profiling")) - if profiling is None: - return None - return PhaseWindows(profiling, _time_range(ranges.get("warmup"))) - - -def warmup_totals(server_metrics: Any) -> dict[tuple, float | None]: - """Warmup-phase delta of every series exported under ``warmup_metrics``. - - Presence means the series has a sample before its first PROFILING sample; a - zero total means no increment was counted during warmup. - """ - warmup = server_metrics.get("warmup_metrics") if isinstance(server_metrics, Mapping) else None - totals: dict[tuple, float | None] = {} - for name, entry in warmup.items() if isinstance(warmup, Mapping) else (): - if not isinstance(entry, Mapping): - continue - key_name = "count" if entry.get("type") == "histogram" else "total" - series_list = entry.get("series") - for series in series_list if isinstance(series_list, list) else (): - if isinstance(series, Mapping): - stats = series.get("stats") if isinstance(series.get("stats"), Mapping) else {} - totals[series_key(name, series)] = to_float(stats.get(key_name)) - return totals - - -def export_context(server_metrics: Any, windows: PhaseWindows | None) -> dict[str, Any]: - summary = server_metrics.get("summary") if isinstance(server_metrics, Mapping) else None - summary = summary if isinstance(summary, Mapping) else {} - configured = summary.get("endpoints_configured") - successful = summary.get("endpoints_successful") - if isinstance(configured, list) and isinstance(successful, list): - scraped = {str(url) for url in successful} - endpoints = { - "configured": len(configured), - "unscraped": [str(url) for url in configured if str(url) not in scraped], - } - else: - endpoints = {"configured": None, "unscraped": None} - return {"phase_windows": windows.as_dict() if windows else None, "endpoints": endpoints} - - -def worst(statuses: Iterable[str]) -> str: - return max(statuses, key=_SEVERITY.__getitem__, default=VALID) - - -def classify_series( - series: Mapping[str, Any], - metric_type: str, - windows: PhaseWindows | None, - *, - has_reference: bool = False, - warmup_total: float | None = None, -) -> dict[str, Any]: - histogram = metric_type == "histogram" - keys = ("sum", "count") if histogram else ("total",) - stats = series.get("stats") - stats = stats if isinstance(stats, Mapping) else {} - totals = {key: to_float(stats.get(key)) for key in keys} - raw_slices = series.get("timeslices") - slices = ( - [s for s in raw_slices if isinstance(s, Mapping)] if isinstance(raw_slices, list) else [] - ) - findings: list[tuple[str, str]] = [] - record: dict[str, Any] = { - "endpoint_url": series.get("endpoint_url"), - "total": totals[keys[0]], - "slice_sum": None, - "residual": None, - "first_slice_offset_s": None, - "last_slice_offset_s": None, - } - - for value in totals.values(): - if value is None: - findings.append((UNKNOWN, "missing_total")) - elif not math.isfinite(value): - findings.append((REJECTED, "nonfinite_total")) - elif value < 0: - findings.append((REJECTED, "negative_total")) - if windows is None: - findings.append((UNKNOWN, "phase_window_missing")) - if not slices: - findings.append((UNKNOWN, "no_timeslices")) - else: - findings.extend( - _slice_findings(slices, windows, record, histogram, has_reference, warmup_total) - ) - for key in keys: - slice_sum = _slice_sum(slices, key, histogram) - if slice_sum is None: - findings.append((REJECTED, "negative_or_nonfinite_slice")) - continue - total = totals[key] - if key == keys[0]: - record["slice_sum"] = slice_sum - if total is None or not math.isfinite(total): - continue - residual = total - slice_sum - if key == keys[0]: - record["residual"] = residual - tolerance = COUNT_TOLERANCE - if key == "sum": - tolerance = max(COUNT_TOLERANCE, SUM_RELATIVE_TOLERANCE * abs(total)) - if residual > tolerance: - findings.append((BOUNDARY_RESIDUAL, "total_exceeds_slice_sum")) - elif residual < -tolerance: - findings.append((REJECTED, "slice_sum_exceeds_total")) - - record["status"] = worst(status for status, _ in findings) - record["reasons"] = list(dict.fromkeys(code for _, code in findings)) - return record - - -def summarize( - metrics: Mapping[str, Any], - windows: PhaseWindows | None, - warmup: Mapping[tuple, float | None] | None = None, -) -> dict[str, Any]: - warmup = warmup or {} - series_by_status: Counter[str] = Counter() - metrics_by_status: dict[str, list[str]] = {} - for name in sorted(metrics): - entry = metrics[name] - if not isinstance(entry, Mapping) or entry.get("type") not in ("counter", "histogram"): - continue - series_list = entry.get("series") - if not isinstance(series_list, list): - continue - statuses = [ - classify_series( - series, - entry["type"], - windows, - has_reference=series_key(name, series) in warmup, - warmup_total=warmup.get(series_key(name, series)), - )["status"] - for series in series_list - if isinstance(series, Mapping) - ] - series_by_status.update(statuses) - status = worst(statuses) - if statuses and status != VALID: - metrics_by_status.setdefault(status, []).append(name) - return { - "series_by_status": dict(series_by_status), - "metrics_by_status": metrics_by_status, - } - - -def resolve_aliases( - metrics: Mapping[str, Any], base: str -) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """Series of ``base`` and its Prometheus ``_total`` alias, one per endpoint and label set. - - Cross-name aliases collapse only when they are the same observation (equal - ``stats`` and ``timeslices``); any other overlap is a conflict, whatever - the order. Repeated series under one name are reported as duplicates. Both - are dropped so they are never summed. - """ - groups: dict[tuple, list[tuple[str, Mapping[str, Any]]]] = {} - for name in (base, f"{base}_total"): - entry = metrics.get(name) - series_list = entry.get("series") if isinstance(entry, Mapping) else None - for series in series_list if isinstance(series_list, list) else []: - if isinstance(series, Mapping): - groups.setdefault(series_key(name, series), []).append((name, series)) - chosen: list[dict[str, Any]] = [] - conflicts: list[dict[str, Any]] = [] - for key, items in groups.items(): - names = [name for name, _ in items] - if len(items) == 1: - chosen.append(items[0][1]) - continue - kind = "duplicate_series" if len(set(names)) < len(names) else "alias_conflict" - first = items[0][1] - if kind == "alias_conflict" and all(_same_observation(first, s) for _, s in items[1:]): - chosen.append(first) - continue - conflicts.append( - {"metric": base, "endpoint_url": key[1], "names": sorted(names), "kind": kind} - ) - return chosen, conflicts - - -def _time_range(value: Any) -> tuple[int, int] | None: - if not isinstance(value, Mapping): - return None - start, end = value.get("start_ns"), value.get("end_ns") - if not isinstance(start, int) or not isinstance(end, int) or start >= end: - return None - return start, end - - -def _slice_findings( - slices: list[Mapping[str, Any]], - windows: PhaseWindows | None, - record: dict[str, Any], - histogram: bool, - has_reference: bool, - warmup_total: float | None, -) -> list[tuple[str, str]]: - bounds = [(s.get("start_ns"), s.get("end_ns")) for s in slices] - if not all(isinstance(a, int) and isinstance(b, int) for a, b in bounds): - return [(REJECTED, "slice_bounds_missing")] - findings: list[tuple[str, str]] = [] - if any(end <= start for start, end in bounds): - findings.append((REJECTED, "slice_nonpositive_duration")) - if any(bounds[i][0] < bounds[i - 1][1] for i in range(1, len(bounds))): - findings.append((REJECTED, "slice_overlap_or_backward")) - if any(bounds[i][0] > bounds[i - 1][1] for i in range(1, len(bounds))): - findings.append((REJECTED, "slice_gap")) - if findings or windows is None: - return findings - - start, end = windows.profiling - first, last = bounds[0][0], bounds[-1][1] - step = max(b - a for a, b in bounds) - activity = [_slice_activity(s, histogram) for s in slices] - before = sum(v for (a, b), v in zip(bounds, activity, strict=True) if b <= start) - straddle_start = sum(v for (a, b), v in zip(bounds, activity, strict=True) if a < start < b) - straddle_end = sum(v for (a, b), v in zip(bounds, activity, strict=True) if a < end < b) - after = sum(v for (a, b), v in zip(bounds, activity, strict=True) if a >= end) - record.update( - first_slice_offset_s=(first - start) / NS_PER_S, - last_slice_offset_s=(last - end) / NS_PER_S, - activity_before_profiling=before, - activity_straddling_start=straddle_start, - activity_straddling_end=straddle_end, - activity_after_profiling=after, - ) - # A window that opens before the phase holds PROFILING-tagged samples from - # before it. Increments between the last of those and the first in-phase - # sample land in whichever slice holds that later sample, so slice position - # cannot clear them; only a zero warmup-phase total can. - if first < start: - if before > 0 or (straddle_start > 0 and first <= start - step): - findings.append((PHASE_CONTAMINATED, "activity_before_profiling")) - elif warmup_total != 0 and first <= start - step: - findings.append((PHASE_CONTAMINATED, "window_opens_before_profiling")) - elif straddle_start > 0: - findings.append((BOUNDARY_UNCERTAIN, "activity_straddles_profiling_start")) - elif warmup_total != 0: - findings.append((BOUNDARY_UNCERTAIN, "window_opens_before_profiling")) - if after > 0: - findings.append((PHASE_CONTAMINATED, "activity_after_profiling")) - elif straddle_end > 0: - findings.append((BOUNDARY_UNCERTAIN, "activity_straddles_profiling_end")) - # A late first slice is covered only if a reference sample exists, so that - # stats.total spans the gap and the residual check sees any increment in it. - if first > start and not has_reference: - findings.append((PARTIAL_COVERAGE, "window_starts_after_profiling")) - if last < end: - findings.append((PARTIAL_COVERAGE, "window_ends_before_profiling")) - return findings - - -def _slice_activity(item: Mapping[str, Any], histogram: bool) -> float: - value = to_float(item.get("count" if histogram else "total")) - return value if value is not None and math.isfinite(value) and value > 0 else 0.0 - - -def _slice_sum(slices: list[Mapping[str, Any]], key: str, histogram: bool) -> float | None: - total = 0.0 - for item in slices: - if histogram: - count = to_float(item.get("count")) - value = count if key == "count" else _product(count, to_float(item.get("avg"))) - else: - value = to_float(item.get("total")) - if value is None or not math.isfinite(value) or value < 0: - return None - total += value - return total - - -def _product(count: float | None, avg: float | None) -> float | None: - if count is None or avg is None: - return None - return count * avg - - -def _same_observation(left: Mapping[str, Any], right: Mapping[str, Any]) -> bool: - return left.get("stats") == right.get("stats") and left.get("timeslices") == right.get( - "timeslices" - ) diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index ad34632a46..ce8167e973 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -14,7 +14,6 @@ ) from .cache_validity import summarize_cache_validity from .common import index_server_metrics -from .phase_validity import PhaseContext, export_context, phase_windows, summarize, warmup_totals def request_scope_from_records( @@ -57,17 +56,6 @@ def deployment_from_env(env: Mapping[str, str]) -> dict[str, Any]: } -def _profiling_window(server_metrics: dict[str, Any]) -> tuple[int, int] | None: - summary = server_metrics.get("summary") - ranges = summary.get("phase_time_ranges") if isinstance(summary, dict) else None - profiling = ranges.get("profiling") if isinstance(ranges, dict) else None - if isinstance(profiling, dict): - start, end = profiling.get("start_ns"), profiling.get("end_ns") - if type(start) is int and type(end) is int and start < end: - return start, end - return None - - def endpoint_roles_from_env(env: Mapping[str, str]) -> dict[str, str] | None: if env.get("SRTCTL_FRONTEND_TYPE") == "dynamo": return {} @@ -167,18 +155,7 @@ def compute_server_metrics( deployment = {**deployment_from_env(source_env), **dict(deployment or {})} if disaggregated is None: disaggregated = deployment.get("disagg") - windows = phase_windows(server_metrics) - exported = export_context(server_metrics, windows) - unscraped = exported["endpoints"]["unscraped"] - context = PhaseContext( - windows=windows, - warmup_totals=warmup_totals(server_metrics), - unscraped=None if unscraped is None else tuple(unscraped), - disaggregated=disaggregated, - ) - nested["validity"].update(exported) nested["validity"]["deployment"] = {"disaggregated": disaggregated} - nested["validity"].update(summarize(metrics, windows, context.warmup_totals)) adapter = detect_backend(metrics, framework) if not metrics: @@ -204,7 +181,6 @@ def compute_server_metrics( ) nested["adapter"] = adapter.name if getattr(adapter, "accepts_context", False): - warmup = server_metrics.get("warmup_metrics") adapter.populate( metrics, flat, @@ -214,17 +190,15 @@ def compute_server_metrics( "endpoint_roles": endpoint_roles if endpoint_roles is not None else endpoint_roles_from_env(source_env), - "profiling_window": _profiling_window(server_metrics), - "warmup_metrics": warmup if isinstance(warmup, dict) else None, "request_scope": request_scope, **deployment, + "disagg": disaggregated, "collector_export": server_metrics, "warnings": warnings, }, ) else: adapter.populate(metrics, flat, nested) - adapter.apply_validity(metrics, flat, nested, context) if framework.lower() == "dynamo-sglang" and adapter.name == "sglang": nested["kv_cache"]["gpu_total_tokens"] = None diff --git a/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py b/inferencex-e2e/infx/tests/results/agentic/test_disaggregated_prompt_totals.py similarity index 65% rename from inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py rename to inferencex-e2e/infx/tests/results/agentic/test_disaggregated_prompt_totals.py index 621d7e986c..4242da3e99 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_cache_validity_core.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_disaggregated_prompt_totals.py @@ -3,10 +3,6 @@ import pytest from infx.results.agentic.server_metrics import compute_server_metrics -from infx.tests.results.agentic.test_atom_cache_validity import _export as _atom_export -from infx.tests.results.agentic.test_atom_cache_validity import _run as _run_atom -from infx.tests.results.agentic.test_server_metrics_closeout import _pair -from infx.tests.results.agentic.test_server_metrics_closeout import _run as _run_trt def _counter(value: float) -> dict: @@ -22,68 +18,6 @@ def _counter(value: float) -> dict: } -class TestCacheValidityCore: - def test_atom_detail_maps_complete_partial_and_invalid_states(self) -> None: - complete = _run_atom(_atom_export())["cache_validity"] - export = _atom_export() - del export["metrics"]["atom:prefix_cache_offload_tokens"] - partial = _run_atom(export)["cache_validity"] - export = _atom_export() - cached = export["metrics"]["atom:prefix_cache_cached_tokens"]["series"][0] - cached["stats"]["total"] = 1300 - cached["timeslices"][1]["total"] = 1300 - invalid = _run_atom(export)["cache_validity"] - - assert complete["status_class"] == "VALID_COMPLETE" - assert complete["published"] is True - assert complete["basis"] == "tiers" - assert partial["status_class"] == "PARTIAL" - assert partial["published"] is True - assert partial["pricing_eligible"] is False - assert invalid["status_class"] == "INVALID" - assert invalid["published"] is False - - def test_trt_interval_telemetry_is_published_but_not_exact(self) -> None: - core = _run_trt(_pair())["cache_validity"] - - assert core["status"] == "CONDITIONAL_INTERVAL" - assert core["status_class"] == "VALID_INTERVAL" - assert core["published"] is True - assert core["pricing_eligible"] is False - assert core["basis"] == "combined" - - def test_trt_rejected_rate_is_not_published(self) -> None: - core = _run_trt(_pair(cached=1200.0))["cache_validity"] - - assert core["status_class"] == "INVALID" - assert core["published"] is False - assert core["basis"] is None - assert "cached_exceeds_prompt" in core["reasons"] - - def test_adapter_without_assessment_is_not_evaluated_on_its_tiers(self) -> None: - export = { - "metrics": { - "vllm:prefix_cache_hits": _counter(600.0), - "vllm:prefix_cache_queries": _counter(1000.0), - } - } - _, nested, _ = compute_server_metrics(export, framework="vllm", records=[]) - core = nested["cache_validity"] - - assert core["status_class"] == "NOT_EVALUATED" - assert core["published"] is True - assert core["pricing_eligible"] is False - assert core["basis"] == "tiers" - - def test_missing_server_metrics_is_missing(self) -> None: - _, nested, _ = compute_server_metrics({}, framework="vllm", records=[]) - core = nested["cache_validity"] - - assert core["status_class"] == "MISSING" - assert core["published"] is False - assert core["reasons"] == ["server_metrics_missing"] - - class TestDisaggregatedPromptTotals: @pytest.mark.parametrize( "framework, metric", diff --git a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py index f31d4f0ee8..132268ca7f 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py @@ -1567,10 +1567,11 @@ def test_processor_normalizes_trtllm_server_metrics(tmp_path: Path): assert agg["server_metrics"]["cache"]["overall_cache_hit_rate"] is None validity = agg["server_metrics"]["validity"] assert validity["cache_rate"]["status"] == "UNKNOWN" - assert "phase_window_missing" in validity["cache_rate"]["reasons"] - gauge = validity["diagnostics"]["trtllm_kv_cache_hit_rate"] - assert gauge["all_series_avg"] == pytest.approx(0.5) - assert gauge["by_role"] == pytest.approx({"prefill": 0.4, "decode": 0.6}) + assert agg["server_metrics"]["cache_validity"]["pricing_eligible"] is False + assert {source["prefix_cache_hit_rate"] for source in agg["server_metrics"]["sources"]} == { + 0.4, + 0.6, + } assert agg["server_metrics"]["kv_cache"]["gpu_usage_pct"] == pytest.approx(0.8) assert agg["server_metrics"]["kv_cache"]["cpu_usage_pct"] == pytest.approx(0.25) assert agg["server_metrics"]["kv_cache"]["gpu_total_tokens"] == 19_200 diff --git a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py deleted file mode 100644 index f962dd9d75..0000000000 --- a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_closeout.py +++ /dev/null @@ -1,484 +0,0 @@ -from __future__ import annotations - -import copy - -import pytest - -from infx.results.agentic.server_metrics import compute_server_metrics - -NS = 1_000_000_000 -START = 1_000 * NS -END = START + 60 * NS -PRE = ["http://10.0.0.2:7500/metrics", "http://10.0.0.3:7501/metrics"] -DEC = "http://10.0.0.4:7502/metrics" -UNLABELLED = "http://10.0.0.9:7509/metrics" - - -def _slices(total: float, start_ns: int = START, end_ns: int = END) -> list[dict]: - bounds = list(range(start_ns, end_ns, NS)) - per = total / len(bounds) - return [ - {"start_ns": s, "end_ns": min(s + NS, end_ns), "total": per, "rate": per} - for s in bounds - ] - - -def _series( - endpoint: str, - role: str | None, - total: float, - *, - start_ns: int = START, - end_ns: int = END, - labels: dict | None = None, - slices: list[dict] | None = None, -) -> dict: - series_labels = {"model_name": "m", **(labels or {})} - if role is not None: - series_labels["dynamo_component"] = role - return { - "endpoint_url": endpoint, - "labels": series_labels, - "stats": {"total": total}, - "timeslices": slices if slices is not None else _slices(total, start_ns, end_ns), - } - - -def _export(prompt: list[dict], cached: list[dict], *, endpoints: bool = True) -> dict: - summary: dict = { - "phase_time_ranges": { - "profiling": {"start_ns": START, "end_ns": END}, - "warmup": {"start_ns": START - 120 * NS, "end_ns": START}, - } - } - if endpoints: - urls = sorted({s["endpoint_url"] for s in prompt + cached}) - summary["endpoints_configured"] = urls - summary["endpoints_successful"] = list(urls) - return { - "summary": summary, - "metrics": { - "trtllm_prompt_tokens": {"type": "counter", "series": prompt}, - "trtllm_prompt_cached_tokens": {"type": "counter", "series": cached}, - }, - } - - -def _with_warmup(export: dict, total: float) -> dict: - """Add the same series to ``warmup_metrics`` with the given warmup-phase total.""" - export["warmup_metrics"] = { - name: { - "type": "counter", - "series": [ - _series( - s["endpoint_url"], - s["labels"].get("dynamo_component"), - total, - start_ns=START - 120 * NS, - end_ns=START, - labels={k: v for k, v in s["labels"].items() if k != "dynamo_component"}, - ) - for s in entry["series"] - ], - } - for name, entry in export["metrics"].items() - } - return export - - -def _run(export: dict, **kwargs) -> dict: - _, nested, _ = compute_server_metrics( - export, framework="dynamo-trt", records=[], **kwargs - ) - return nested - - -def _rate(nested: dict) -> dict: - return nested["validity"]["cache_rate"] - - -def _pair(prompt: float = 1000.0, cached: float = 900.0, **kwargs) -> dict: - return _export( - [_series(PRE[0], "prefill", prompt, **kwargs)], - [_series(PRE[0], "prefill", cached, **kwargs)], - ) - - -class TestPairedCounterBounds: - def test_cached_above_prompt_is_rejected_and_not_published(self) -> None: - nested = _run(_pair(cached=1200.0)) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert nested["tokens"]["prompt_by_source"]["computed"] is None - assert _rate(nested)["status"] == "REJECTED" - assert "cached_exceeds_prompt" in _rate(nested)["reasons"] - assert nested["sources"][0]["prefix_cache_hit_rate"] is None - assert _rate(nested)["raw_rate"] == pytest.approx(1.2) - - def test_one_domain_above_prompt_is_not_offset_by_the_sum(self) -> None: - export = _export( - [_series(PRE[0], "prefill", 1000.0), _series(PRE[1], "prefill", 3000.0)], - [_series(PRE[0], "prefill", 1200.0), _series(PRE[1], "prefill", 1000.0)], - ) - - nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert "cached_exceeds_prompt" in _rate(nested)["reasons"] - - def test_cached_in_a_domain_without_prompt_is_rejected(self) -> None: - export = _export( - [_series(PRE[0], "prefill", 1000.0), _series(PRE[1], "prefill", 0.0)], - [_series(PRE[0], "prefill", 900.0), _series(PRE[1], "prefill", 5.0)], - ) - - nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert "cached_exceeds_prompt" in _rate(nested)["reasons"] - - @pytest.mark.parametrize(("cached", "expected"), [(0.0, 0.0), (1000.0, 1.0)]) - def test_zero_and_full_reuse_stay_publishable(self, cached, expected) -> None: - nested = _run(_pair(cached=cached)) - - assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(expected) - assert _rate(nested)["status"] == "CONDITIONAL_INTERVAL" - assert nested["tokens"]["prompt_by_source"]["computed"] == pytest.approx( - 1000.0 - cached - ) - - def test_idle_prefill_domain_does_not_block(self) -> None: - export = _export( - [_series(PRE[0], "prefill", 1000.0), _series(PRE[1], "prefill", 0.0)], - [_series(PRE[0], "prefill", 900.0), _series(PRE[1], "prefill", 0.0)], - ) - - nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) - - -class TestDeploymentRolesAndCounterPairs: - @pytest.mark.parametrize( - "explicit_mode, expected", [(None, 0.9), ("decode", None), ("garbage", None)] - ) - def test_aggregated_backend_component_does_not_override_explicit_mode( - self, explicit_mode, expected - ) -> None: - labels = {"disaggregation_mode": explicit_mode} if explicit_mode else {} - export = _export( - [_series(PRE[0], "backend", 1000.0, labels=labels)], - [_series(PRE[0], "backend", 900.0, labels=labels)], - ) - nested = _run(export, disaggregated=False) - - assert nested["cache"]["overall_cache_hit_rate"] == expected - assert nested["cache_validity"]["pricing_eligible"] is False - assert nested["sources"][0]["prefix_cache_hit_rate"] == expected - - def test_non_role_components_do_not_conflict_with_explicit_worker_roles(self) -> None: - export = _export( - [ - _series(PRE[0], None, 1000.0, labels={"disaggregation_mode": "prefill"}), - _series(DEC, None, 1000.0, labels={"disaggregation_mode": "decode"}), - ], - [ - _series(PRE[0], None, 900.0, labels={"disaggregation_mode": "prefill"}), - _series(DEC, None, 1000.0, labels={"disaggregation_mode": "decode"}), - ], - ) - export["metrics"]["other_metric"] = { - "type": "gauge", - "series": [_series(PRE[0], "tensorrt_llm", 1.0), _series(DEC, "tensorrt_llm", 1.0)], - } - nested = _run(export, disaggregated=True) - - assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) - assert _rate(nested)["role_scope"] == "prefill" - assert {s["endpoint_url"] for s in _rate(nested)["excluded"]} == {DEC} - - def test_decode_only_is_not_treated_as_combined(self) -> None: - export = _export( - [_series(DEC, "backend", 1000.0)], [_series(DEC, "backend", 1000.0)] - ) - - nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _rate(nested)["role_scope"] != "combined" - assert "decode_only_no_prefill_scope" in _rate(nested)["reasons"] - - def test_unknown_role_beside_prefill_blocks_instead_of_being_excluded(self) -> None: - export = _export( - [_series(PRE[0], "prefill", 1000.0), _series(UNLABELLED, None, 1000.0)], - [_series(PRE[0], "prefill", 900.0), _series(UNLABELLED, None, 0.0)], - ) - - nested = _run(export) - - rate = _rate(nested) - assert nested["cache"]["overall_cache_hit_rate"] is None - assert "role_unknown" in rate["reasons"] - assert {s["endpoint_url"] for s in rate["unresolved"]} == {UNLABELLED} - assert UNLABELLED not in {s["endpoint_url"] for s in rate["excluded"]} - - def test_unlabelled_series_need_an_authoritative_deployment_mode(self) -> None: - export = _export( - [_series(PRE[0], None, 1000.0)], [_series(PRE[0], None, 900.0)] - ) - - unknown = _run(copy.deepcopy(export)) - unified = _run(copy.deepcopy(export), disaggregated=False) - - assert unknown["cache"]["overall_cache_hit_rate"] is None - assert "deployment_mode_unknown" in _rate(unknown)["reasons"] - assert unified["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) - assert _rate(unified)["role_scope"] == "combined" - - def test_disaggregated_deployment_without_prefill_series_publishes_nothing( - self, - ) -> None: - export = _export( - [_series(PRE[0], None, 1000.0)], [_series(PRE[0], None, 900.0)] - ) - - nested = _run(export, disaggregated=True) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert "no_prefill_scope" in _rate(nested)["reasons"] - - def test_role_labels_contradicting_the_deployment_block(self) -> None: - nested = _run(_pair(), disaggregated=False) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert "role_conflicts_with_deployment" in _rate(nested)["reasons"] - - def test_same_endpoint_with_different_rank_is_not_a_pair(self) -> None: - export = _export( - [_series(PRE[0], "prefill", 1000.0, labels={"rank": "0"})], - [_series(PRE[0], "prefill", 900.0, labels={"rank": "1"})], - ) - - nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert "missing_pair" in _rate(nested)["reasons"] - - def test_instance_order_does_not_change_the_result(self) -> None: - prompt = [ - _series(PRE[0], "prefill", 1000.0), - _series(PRE[1], "prefill", 3000.0), - _series(DEC, "backend", 4000.0), - ] - cached = [ - _series(PRE[0], "prefill", 900.0), - _series(PRE[1], "prefill", 2100.0), - _series(DEC, "backend", 4000.0), - ] - - forward = _run(_export(prompt, cached)) - backward = _run(_export(prompt[::-1], cached[::-1])) - - assert forward["cache"] == backward["cache"] - assert forward["cache"]["overall_cache_hit_rate"] == pytest.approx(0.75) - assert _rate(forward)["status"] == _rate(backward)["status"] - - def test_missing_scrape_metadata_is_not_full_coverage(self) -> None: - export = _export( - [_series(PRE[0], "prefill", 1000.0)], - [_series(PRE[0], "prefill", 900.0)], - endpoints=False, - ) - - nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert "scrape_coverage_unknown" in _rate(nested)["reasons"] - - -class TestCounterAliases: - def test_equal_total_aliases_with_different_windows_conflict_in_either_order( - self, - ) -> None: - clean = _series(PRE[0], "prefill", 1000.0) - early = _series(PRE[0], "prefill", 1000.0, start_ns=START - 120 * NS) - outcomes = [] - for first, second in ((clean, early), (early, clean)): - export = _export([copy.deepcopy(first)], [_series(PRE[0], "prefill", 900.0)]) - export["metrics"]["trtllm_prompt_tokens_total"] = { - "type": "counter", - "series": [copy.deepcopy(second)], - } - nested = _run(export) - outcomes.append( - ( - nested["cache"]["overall_cache_hit_rate"], - _rate(nested)["status"], - "alias_conflict" in _rate(nested)["reasons"], - ) - ) - - assert outcomes[0] == outcomes[1] == (None, "REJECTED", True) - - def test_duplicate_series_under_one_name_is_not_an_alias(self) -> None: - export = _export( - [_series(PRE[0], "prefill", 1000.0), _series(PRE[0], "prefill", 1000.0)], - [_series(PRE[0], "prefill", 900.0)], - ) - - nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert "duplicate_series" in _rate(nested)["reasons"] - assert "alias_conflict" not in _rate(nested)["reasons"] - - -class TestProfilingWindowEvidence: - def test_activity_after_profiling_end_is_not_valid(self) -> None: - nested = _run(_pair(end_ns=END + 100 * NS)) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _rate(nested)["status"] == "PHASE_CONTAMINATED" - assert "activity_after_profiling" in _rate(nested)["reasons"] - - def test_nonpositive_duration_slice_is_rejected(self) -> None: - def backward(total: float) -> list[dict]: - return [ - {"start_ns": START, "end_ns": START - NS, "total": total / 2}, - {"start_ns": START - NS, "end_ns": END, "total": total / 2}, - ] - - export = _export( - [_series(PRE[0], "prefill", 1000.0, slices=backward(1000.0))], - [_series(PRE[0], "prefill", 900.0, slices=backward(900.0))], - ) - - nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _rate(nested)["status"] == "REJECTED" - assert "slice_nonpositive_duration" in _rate(nested)["reasons"] - - def test_activity_one_second_before_profiling_is_not_valid(self) -> None: - nested = _run(_pair(start_ns=START - NS)) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _rate(nested)["status"] == "PHASE_CONTAMINATED" - assert "activity_before_profiling" in _rate(nested)["reasons"] - - def test_activity_in_a_slice_straddling_the_start_is_uncertain(self) -> None: - nested = _run(_pair(start_ns=START - NS // 2)) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _rate(nested)["status"] == "BOUNDARY_UNCERTAIN" - assert "activity_straddles_profiling_start" in _rate(nested)["reasons"] - - def test_idle_slices_outside_the_window_need_zero_warmup_evidence(self) -> None: - def padded(total: float) -> list[dict]: - inside = _slices(total) - before = [ - {"start_ns": START - k * NS, "end_ns": START - (k - 1) * NS, "total": 0.0} - for k in (2, 1) - ] - after = [ - {"start_ns": END + k * NS, "end_ns": END + (k + 1) * NS, "total": 0.0} - for k in (0, 1) - ] - return before + inside + after - - export = _export( - [_series(PRE[0], "prefill", 1000.0, slices=padded(1000.0))], - [_series(PRE[0], "prefill", 900.0, slices=padded(900.0))], - ) - with_zero_warmup = _with_warmup(copy.deepcopy(export), 0.0) - - without = _run(export) - idle_warmup = _run(with_zero_warmup) - - assert without["cache"]["overall_cache_hit_rate"] is None - assert "window_opens_before_profiling" in _rate(without)["reasons"] - assert idle_warmup["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) - assert _rate(idle_warmup)["status"] == "CONDITIONAL_INTERVAL" - - def test_warmup_jump_landing_after_the_start_is_not_hidden(self) -> None: - def early_with_jump(profiling_total: float, warmup_total: float) -> list[dict]: - flat = [ - {"start_ns": START - k * NS, "end_ns": START - (k - 1) * NS, "total": 0.0} - for k in range(120, 0, -1) - ] - inside = _slices(profiling_total) - inside[1]["total"] += warmup_total - return flat + inside - - export = _export( - [_series(PRE[0], "prefill", 1500.0, slices=early_with_jump(1000.0, 500.0))], - [_series(PRE[0], "prefill", 1350.0, slices=early_with_jump(900.0, 450.0))], - ) - - nested = _run(_with_warmup(export, 500.0)) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _rate(nested)["status"] == "PHASE_CONTAMINATED" - assert "window_opens_before_profiling" in _rate(nested)["reasons"] - - def test_window_opening_within_one_slice_needs_zero_warmup_evidence(self) -> None: - def early_by_half(total: float) -> list[dict]: - first = {"start_ns": START - NS // 2, "end_ns": START + NS // 2, "total": 0.0} - rest = _slices(total, START + NS // 2, END) - return [first] + rest - - export = _export( - [_series(PRE[0], "prefill", 1000.0, slices=early_by_half(1000.0))], - [_series(PRE[0], "prefill", 900.0, slices=early_by_half(900.0))], - ) - - busy_warmup = _run(_with_warmup(copy.deepcopy(export), 50.0)) - idle_warmup = _run(_with_warmup(copy.deepcopy(export), 0.0)) - - assert busy_warmup["cache"]["overall_cache_hit_rate"] is None - assert _rate(busy_warmup)["status"] == "BOUNDARY_UNCERTAIN" - assert "window_opens_before_profiling" in _rate(busy_warmup)["reasons"] - assert idle_warmup["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) - - def test_late_start_needs_warmup_reference_evidence(self) -> None: - late = _pair(start_ns=START + NS // 5) - with_reference = _with_warmup(copy.deepcopy(late), 50.0) - - without = _run(late) - with_ref = _run(with_reference) - - assert without["cache"]["overall_cache_hit_rate"] is None - assert "window_starts_after_profiling" in _rate(without)["reasons"] - assert with_ref["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) - - def test_numerator_and_denominator_need_the_same_window(self) -> None: - export = _export( - [_series(PRE[0], "prefill", 1000.0)], - [_series(PRE[0], "prefill", 900.0, end_ns=END - 30 * NS)], - ) - - nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert {"window_ends_before_profiling", "window_support_mismatch"} <= set( - _rate(nested)["reasons"] - ) - - -class TestTelemetryVersusEligibility: - @pytest.mark.parametrize( - ("export", "status", "published"), - [ - (_pair(), "CONDITIONAL_INTERVAL", True), - (_pair(cached=1200.0), "REJECTED", False), - ], - ) - def test_publication_follows_status_and_exact_stays_false( - self, export, status, published - ) -> None: - nested = _run(copy.deepcopy(export)) - - rate = _rate(nested) - assert (rate["status"], rate["published"]) == (status, published) - assert rate["eligible_exact_actual"] is False - assert rate["cached_tokens"] is not None and rate["prompt_tokens"] is not None diff --git a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_phase_validity.py b/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_phase_validity.py deleted file mode 100644 index d5cfc1194d..0000000000 --- a/inferencex-e2e/infx/tests/results/agentic/test_server_metrics_phase_validity.py +++ /dev/null @@ -1,402 +0,0 @@ -from __future__ import annotations - -import copy -import math - -import pytest - -from infx.results.agentic.server_metrics import compute_server_metrics - -NS = 1_000_000_000 -P0 = 1_000 * NS -PROFILING = (P0, P0 + 60 * NS) -WARMUP = (P0 - 120 * NS, P0 + int(0.3 * NS)) -FRONTEND = "http://10.0.0.1:8000/metrics" -PREFILL = ["http://10.0.0.2:7500/metrics", "http://10.0.0.3:7501/metrics"] -DECODE = ["http://10.0.0.4:7502/metrics", "http://10.0.0.5:7503/metrics"] -PROMPT = {PREFILL[0]: 1000.0, PREFILL[1]: 2000.0, DECODE[0]: 1000.0, DECODE[1]: 2000.0} -CACHED = {PREFILL[0]: 900.0, PREFILL[1]: 1800.0, DECODE[0]: 1000.0, DECODE[1]: 2000.0} - - -def _role(endpoint: str) -> str: - return "prefill" if endpoint in PREFILL else "backend" - - -def _slices(start_s: float, end_s: float, total: float) -> list[dict]: - count = int(round(end_s - start_s)) - first = P0 + round(start_s * NS) - per = total / count - return [ - { - "start_ns": first + i * NS, - "end_ns": first + (i + 1) * NS, - "total": per, - "rate": per, - } - for i in range(count) - ] - - -def _counter( - endpoint: str, - total: float, - *, - start_s: float = 0.0, - end_s: float = 60.0, - slice_sum: float | None = None, - timeslices: bool = True, -) -> dict: - series = { - "endpoint_url": endpoint, - "labels": {"dynamo_component": _role(endpoint), "model_name": "m"}, - "stats": {"total": total}, - } - if timeslices: - series["timeslices"] = _slices( - start_s, end_s, total if slice_sum is None else slice_sum - ) - return series - - -def _histogram(endpoint: str, total: float, count: int, *, start_s: float = 0.0): - slices = [] - items = _slices(start_s, 60.0, total) - per_count = count / len(items) - for item in items: - slices.append( - { - "start_ns": item["start_ns"], - "end_ns": item["end_ns"], - "count": per_count, - "avg": item["total"] / per_count, - } - ) - return { - "endpoint_url": endpoint, - "labels": {"dynamo_namespace": "dynamo"}, - "stats": {"sum": total, "count": count}, - "timeslices": slices, - } - - -def _export( - *, - names: tuple[str, ...] = ("",), - prompt: dict[str, float] | None = None, - cached: dict[str, float] | None = None, - gauge: bool = False, - phase_time_ranges: bool = True, -) -> dict: - prompt = PROMPT if prompt is None else prompt - cached = CACHED if cached is None else cached - metrics: dict = { - "dynamo_frontend_input_sequence_tokens": { - "type": "histogram", - "series": [_histogram(FRONTEND, 3000.0, 60)], - } - } - for suffix in names: - metrics[f"trtllm_prompt_tokens{suffix}"] = { - "type": "counter", - "series": [_counter(e, v) for e, v in prompt.items()], - } - metrics[f"trtllm_prompt_cached_tokens{suffix}"] = { - "type": "counter", - "series": [_counter(e, v) for e, v in cached.items()], - } - if gauge: - metrics["trtllm_kv_cache_hit_rate"] = { - "type": "gauge", - "series": [ - { - "endpoint_url": e, - "labels": {"dynamo_component": _role(e)}, - "stats": {"avg": 0.93 if e in PREFILL else 0.0}, - } - for e in PREFILL + DECODE - ], - } - endpoints = [FRONTEND, *PREFILL, *DECODE] - summary: dict = { - "endpoints_configured": endpoints, - "endpoints_successful": list(endpoints), - } - if phase_time_ranges: - summary["phase_time_ranges"] = { - "profiling": {"start_ns": PROFILING[0], "end_ns": PROFILING[1]}, - "warmup": {"start_ns": WARMUP[0], "end_ns": WARMUP[1]}, - } - return {"summary": summary, "metrics": metrics} - - -def _run(export: dict, framework: str = "dynamo-trt") -> tuple[dict, dict]: - flat, nested, _ = compute_server_metrics(export, framework=framework, records=[]) - return flat, nested - - -def _cache_rate(nested: dict) -> dict: - return nested["validity"]["cache_rate"] - - -class TestTrtCounterNames: - def test_stripped_counter_names_feed_the_prefill_reuse_rate(self) -> None: - _, nested = _run(_export()) - - assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) - assert _cache_rate(nested)["status"] == "CONDITIONAL_INTERVAL" - assert _cache_rate(nested)["role_scope"] == "prefill" - - def test_native_total_names_are_still_accepted(self) -> None: - _, nested = _run(_export(names=("_total",))) - - assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) - - def test_alias_pairs_with_equal_values_count_once(self) -> None: - _, nested = _run(_export(names=("", "_total"))) - - assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) - assert _cache_rate(nested)["cached_tokens"] == pytest.approx(2700.0) - assert _cache_rate(nested)["prompt_tokens"] == pytest.approx(3000.0) - - def test_alias_conflict_rejects_the_rate(self) -> None: - export = _export(names=("", "_total")) - native = export["metrics"]["trtllm_prompt_cached_tokens_total"]["series"][0] - native["stats"]["total"] = 950.0 - - _, nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _cache_rate(nested)["status"] == "REJECTED" - assert "alias_conflict" in _cache_rate(nested)["reasons"] - - -class TestTrtRoleAndTier: - def test_decode_transfer_is_excluded_and_reported(self) -> None: - _, nested = _run(_export()) - - excluded = { - (item["metric"], item["endpoint_url"], item["reason"]) - for item in _cache_rate(nested)["excluded"] - } - assert excluded == { - (metric, endpoint, "decode_role_excluded") - for metric in ("trtllm_prompt_tokens", "trtllm_prompt_cached_tokens") - for endpoint in DECODE - } - - def test_role_mixed_gauge_is_diagnostic_only(self) -> None: - _, nested = _run(_export(gauge=True)) - - assert nested["cache"]["gpu_cache_hit_rate"] is None - diagnostic = nested["validity"]["diagnostics"]["trtllm_kv_cache_hit_rate"] - assert diagnostic["status"] == "GAUGE_DERIVED" - assert diagnostic["all_series_avg"] == pytest.approx(0.465) - assert diagnostic["by_role"] == pytest.approx({"prefill": 0.93, "decode": 0.0}) - assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.9) - - def test_combined_cached_is_not_presented_as_gpu_tier(self) -> None: - _, nested = _run(_export()) - - assert nested["cache"]["cached_tokens_by_source"] == {"combined": 2700.0} - assert _cache_rate(nested)["tier"] == "combined" - prompt_by_source = nested["tokens"]["prompt_by_source"] - assert prompt_by_source["gpu_cache_hit"] is None - assert prompt_by_source["computed"] == pytest.approx(300.0) - assert prompt_by_source["raw"] == {"combined": 2700.0} - - def test_rate_is_never_marked_exact_without_request_binding(self) -> None: - _, nested = _run(_export()) - - assert _cache_rate(nested)["eligible_exact_actual"] is False - assert {"request_scope_not_bound", "expected_instances_not_verified"} <= set( - _cache_rate(nested)["reasons"] - ) - - -class TestTrtPhaseValidity: - def test_contaminated_cached_series_blocks_rate_and_keeps_telemetry(self) -> None: - export = _export() - export["metrics"]["trtllm_prompt_cached_tokens"]["series"][0] = _counter( - PREFILL[0], 4900.0, start_s=-120.0 - ) - - _, nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert nested["cache"]["cached_tokens_by_source"] == {} - rate = _cache_rate(nested) - assert rate["status"] == "REJECTED" - assert {"activity_before_profiling", "cached_exceeds_prompt"} <= set(rate["reasons"]) - assert rate["cached_tokens"] == pytest.approx(6700.0) - assert rate["raw_rate"] == pytest.approx(6700.0 / 3000.0) - contaminated = [s for s in rate["series"] if s["status"] == "PHASE_CONTAMINATED"] - assert [(s["endpoint_url"], s["reasons"]) for s in contaminated] == [ - (PREFILL[0], ["activity_before_profiling"]) - ] - assert contaminated[0]["first_slice_offset_s"] == pytest.approx(-120.0) - - def test_total_above_slice_sum_is_a_boundary_residual(self) -> None: - export = _export() - export["metrics"]["trtllm_prompt_tokens"]["series"][0] = _counter( - PREFILL[0], 1098.0, slice_sum=1000.0 - ) - - _, nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - rate = _cache_rate(nested) - assert rate["status"] == "BOUNDARY_RESIDUAL" - residual = [s for s in rate["series"] if s["status"] == "BOUNDARY_RESIDUAL"] - assert residual[0]["residual"] == pytest.approx(98.0) - assert residual[0]["reasons"] == ["total_exceeds_slice_sum"] - - def test_slice_sum_above_total_is_a_suspected_reset(self) -> None: - export = _export() - export["metrics"]["trtllm_prompt_cached_tokens"]["series"][1] = _counter( - PREFILL[1], 0.0, slice_sum=1800.0 - ) - - _, nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _cache_rate(nested)["status"] == "REJECTED" - assert "slice_sum_exceeds_total" in _cache_rate(nested)["reasons"] - - @pytest.mark.parametrize( - ("total", "reason"), - [(-5.0, "negative_total"), (math.nan, "nonfinite_total"), (math.inf, "nonfinite_total")], - ) - def test_negative_and_nonfinite_counters_are_rejected(self, total, reason) -> None: - export = _export() - series = export["metrics"]["trtllm_prompt_cached_tokens"]["series"][0] - series["stats"]["total"] = total - - _, nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _cache_rate(nested)["status"] == "REJECTED" - assert reason in _cache_rate(nested)["reasons"] - - def test_missing_counter_total_is_unknown_not_rejected(self) -> None: - export = _export() - del export["metrics"]["trtllm_prompt_tokens"]["series"][1]["stats"]["total"] - - _, nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _cache_rate(nested)["status"] == "UNKNOWN" - assert "missing_total" in _cache_rate(nested)["reasons"] - - def test_missing_phase_window_is_unknown(self) -> None: - _, nested = _run(_export(phase_time_ranges=False)) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _cache_rate(nested)["status"] == "UNKNOWN" - assert "phase_window_missing" in _cache_rate(nested)["reasons"] - assert nested["validity"]["phase_windows"] is None - - def test_series_without_timeslices_is_unknown(self) -> None: - export = _export() - export["metrics"]["trtllm_prompt_tokens"]["series"][1] = _counter( - PREFILL[1], 2000.0, timeslices=False - ) - - _, nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _cache_rate(nested)["status"] == "UNKNOWN" - assert "no_timeslices" in _cache_rate(nested)["reasons"] - - def test_late_created_series_is_partial_coverage(self) -> None: - export = _export() - export["metrics"]["trtllm_prompt_tokens"]["series"][1] = _counter( - PREFILL[1], 2000.0, start_s=40.0 - ) - - _, nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _cache_rate(nested)["status"] == "PARTIAL_COVERAGE" - assert "window_starts_after_profiling" in _cache_rate(nested)["reasons"] - - def test_prefill_endpoint_missing_its_cached_series_is_partial_coverage(self) -> None: - cached = {k: v for k, v in CACHED.items() if k != PREFILL[1]} - - _, nested = _run(_export(cached=cached)) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _cache_rate(nested)["status"] == "PARTIAL_COVERAGE" - assert "missing_pair" in _cache_rate(nested)["reasons"] - - def test_unscraped_configured_endpoint_is_partial_coverage(self) -> None: - prompt = {k: v for k, v in PROMPT.items() if k != PREFILL[1]} - cached = {k: v for k, v in CACHED.items() if k != PREFILL[1]} - export = _export(prompt=prompt, cached=cached) - export["summary"]["endpoints_successful"].remove(PREFILL[1]) - - _, nested = _run(export) - - assert nested["cache"]["overall_cache_hit_rate"] is None - assert _cache_rate(nested)["status"] == "PARTIAL_COVERAGE" - assert "endpoint_not_scraped" in _cache_rate(nested)["reasons"] - - -class TestSharedPhaseSummary: - def test_histograms_pass_through_the_same_phase_check(self) -> None: - export = _export() - export["metrics"]["dynamo_frontend_cached_tokens"] = { - "type": "histogram", - "series": [_histogram(FRONTEND, 2700.0, 60, start_s=-119.9)], - } - gapped = _histogram(FRONTEND, 3000.0, 60) - del gapped["timeslices"][30] - export["metrics"]["dynamo_frontend_output_tokens"] = { - "type": "histogram", - "series": [gapped], - } - - _, nested = _run(export) - - by_status = nested["validity"]["metrics_by_status"] - assert "dynamo_frontend_cached_tokens" in by_status["PHASE_CONTAMINATED"] - assert "dynamo_frontend_output_tokens" in by_status["REJECTED"] - assert "dynamo_frontend_input_sequence_tokens" not in { - name for names in by_status.values() for name in names - } - - def test_empty_histogram_is_unknown_not_rejected(self) -> None: - export = _export() - export["metrics"]["dynamo_component_router_shared_cache_hit_rate"] = { - "type": "histogram", - "series": [{"endpoint_url": FRONTEND, "labels": {}, "stats": {"count": 0}}], - } - - _, nested = _run(export) - - by_status = nested["validity"]["metrics_by_status"] - assert by_status == {"UNKNOWN": ["dynamo_component_router_shared_cache_hit_rate"]} - - def test_other_backends_keep_published_values_and_are_not_evaluated(self) -> None: - def vllm_export(with_phase_data: bool) -> dict: - hits = _counter(PREFILL[0], 900.0) - queries = _counter(PREFILL[0], 1000.0) - if not with_phase_data: - hits.pop("timeslices") - queries.pop("timeslices") - export = { - "metrics": { - "vllm:prefix_cache_hits": {"type": "counter", "series": [hits]}, - "vllm:prefix_cache_queries": {"type": "counter", "series": [queries]}, - } - } - if with_phase_data: - export["summary"] = copy.deepcopy(_export()["summary"]) - return export - - _, with_phase = _run(vllm_export(True), framework="vllm") - _, without_phase = _run(vllm_export(False), framework="vllm") - - assert with_phase["cache"] == without_phase["cache"] - assert with_phase["cache"]["gpu_cache_hit_rate"] == pytest.approx(0.9) - assert with_phase["validity"]["cache_rate"]["status"] == "NOT_EVALUATED" - assert with_phase["validity"]["cache_rate"]["eligible_exact_actual"] is False diff --git a/inferencex-e2e/infx/tests/results/agentic/test_trtllm_cache_counters.py b/inferencex-e2e/infx/tests/results/agentic/test_trtllm_cache_counters.py new file mode 100644 index 0000000000..d497950f7b --- /dev/null +++ b/inferencex-e2e/infx/tests/results/agentic/test_trtllm_cache_counters.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +import copy + +import pytest + +from infx.results.agentic.server_metrics import compute_server_metrics + +PREFILL = "http://prefill/metrics" +DECODE = "http://decode/metrics" +PROMPT = "trtllm_prompt_tokens" +CACHED = "trtllm_prompt_cached_tokens" + + +def _series(endpoint: str, value: float, mode: str = "prefill") -> dict: + return { + "endpoint_url": endpoint, + "labels": {"dynamo_component": "backend", "disaggregation_mode": mode, "model_name": "m"}, + "stats": {"total": value}, + } + + +def _pair(cached: float = 600, suffix: str = "", mode: str = "prefill") -> dict: + return { + "schema_version": "1.1", + "aiperf_version": "1.0.12", + "metrics_phase": "profiling", + "summary": { + "phase_time_ranges": {"profiling": {"start_ns": 1, "end_ns": 100}}, + "endpoints_configured": [PREFILL, DECODE], + "endpoints_successful": [PREFILL, DECODE], + }, + "metrics": { + name + suffix: {"type": "counter", "series": [_series(PREFILL, value, mode)]} + for name, value in ((PROMPT, 1000), (CACHED, cached)) + }, + } + + +def _run(export: dict, disaggregated: bool | None = True, **kwargs) -> dict: + return compute_server_metrics( + export, framework="dynamo-trt", records=[], disaggregated=disaggregated, **kwargs + )[1] + + +@pytest.mark.parametrize("suffix", ["", "_total", "both"]) +def test_prefill_counter_aliases_exclude_decode_and_publish_only_combined_interval(suffix): + export = _pair(suffix="_total" if suffix == "_total" else "") + for name, entry in list(export["metrics"].items()): + entry["series"].append(_series(DECODE, 1000, "decode")) + if suffix == "both": + export["metrics"][name + "_total"] = copy.deepcopy(entry) + nested = _run(export) + + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.6) + assert nested["cache"]["gpu_cache_hit_rate"] is None + assert nested["cache"]["cpu_cache_hit_rate"] is None + assert nested["cache"]["cached_tokens_by_source"] == {"combined": 600} + assert nested["tokens"]["prompt_total"] == 1000 + assert nested["tokens"]["prompt_by_source"]["computed"] == 400 + core = nested["cache_validity"] + assert (core["status_class"], core["basis"], core["published"], core["pricing_eligible"]) == ( + "VALID_INTERVAL", + "combined", + True, + False, + ) + source = next(source for source in nested["sources"] if source["role"] == "prefill") + assert source["prefix_cache_hit_rate"] == pytest.approx(0.6) + + +@pytest.mark.parametrize("cached", [0, 1000]) +def test_zero_and_full_reuse_remain_publishable(cached): + nested = _run(_pair(cached)) + assert nested["cache"]["overall_cache_hit_rate"] == (0 if cached == 0 else 1) + assert nested["cache_validity"]["pricing_eligible"] is False + + +@pytest.mark.parametrize("labelled", [False, True]) +def test_aggregated_deployment_accepts_unlabelled_or_explicit_unified_counter(labelled): + export = _pair(mode="aggregated") + if not labelled: + for entry in export["metrics"].values(): + entry["series"][0]["labels"] = None + nested = _run(export, False) + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.6) + + +@pytest.mark.parametrize("bad", [None, True, -1, 0.5, float("nan"), float("inf"), "600"]) +def test_invalid_counter_total_blocks_headline_and_source_fallback(bad): + nested = _run(_pair(bad)) + assert nested["cache"]["overall_cache_hit_rate"] is None + assert nested["cache_validity"]["pricing_eligible"] is False + assert nested["sources"][0]["prefix_cache_hit_rate"] is None + + +@pytest.mark.parametrize( + "fault", ["unpaired", "rank", "duplicate", "alias", "bounds", "labels", "type", "empty"] +) +def test_counter_domains_must_pair_without_conflicts_or_compensating_totals(fault): + export = _pair() + prompt = export["metrics"][PROMPT]["series"] + cached = export["metrics"][CACHED]["series"] + if fault == "unpaired": + cached.append(_series(DECODE, 1)) + elif fault == "rank": + cached[0]["labels"]["rank"] = "1" + elif fault == "duplicate": + cached.append(copy.deepcopy(cached[0])) + elif fault == "alias": + export["metrics"][CACHED + "_total"] = { + "type": "counter", + "series": [_series(PREFILL, 601)], + } + elif fault == "bounds": + prompt[0]["stats"]["total"] = 100 + prompt.append(_series(DECODE, 1000)) + cached.append(_series(DECODE, 0)) + elif fault == "labels": + cached[0]["labels"] = [] + elif fault == "type": + export["metrics"][CACHED]["type"] = "gauge" + else: + prompt[0]["stats"]["total"] = cached[0]["stats"]["total"] = 0 + nested = _run(export) + assert nested["cache"]["overall_cache_hit_rate"] is None + assert all(source["prefix_cache_hit_rate"] is None for source in nested["sources"]) + + +@pytest.mark.parametrize("fault", ["unknown", "decode_only", "conflict", "deployment", "declared"]) +def test_unproven_or_contradictory_roles_do_not_publish(fault): + export = _pair() + kwargs = {} + disaggregated = None + if fault == "unknown": + for entry in export["metrics"].values(): + entry["series"].append({**_series(DECODE, 1), "labels": {}}) + elif fault == "decode_only": + for entry in export["metrics"].values(): + entry["series"][0]["labels"]["disaggregation_mode"] = "decode" + elif fault == "conflict": + export["metrics"]["trtllm_requests"] = {"series": [_series(PREFILL, 1, "decode")]} + elif fault == "deployment": + disaggregated = False + else: + kwargs["endpoint_roles"] = {PREFILL: "decode"} + nested = _run(export, disaggregated, **kwargs) + assert nested["cache"]["overall_cache_hit_rate"] is None + assert nested["cache_validity"]["published"] is False + + +@pytest.mark.parametrize("fault", ["old", "phase", "window", "unscraped", "unconfigured"]) +def test_only_modern_profiling_totals_with_accounted_endpoint_coverage_publish(fault): + export = _pair() + if fault == "old": + export["aiperf_version"] = "1.0.11" + elif fault == "phase": + export["metrics_phase"] = "warmup" + elif fault == "window": + export["summary"]["phase_time_ranges"]["profiling"]["end_ns"] = 0 + else: + key = "endpoints_successful" if fault == "unscraped" else "endpoints_configured" + export["summary"][key].remove(PREFILL) + nested = _run(export) + assert nested["cache"]["overall_cache_hit_rate"] is None + assert nested["cache_validity"]["pricing_eligible"] is False + + +def test_gauge_remains_source_diagnostic_and_never_overrides_counter_rate(): + export = _pair() + export["metrics"]["trtllm_kv_cache_hit_rate"] = { + "type": "gauge", + "series": [{"endpoint_url": PREFILL, "stats": {"avg": 0.9}}], + } + nested = _run(export) + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.6) + assert nested["sources"][0]["prefix_cache_hit_rate"] == pytest.approx(0.9) + + +def test_idle_domain_and_series_order_do_not_change_weighted_rate(): + export = _pair() + for name, value in ((PROMPT, 3000), (CACHED, 900)): + series = export["metrics"][name]["series"] + series.append( + {**_series(PREFILL, value), "labels": {"rank": "1", "disaggregation_mode": "prefill"}} + ) + series.append(_series(DECODE, 0)) + export["metrics"][CACHED]["series"].reverse() + nested = _run(export) + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.375) + assert nested["tokens"]["prompt_total"] == 4000 + + +@pytest.mark.parametrize("mode", [None, "", "router"]) +def test_explicit_invalid_role_cannot_fall_back_to_aggregated_deployment(mode): + export = _pair() + for entry in export["metrics"].values(): + entry["series"][0]["labels"]["disaggregation_mode"] = mode + assert _run(export, False)["cache"]["overall_cache_hit_rate"] is None + + +def test_conflicts_on_excluded_decode_do_not_block_prefill_measurement(): + export = _pair() + for entry in export["metrics"].values(): + entry["series"].append(_series(DECODE, 1000, "decode")) + export["metrics"]["trtllm_other_metric"] = {"series": [_series(DECODE, 1, "aggregated")]} + assert _run(export)["cache"]["overall_cache_hit_rate"] == pytest.approx(0.6) From 6022799543527a005a9a65673084c47a43dde5bf Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Thu, 22 Oct 2026 21:20:51 +0000 Subject: [PATCH 11/13] fix(agentx): reject explicit TRT non-worker components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：明确标为 router 或非法 component 的计数不能回退为聚合 worker。 --- .../infx/results/agentic/backends/trtllm.py | 12 ++++++------ .../results/agentic/test_trtllm_cache_counters.py | 7 ++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/backends/trtllm.py b/inferencex-e2e/infx/results/agentic/backends/trtllm.py index ad3762eb22..3478a0e7ed 100644 --- a/inferencex-e2e/infx/results/agentic/backends/trtllm.py +++ b/inferencex-e2e/infx/results/agentic/backends/trtllm.py @@ -9,7 +9,6 @@ from infx.results.agentic.cache_validity import modern_profile_error from infx.results.agentic.common import ( gauge_stat, - label_value, metric_series, normalize_fraction, rate, @@ -314,19 +313,20 @@ def _endpoint_role_sets( continue endpoint = str(series.get("endpoint_url", "")) labels = series.get("labels") - component = label_value(series, "dynamo_component") if isinstance(labels, dict) and "disaggregation_mode" in labels: mode = labels["disaggregation_mode"] role = _normalize_role(mode) if isinstance(mode, str) and mode else "unknown" explicit.setdefault(endpoint, set()).add(role) - elif component: + elif isinstance(labels, dict) and "dynamo_component" in labels: + component = labels["dynamo_component"] role = ( - "combined" + "unknown" + if not isinstance(component, str) or not component + else "combined" if component == "backend" and disaggregated is False else _normalize_role(component) ) - if role in ("prefill", "decode", "combined"): - inferred.setdefault(endpoint, set()).add(role) + inferred.setdefault(endpoint, set()).add(role) roles = {**inferred, **explicit} for endpoint, role in (declared or {}).items(): roles.setdefault(endpoint, set()).add(_normalize_role(role)) diff --git a/inferencex-e2e/infx/tests/results/agentic/test_trtllm_cache_counters.py b/inferencex-e2e/infx/tests/results/agentic/test_trtllm_cache_counters.py index d497950f7b..ba9d7920f8 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_trtllm_cache_counters.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_trtllm_cache_counters.py @@ -191,11 +191,12 @@ def test_idle_domain_and_series_order_do_not_change_weighted_rate(): assert nested["tokens"]["prompt_total"] == 4000 -@pytest.mark.parametrize("mode", [None, "", "router"]) -def test_explicit_invalid_role_cannot_fall_back_to_aggregated_deployment(mode): +@pytest.mark.parametrize("label", ["disaggregation_mode", "dynamo_component"]) +@pytest.mark.parametrize("mode", [None, "", "router", "unexpected"]) +def test_explicit_invalid_role_cannot_fall_back_to_aggregated_deployment(label, mode): export = _pair() for entry in export["metrics"].values(): - entry["series"][0]["labels"]["disaggregation_mode"] = mode + entry["series"][0]["labels"] = {label: mode} assert _run(export, False)["cache"]["overall_cache_hit_rate"] is None From cbb1790f46d135879c29008224726729998b5224 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Thu, 22 Oct 2026 21:21:10 +0000 Subject: [PATCH 12/13] fix(agentx): treat malformed frontend totals as unknown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：非法 frontend 条目返回未知总量，避免 P/D 聚合崩溃。 --- inferencex-e2e/infx/results/agentic/server_metrics.py | 2 ++ .../results/agentic/test_disaggregated_prompt_totals.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index ce8167e973..e07f669676 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -104,6 +104,8 @@ def _disaggregated_prompt_total( return prompt frontend = metrics.get("dynamo_frontend_input_sequence_tokens") if frontend is not None: + if not isinstance(frontend, dict): + return None series = frontend.get("series") key = {"counter": "total", "histogram": "sum"}.get(frontend.get("type")) if not isinstance(series, list) or key is None: diff --git a/inferencex-e2e/infx/tests/results/agentic/test_disaggregated_prompt_totals.py b/inferencex-e2e/infx/tests/results/agentic/test_disaggregated_prompt_totals.py index 4242da3e99..970cf0cb5b 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_disaggregated_prompt_totals.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_disaggregated_prompt_totals.py @@ -60,10 +60,13 @@ def test_missing_metrics_does_not_publish_a_partial_client_total(self): assert nested["tokens"]["prompt_total"] is None assert nested["cache_validity"]["status_class"] == "MISSING" - @pytest.mark.parametrize("totals", [[1234, None], [-10, 1244], [True], [float("inf")]]) + @pytest.mark.parametrize("totals", [None, [1234, None], [-10, 1244], [True], [float("inf")]]) def test_incomplete_or_invalid_frontend_cannot_supply_a_prompt_total(self, totals): counter = _counter(0) - counter["series"] = [{"stats": {"total": value}} for value in totals] + if totals is None: + counter = [] + else: + counter["series"] = [{"stats": {"total": value}} for value in totals] flat, nested, _ = compute_server_metrics( {"metrics": {"dynamo_frontend_input_sequence_tokens": counter}}, framework="dynamo-trt", From 0a0e364972fa3a1b23c2c2b21cb21c4155f7a282 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Sun, 25 Oct 2026 17:25:56 +0000 Subject: [PATCH 13/13] fix(results): preserve valid prompt fallbacks and TRT alias domains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复不完整 frontend 计数阻断有效 prefill 总量的问题，并按 endpoint 与 labels 合并 TRT counter 别名，避免漏记或重复计数。补充实际行为回归。 --- .../infx/results/agentic/backends/trtllm.py | 40 ++++++++++++-- .../infx/results/agentic/server_metrics.py | 25 +++++---- .../agentic/test_trtllm_cache_counters.py | 52 +++++++++++++++++++ 3 files changed, 100 insertions(+), 17 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/backends/trtllm.py b/inferencex-e2e/infx/results/agentic/backends/trtllm.py index 3478a0e7ed..ff988709a7 100644 --- a/inferencex-e2e/infx/results/agentic/backends/trtllm.py +++ b/inferencex-e2e/infx/results/agentic/backends/trtllm.py @@ -290,11 +290,43 @@ def _alias_total( base: str, series_filter: Callable[[dict[str, Any]], bool] | None = None, ) -> float | None: - # Native and exporter-stripped names represent the same counter. + totals: dict[tuple, float] = {} for name in (base, f"{base}_total"): - if name in metrics: - return sum_stat(metrics, name, series_filter=series_filter) - return None + entry = metrics.get(name) + if entry is None: + continue + if ( + not isinstance(entry, dict) + or entry.get("type") != "counter" + or not isinstance(entry.get("series"), list) + ): + return None + seen = set() + for series in metric_series(metrics, name): + if series_filter is not None and not series_filter(series): + continue + labels = series.get("labels") + labels = {} if labels is None else labels + if not isinstance(labels, dict) or any( + not isinstance(key, str) or not isinstance(value, str) + for key, value in labels.items() + ): + return None + key = str(series.get("endpoint_url", "")), tuple(sorted(labels.items())) + stats = series.get("stats") + value = stats.get("total") if isinstance(stats, dict) else None + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + or key in seen + or (key in totals and totals[key] != value) + ): + return None + seen.add(key) + totals[key] = value + return sum(totals.values()) if totals else None def _endpoint_role_sets( diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index e07f669676..49c2072364 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -103,21 +103,20 @@ def _disaggregated_prompt_total( if prompt is not None: return prompt frontend = metrics.get("dynamo_frontend_input_sequence_tokens") - if frontend is not None: - if not isinstance(frontend, dict): - return None + if isinstance(frontend, dict): series = frontend.get("series") key = {"counter": "total", "histogram": "sum"}.get(frontend.get("type")) - if not isinstance(series, list) or key is None: - return None - return _integer_total( - [ - item.get("stats", {}).get(key) - if isinstance(item, dict) and isinstance(item.get("stats"), dict) - else None - for item in series - ] - ) + if isinstance(series, list) and key is not None: + prompt = _integer_total( + [ + item.get("stats", {}).get(key) + if isinstance(item, dict) and isinstance(item.get("stats"), dict) + else None + for item in series + ] + ) + if prompt is not None: + return prompt reuse = nested["validity"]["cache_rate"] if reuse.get("published") and reuse.get("role_scope") == "prefill": return _integer_total([reuse.get("prompt_tokens")]) diff --git a/inferencex-e2e/infx/tests/results/agentic/test_trtllm_cache_counters.py b/inferencex-e2e/infx/tests/results/agentic/test_trtllm_cache_counters.py index ba9d7920f8..18797d7cc8 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_trtllm_cache_counters.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_trtllm_cache_counters.py @@ -206,3 +206,55 @@ def test_conflicts_on_excluded_decode_do_not_block_prefill_measurement(): entry["series"].append(_series(DECODE, 1000, "decode")) export["metrics"]["trtllm_other_metric"] = {"series": [_series(DECODE, 1, "aggregated")]} assert _run(export)["cache"]["overall_cache_hit_rate"] == pytest.approx(0.6) + + +@pytest.mark.parametrize("fault", ["shape", "missing", "partial", "type"]) +def test_incomplete_frontend_does_not_hide_the_validated_prefill_total(fault): + export = _pair() + frontend = {"type": "counter", "series": [{"stats": {"total": None}}]} + if fault == "shape": + frontend = [] + elif fault == "partial": + frontend["series"].append({"stats": {"total": 1234}}) + elif fault == "type": + frontend["type"] = "gauge" + export["metrics"]["dynamo_frontend_input_sequence_tokens"] = frontend + flat, nested, _ = compute_server_metrics( + export, framework="dynamo-trt", records=[], disaggregated=True + ) + assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 1000 + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.6) + assert nested["cache_validity"]["pricing_eligible"] is False + + +@pytest.mark.parametrize("overlap", ["none", "identical", "conflicting"]) +def test_alias_totals_combine_disjoint_domains_without_counting_aliases_twice(overlap): + export = _pair() + other = "http://prefill-2/metrics" + for key in ("endpoints_configured", "endpoints_successful"): + export["summary"][key].append(other) + for name, first, second in ( + (PROMPT, 1000, 3000), + (CACHED, 600, 900), + ("trtllm_generation_tokens", 200, 600), + ("trtllm_kv_cache_offload_bytes", 100, 300), + ): + export["metrics"][name] = {"type": "counter", "series": [_series(PREFILL, first)]} + export["metrics"][name + "_total"] = { + "type": "counter", + "series": [_series(other, second)], + } + if overlap != "none": + export["metrics"][name + "_total"]["series"].append(_series(PREFILL, first)) + if overlap == "conflicting": + export["metrics"]["trtllm_generation_tokens_total"]["series"][-1]["stats"]["total"] = 201 + flat, nested, _ = compute_server_metrics( + export, framework="dynamo-trt", records=[], disaggregated=True + ) + assert nested["tokens"]["prompt_total"] == 4000 + assert nested["cache"]["overall_cache_hit_rate"] == pytest.approx(0.375) + assert flat["total_generation_tokens"] == (None if overlap == "conflicting" else 800) + assert flat["kv_offload_bytes_gpu_to_cpu"] == 400 + source = next(row for row in nested["sources"] if row["id"].endswith(other)) + assert source["prompt_tokens"] == 3000 + assert source["generation_tokens"] == 600