From 8df9856e3b1d0796a8a7c0803a9a053325d23aad Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Wed, 14 Oct 2026 05:03:01 +0000 Subject: [PATCH 01/12] feat(agentx): add GLM-5.2 MI355X ATOM PD disaggregation points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: eight AgentX points with a CPP4 (PP4 x TP1) ATOM prefill behind AToMesh and Mooncake RDMA to the decodes. 1P1D on one node at C56-C112 keeps the prefill's prefix tier in a Mooncake Store of four per-NIC pools on reserved 2 MiB HugeTLB pages; 1P2D at C56 and 1P3D at C12-C24 use LMCache in the prefill. host_setup drops the page cache and compacts memory before every point. Ship NVIDIA/srt-slurm#604 as runners/srt-slurm/patches/atom-tp-from-pp.patch so srtctl launches the PP4 prefill with -tp 1. 新增 glm5.2-fp4-mi355x-atom-disagg-agentic-mtp：八个 AgentX 测试点。ATOM prefill 为 CPP4（PP4 x TP1），经 AToMesh 路由，并通过 Mooncake RDMA 把 KV 传给 decode。单节点 1P1D（C56-C112）把 prefill 的前缀缓存层放在 Mooncake Store 中：四个按网卡划分的池，使用预留的 2 MiB HugeTLB 大页；1P2D（C56）和 1P3D（C12-C24）在 prefill 中使用 LMCache。每个测试点开始前，host_setup 都会 清理页缓存并整理内存。 以 runners/srt-slurm/patches/atom-tp-from-pp.patch 的形式附带 NVIDIA/srt-slurm#604，使 srtctl 以 -tp 1 启动 PP4 prefill。 Co-Authored-By: Claude Opus 5.5 --- .../mi355x-fp4/agentx/disagg-variants.yaml | 335 ++++++++++++++++++ inferencex-e2e/configs/amd-master.yaml | 88 +++++ inferencex-e2e/perf-changelog.yaml | 12 + .../runners/srt-slurm/patches/README.md | 1 + .../srt-slurm/patches/atom-tp-from-pp.patch | 60 ++++ 5 files changed, 496 insertions(+) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml create mode 100644 inferencex-e2e/runners/srt-slurm/patches/atom-tp-from-pp.patch diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..70a67211cc --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,335 @@ +base: + schema: 2 + name: mi355x-glm52-atom-agentx-disagg-mtp + slurm: + time_limit: "08:00:00" + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: atomesh + enable_multiple_frontends: false + args: + log-level: info + disable-health-check: true + disable-circuit-breaker: true + prometheus-port: 29100 + prefill-policy: round_robin + decode-policy: round_robin + atom-pd-rank-mapping-policy: none + engine: atom + # A recipe host_setup replaces the cluster default, so run that first (NIC + # checks, GPU drain, release of hugepages a previous job left). Then reclaim + # memory on every node: drop the clean page cache and compact what is free. + host_setup: + nodes: all + timeout_seconds: 1800 + commands: + - &cluster_setup IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" + - &reclaim_memory sync && echo 1 | sudo -n tee /proc/sys/vm/drop_caches >/dev/null && echo 1 | sudo -n tee /proc/sys/vm/compact_memory >/dev/null + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + <<: &environment + PYTHONUNBUFFERED: "1" + PYTHONHASHSEED: "0" + AITER_LOG_LEVEL: WARNING + AITER_QUICK_REDUCE_QUANTIZATION: INT4 + AITER_USE_FLYDSL_MOE_SORTING: "1" + ATOM_MLA_PAGE_SIZE: "1" + ATOM_ONLINE_QUANT_STREAMING: "0" + ATOM_SPARSE_INDEXER_LOGITS_BUDGET_MB: "2047" + ATOM_USE_TRITON_MLA: "0" + TOPK_FORCE_PATH: one + MAX_JOBS: "16" + HSA_ENABLE_IPC_MODE_LEGACY: "1" + MC_DISABLE_HIP: "1" + # Rail-isolated RDMA: each transfer uses the same rail on both ends. + MC_GID_INDEX: "@fabric.rdma-gid-index" + ATOM_MOONCAKE_MATCHED_RAILS: auto + NCCL_IB_DISABLE: "1" + VLLM_PP_LAYER_PARTITION: "20,20,20,18" + OFFLOAD_PROFILE: "1" + OFFLOAD_MIN_LOAD_TOKENS: "0" + args: + <<: &server + served-model-name: amd/GLM-5.2-MXFP4 + trust-remote-code: true + kv_cache_dtype: fp8 + block-size: 16 + # model.layers.78 is the MTP head, shipped in BF16; excluding it keeps + # online ptpc_fp8 quantization off the draft while experts stay MXFP4. + online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","model.layers.[0-9].mlp.*expert*","model.layers.[1-6][0-9].mlp.*expert*","model.layers.7[0-7].mlp.*expert*","model.layers.78.*"]}' + level: 3 + cudagraph-mode: FULL + method: mtp + timeout-keep-alive: 900 + pipeline-parallel-size: 4 + enforce-eager: true + gpu-memory-utilization: 0.85 + max-num-seqs: 512 + max-num-batched-tokens: 8192 + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: *environment + args: *server + sbatch_directives: + cpus-per-task: "128" + mem: "0" + srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + health_check: + max_attempts: 1080 + interval_seconds: 5 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "atom:" + AIPERF_APPLY_CHAT_TEMPLATE: "true" + TOKENIZERS_PARALLELISM: "false" + TRANSFORMERS_VERBOSITY: error + +# One variant per point. A DCP4 decode admits 512 and captures every batch up +# to 256; a 1P3D decode admits twice CONC and captures up to that. + +override_1p1d_mcstore_c56: &mcstore_1p1d + host_setup: + nodes: all + timeout_seconds: 1800 + commands: + - *cluster_setup + - *reclaim_memory + # Owners bind NUMA0 (rdma0-1) and NUMA1 (rdma2-3), 178,688 2 MiB pages + # each, so each NUMA node needs 357,376 free. Reserve both nodes' share + # with 2 GiB spare each on the prefill node, and fail the node here when + # the kernel grants fewer. + - >- + if [[ "$(hostname -s)" == "$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)" ]]; then + HUGEPAGES_TARGET=716800 HUGEPAGE_GROW_TOLERANCE_PCT=0 + bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" || exit 1; + for node in 0 1; do + free=$(cat /sys/devices/system/node/node$node/hugepages/hugepages-2048kB/free_hugepages); + echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 357376"; + [[ $free -ge 357376 ]] || exit 1; + done; fi + teardown: + - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" + # Pool k is rdma k: its master listens on 26051 + 100k (HTTP metadata on + # 26080 + 100k), and its owner registers a 349 GiB segment on rdma k. + # numa_exec.py (shipped in the image) binds each process's CPUs and memory + # to one NUMA node. + services: + - &master + name: mc-master-rdma0 + type: generic + placement: + node: prefill + start: before_workers + critical: true + inherit_discovery_env: false + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26051, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26080, --metrics_port=26090, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + env: &store_env + # A Mooncake process that sees a GPU probes every buffer through HIP. + HIP_VISIBLE_DEVICES: "-1" + CUDA_VISIBLE_DEVICES: "-1" + # Peers with different QP counts cannot connect. + MC_NUM_QP_PER_EP: "1" + MC_GID_INDEX: "@fabric.rdma-gid-index" + readiness: + port: 26051 + timeout_seconds: 300 + - <<: *master + name: mc-master-rdma1 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26151, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26180, --metrics_port=26190, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26151 + timeout_seconds: 300 + - <<: *master + name: mc-master-rdma2 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26251, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26280, --metrics_port=26290, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26251 + timeout_seconds: 300 + - <<: *master + name: mc-master-rdma3 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26351, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26380, --metrics_port=26390, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26351 + timeout_seconds: 300 + - &owner + <<: *master + name: mc-owner-rdma0 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + env: + <<: *store_env + # Map the segment from the reserved 2 MiB pages, never 4 KiB ones the + # NIC cannot register in 64 GiB regions. + MC_STORE_USE_HUGEPAGE: "1" + MC_MAX_MR_SIZE: "68719476736" + MC_TCP_BIND_ADDRESS: "{node_ip}" + readiness: + log: + pattern: Starting real client service + timeout_seconds: 1800 + interval_seconds: 5 + - <<: *owner + name: mc-owner-rdma1 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26151, --metadata_server=http://127.0.0.1:26180/metadata, --protocol=rdma, --device_names=rdma1, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + - <<: *owner + name: mc-owner-rdma2 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26054, --master_server_address=127.0.0.1:26251, --metadata_server=http://127.0.0.1:26280/metadata, --protocol=rdma, --device_names=rdma2, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + - <<: *owner + name: mc-owner-rdma3 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26055, --master_server_address=127.0.0.1:26351, --metadata_server=http://127.0.0.1:26380/metadata, --protocol=rdma, --device_names=rdma3, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + roles: + prefill: + env: + # More than one load worker hung the PP prefill pipeline at C96. + OFFLOAD_LOAD_WORKERS: "1" + MC_NUM_QP_PER_EP: "1" + # Registers the worker's HBM transfer pool in 1 GiB regions. + MC_MAX_MR_SIZE: "1073741824" + args: + num-speculative-tokens: 3 + # Each stage's Store client uses its GPU's NIC and that NIC's pool. + extra-kv-connectors: + - kv_connector: mooncake_store + kv_role: offload + kv_connector_extra_config: + mooncake_store.protocol: rdma + mooncake_store.pools: + rdma0: {master: "127.0.0.1:26051", metadata: "http://127.0.0.1:26080/metadata"} + rdma1: {master: "127.0.0.1:26151", metadata: "http://127.0.0.1:26180/metadata"} + rdma2: {master: "127.0.0.1:26251", metadata: "http://127.0.0.1:26280/metadata"} + rdma3: {master: "127.0.0.1:26351", metadata: "http://127.0.0.1:26380/metadata"} + decode: + nodes: colocate + env: + AITER_REUSE_IDENTICAL_COMM_GROUPS: "1" + # The P/D transfer must match the prefill's QP count, one with the Store. + MC_NUM_QP_PER_EP: "1" + args: + <<: &dcp_decode + decode-context-parallel-size: 4 + gpu-memory-utilization: 0.94 + max-num-seqs: 512 + max-num-batched-tokens: 2048 + cudagraph-capture-sizes: '[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' + num-speculative-tokens: 3 + +override_1p1d_mcstore_c72: *mcstore_1p1d + +override_1p1d_mcstore_c96: *mcstore_1p1d + +override_1p1d_mcstore_c112: *mcstore_1p1d + +override_1p2d_lmcache_c56: + resources: &spread + # A node per decode; srtctl otherwise packs two TP4 decodes onto one node. + spread_workers: true + frontend: + args: &cache_aware + policy: cache_aware + prefill-policy: cache_aware + decode-policy: cache_aware + cache-threshold: 0.8 + balance-abs-threshold: 2 + balance-rel-threshold: 2.0 + eviction-interval: 300 + roles: + prefill: + env: &lmcache_env + LMCACHE_LOCAL_CPU: "True" + LMCACHE_NUMA_MODE: auto + LMCACHE_MAX_LOCAL_CPU_SIZE: "256" + LMCACHE_CHUNK_SIZE: "256" + # Every PP stage runs a lookup server, so each stage refreshes its own LRU. + LMCACHE_LOOKUP_SERVER_WORKER_IDS: "0,1,2,3" + args: + num-speculative-tokens: 4 + extra-kv-connectors: &lmcache_connector + - kv_connector: lmcache_offload + kv_role: offload + decode: + nodes: 2 + workers: 2 + env: + AITER_REUSE_IDENTICAL_COMM_GROUPS: "1" + args: + <<: *dcp_decode + num-speculative-tokens: 4 + +override_1p3d_lmcache_c12: + resources: *spread + frontend: + args: *cache_aware + roles: + prefill: + env: *lmcache_env + args: + num-speculative-tokens: 5 + extra-kv-connectors: *lmcache_connector + decode: + nodes: 3 + workers: 3 + env: + PYTHONNOUSERSITE: "1" + args: + num-speculative-tokens: 5 + gpu-memory-utilization: 0.95 + max-num-seqs: 24 + max-num-batched-tokens: 16384 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24]' + disable-uvicorn-access-log: true + +override_1p3d_lmcache_c16: + resources: *spread + frontend: + args: *cache_aware + roles: + prefill: + env: *lmcache_env + args: + num-speculative-tokens: 5 + extra-kv-connectors: *lmcache_connector + decode: + nodes: 3 + workers: 3 + env: + PYTHONNOUSERSITE: "1" + args: + num-speculative-tokens: 5 + gpu-memory-utilization: 0.95 + max-num-seqs: 32 + max-num-batched-tokens: 16384 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32]' + disable-uvicorn-access-log: true + +override_1p3d_lmcache_c24: + resources: *spread + frontend: + args: *cache_aware + roles: + prefill: + env: *lmcache_env + args: + num-speculative-tokens: 5 + extra-kv-connectors: *lmcache_connector + decode: + nodes: 3 + workers: 3 + env: + PYTHONNOUSERSITE: "1" + args: + num-speculative-tokens: 5 + gpu-memory-utilization: 0.95 + max-num-seqs: 48 + max-num-batched-tokens: 16384 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32,36,40,44,48]' + disable-uvicorn-access-log: true diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 24f60ae790..39a8d7bb57 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1484,6 +1484,94 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: - { tp: 4, kv-offloading: none, conc-list: [2, 4, 8, 10], spec-decoding: mtp, srt-recipe: agentic.yaml } - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } +# GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM prefill/decode +# disaggregation behind AToMesh, with Mooncake RDMA moving KV from prefill to +# decode and MTP drafts. The prefill is one CPP4 worker: PP4 x TP1 on four GPUs. +# (1) 1P1D on one node, a DCP4 decode on GPUs 4-7, MTP3, concurrency 56-112, +# with the prefill's prefix tier in a Mooncake Store of four per-NIC pools +# on 2 MiB HugeTLB pages. +# (2) LMCache in the prefill: 1P2D with two DCP4 decodes, MTP4, concurrency 56; +# 1P3D with three TP4 decodes, MTP5, concurrency 12-24. Each decode has a +# node of its own. +# dram-utilization is over the prefill's four GPUs, half of the node's capped +# 3 TB: 1.0 gives the Store's 4 x 349 GiB (1499 GB), and 0.733 gives LMCache's +# 4 x 256 GiB (1099 GB). srtctl sizes the PP4 prefill as -tp 1 through +# runners/srt-slurm/patches/atom-tp-from-pp.patch. +glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: + image: rocm/atom-dev:nightly_202610131249@sha256:d5b3fcd99bd44e3489740d1b026037f49f2e3f6160dc70b70c3fd23a125a4905 + model: amd/GLM-5.2-MXFP4 + model-prefix: glm5.2 + runner: cluster:mi355x-amds + precision: fp4 + framework: atom-disagg + router: { name: atomesh, version: "6fd1bb1be97c79b571eb86674786d0d888353b73" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: glm5.2/atom/mi355x-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 1.0 + search-space: + - spec-decoding: "mtp" + conc-list: [ 56 ] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c56 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 72 ] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c72 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 96 ] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c96 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 112 ] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c112 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + - dram-utilization: 0.733 + search-space: + - spec-decoding: "mtp" + conc-list: [ 56 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-variants.yaml:override_1p2d_lmcache_c56 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 2, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 12 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-variants.yaml:override_1p3d_lmcache_c12 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 16 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-variants.yaml:override_1p3d_lmcache_c16 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 24 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-variants.yaml:override_1p3d_lmcache_c24 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } + dsv4-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261007 model: deepseek-ai/DeepSeek-V4-Pro-0813 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 74c014908a..54832f1d5f 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9718,3 +9718,15 @@ - "Enable INT4 QuickReduce with a 256 MiB size limit and FlyDSL stage-2 FP8 on prefill and decode workers." - "Configure MoRIIO READ and prefill-only SimpleCPUOffloadConnector through native role arguments. Use recipe-scoped draft checkpoint and RDMA device/provider mounts. Use the upstream discovery-template fix from srt-slurm #508 through the shared v2.51.1 dependency." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3582 + +- config-keys: + - glm5.2-fp4-mi355x-atom-disagg-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2 MXFP4 on MI355X with ATOM prefill/decode disaggregation behind AToMesh, Mooncake RDMA from prefill to decode and MTP, on rocm/atom-dev:nightly_202610131249 (digest-pinned). The prefill is one CPP4 worker (PP4 x TP1 on four GPUs)." + - "1P1D on one node at concurrency 56, 72, 96 and 112: a DCP4 decode on GPUs 4-7 with three draft tokens, and the prefill's prefix tier in a Mooncake Store (0.3.13) of four per-NIC pools, each one master and a 349 GiB owner on 2 MiB HugeTLB pages. host_setup reserves the pages before the job and teardown releases them." + - "LMCache in the prefill (256 GiB per stage): 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens), and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). Each decode has a node of its own." + - "host_setup drops the page cache and compacts free memory on every node before each point." + - "Patch srt-slurm so its ATOM backend passes -tp as the worker's GPUs over its pipeline and prefill-context stages (NVIDIA/srt-slurm#604); unpatched, the PP4 prefill launches with -tp 4." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/inferencex-e2e/runners/srt-slurm/patches/README.md b/inferencex-e2e/runners/srt-slurm/patches/README.md index fcadc7bf88..9207d66b70 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/README.md +++ b/inferencex-e2e/runners/srt-slurm/patches/README.md @@ -8,3 +8,4 @@ Each patch is a temporary fix for an open upstream PR. When the PR merges and th | Patch | Upstream PR | Fix | |-------|-------------|-----| +| `atom-tp-from-pp.patch` | [NVIDIA/srt-slurm#604](https://github.com/NVIDIA/srt-slurm/pull/604) | ATOM's `-tp` is the worker's GPUs divided by its pipeline and prefill-context stages, so a 4-GPU PP4 prefill launches with `-tp 1` instead of `-tp 4` | diff --git a/inferencex-e2e/runners/srt-slurm/patches/atom-tp-from-pp.patch b/inferencex-e2e/runners/srt-slurm/patches/atom-tp-from-pp.patch new file mode 100644 index 0000000000..33a6019d1f --- /dev/null +++ b/inferencex-e2e/runners/srt-slurm/patches/atom-tp-from-pp.patch @@ -0,0 +1,60 @@ +From e23ca1bed9aee9a59905fb30b27f09b3ef9e9224 Mon Sep 17 00:00:00 2001 +From: Jasen2201 +Date: Wed, 14 Oct 2026 04:15:36 +0000 +Subject: [PATCH] fix(atom): size -tp from the worker's GPUs over its PP and + PCP stages + +ATOM runs a worker on tp x pp x pcp GPUs, but srtctl always passed +`-tp `. A 4-GPU prefill worker with +`pipeline-parallel-size: 4` therefore launched as +`-tp 4 --pipeline-parallel-size 4`, and ATOM sized its world to 16 GPUs. + +`-tp` is now the worker's GPU count divided by `pipeline-parallel-size` +and `prefill-context-parallel-size`. Decode context parallelism reuses the +TP GPUs and does not change it. A split that does not divide the worker's +GPUs fails before launch. + +Co-Authored-By: Claude Opus 5.5 +Signed-off-by: Jasen2201 +--- + src/srtctl/backends/atom.py | 20 +++++++++++++++++++- + 1 file changed, 19 insertions(+), 1 deletion(-) + +diff --git a/src/srtctl/backends/atom.py b/src/srtctl/backends/atom.py +index edf9a9b7..4e5b836e 100644 +--- a/src/srtctl/backends/atom.py ++++ b/src/srtctl/backends/atom.py +@@ -155,7 +155,7 @@ class AtomBackend(Backend): + "--server-port", + str(process.http_port), + "-tp", +- str(len(process.gpu_indices)), ++ str(_tensor_parallel_size(len(process.gpu_indices), config)), + ] + ) + kv_transfer_config = self._kv_transfer_config(process, worker_ip, extra_connectors) +@@ -183,3 +183,21 @@ def _config_to_cli_args(config: dict[str, Any]) -> list[str]: + + def _canonical_arg_key(key: str) -> str: + return key.lstrip("-").replace("_", "-") ++ ++ ++# ATOM sizes a worker as tp x pp x pcp GPUs; decode context parallelism reuses the TP GPUs. ++_GPU_SPLITTING_ARGS = ("pipeline-parallel-size", "prefill-context-parallel-size") ++ ++ ++def _tensor_parallel_size(gpu_count: int, config: Mapping[str, Any]) -> int: ++ """The worker's GPUs left for tensor parallelism after its pipeline and prefill-context stages.""" ++ stages = 1 ++ for key, value in config.items(): ++ if _canonical_arg_key(key) in _GPU_SPLITTING_ARGS: ++ stages *= int(value) ++ if stages < 1 or gpu_count % stages: ++ raise ValueError( ++ f"ATOM worker has {gpu_count} GPU(s), which its pipeline x prefill-context parallelism of {stages} " ++ "does not divide" ++ ) ++ return gpu_count // stages +-- +2.43.0 + From 267bc6cf448637d30431ef6695a4af00c7dfe0d0 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Wed, 14 Oct 2026 05:13:15 +0000 Subject: [PATCH 02/12] chore(changelog): link the GLM-5.2 ATOM PD entry to its PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set the new perf-changelog entry's pr-link to #4045. 将新增的 perf-changelog 条目的 pr-link 设为 #4045。 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 54832f1d5f..3b8861632c 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9729,4 +9729,4 @@ - "LMCache in the prefill (256 GiB per stage): 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens), and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). Each decode has a node of its own." - "host_setup drops the page cache and compacts free memory on every node before each point." - "Patch srt-slurm so its ATOM backend passes -tp as the worker's GPUs over its pipeline and prefill-context stages (NVIDIA/srt-slurm#604); unpatched, the PP4 prefill launches with -tp 4." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4045 From a377cbcb2275afdd1d4521cea7a6956b83eae4e8 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Wed, 14 Oct 2026 07:34:56 +0000 Subject: [PATCH 03/12] fix(agentx): cache the GLM-5.2 checkpoint once and size Store hugepages above held pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Set HF_HOME/HF_HUB_CACHE on the ATOM roles. amd/GLM-5.2-MXFP4 has no staged copy on mi355x-amds, so the job serves hf:; srtctl downloads it once into the shared hub cache only when a role sets HF_HOME. - Follow the UMBP recipe's host_setup for the Store: the cluster setup.sh frees leftover hugepages and drops the page cache, then the prefill node reserves the owners' 714752 2 MiB pages plus 2% on top of the pages host services hold. mi355x-amds nodes hold 5656, which the old fixed target of 716800 left too few for both NUMA nodes. The LMCache points keep the cluster default host setup. - Run the four Store points from one variant, so the planner schedules one GSM8K eval for them instead of four. - Drop header and section comments per AGENTS.md, and rename the srt-slurm patch to 604-atom-tp-from-pp.patch like the other patches. - 为 ATOM 角色设置 HF_HOME/HF_HUB_CACHE。mi355x-amds 上没有预置的 amd/GLM-5.2-MXFP4，任务以 hf: 方式加载；只有角色设置了 HF_HOME，srtctl 才会把模型一次性下载到共享 hub 缓存。 - Store 的 host_setup 参照 UMBP recipe：先由集群 setup.sh 释放残留大页并 清理页缓存，再在 prefill 节点上于主机服务已占用的大页之上，预留 owner 所需 的 714752 个 2 MiB 页并多留 2%。mi355x-amds 节点常驻占用 5656 页，原先固定 的 716800 目标会导致两个 NUMA 节点的空闲页不足。LMCache 测试点沿用集群默认 的 host setup。 - 四个 Store 测试点共用一个变体，规划器只为它们安排一个 GSM8K eval，而不是 四个。 - 按 AGENTS.md 删除头注释与段落注释，并把 srt-slurm 补丁按惯例重命名为 604-atom-tp-from-pp.patch。 Co-Authored-By: Claude Opus 5.5 --- .../mi355x-fp4/agentx/disagg-variants.yaml | 43 +++++-------------- inferencex-e2e/configs/amd-master.yaml | 40 ++--------------- inferencex-e2e/perf-changelog.yaml | 4 +- ...rom-pp.patch => 604-atom-tp-from-pp.patch} | 0 .../runners/srt-slurm/patches/README.md | 2 +- 5 files changed, 18 insertions(+), 71 deletions(-) rename inferencex-e2e/runners/srt-slurm/patches/{atom-tp-from-pp.patch => 604-atom-tp-from-pp.patch} (100%) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml index 70a67211cc..8ec354facb 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -18,15 +18,6 @@ base: decode-policy: round_robin atom-pd-rank-mapping-policy: none engine: atom - # A recipe host_setup replaces the cluster default, so run that first (NIC - # checks, GPU drain, release of hugepages a previous job left). Then reclaim - # memory on every node: drop the clean page cache and compact what is free. - host_setup: - nodes: all - timeout_seconds: 1800 - commands: - - &cluster_setup IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" - - &reclaim_memory sync && echo 1 | sudo -n tee /proc/sys/vm/drop_caches >/dev/null && echo 1 | sudo -n tee /proc/sys/vm/compact_memory >/dev/null roles: prefill: nodes: 1 @@ -36,6 +27,9 @@ base: <<: &environment PYTHONUNBUFFERED: "1" PYTHONHASHSEED: "0" + # srtctl downloads an hf: checkpoint once into the shared hub cache only when HF_HOME is set. + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub AITER_LOG_LEVEL: WARNING AITER_QUICK_REDUCE_QUANTIZATION: INT4 AITER_USE_FLYDSL_MOE_SORTING: "1" @@ -95,23 +89,18 @@ base: TOKENIZERS_PARALLELISM: "false" TRANSFORMERS_VERBOSITY: error -# One variant per point. A DCP4 decode admits 512 and captures every batch up -# to 256; a 1P3D decode admits twice CONC and captures up to that. - -override_1p1d_mcstore_c56: &mcstore_1p1d +override_1p1d_mcstore: host_setup: nodes: all timeout_seconds: 1800 commands: - - *cluster_setup - - *reclaim_memory - # Owners bind NUMA0 (rdma0-1) and NUMA1 (rdma2-3), 178,688 2 MiB pages - # each, so each NUMA node needs 357,376 free. Reserve both nodes' share - # with 2 GiB spare each on the prefill node, and fail the node here when - # the kernel grants fewer. + # A recipe host_setup replaces the cluster default, which also drops the page cache under DRAM KV offloading. + - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" + # Owners take 714752 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. - >- if [[ "$(hostname -s)" == "$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)" ]]; then - HUGEPAGES_TARGET=716800 HUGEPAGE_GROW_TOLERANCE_PCT=0 + held=$(awk '/^HugePages_Total:/ {total = $2} /^HugePages_Free:/ {free = $2} /^HugePages_Rsvd:/ {rsvd = $2} END {print total - free + rsvd}' /proc/meminfo); + HUGEPAGES_TARGET=$(( held + 714752 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" || exit 1; for node in 0 1; do free=$(cat /sys/devices/system/node/node$node/hugepages/hugepages-2048kB/free_hugepages); @@ -120,10 +109,6 @@ override_1p1d_mcstore_c56: &mcstore_1p1d done; fi teardown: - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" - # Pool k is rdma k: its master listens on 26051 + 100k (HTTP metadata on - # 26080 + 100k), and its owner registers a 349 GiB segment on rdma k. - # numa_exec.py (shipped in the image) binds each process's CPUs and memory - # to one NUMA node. services: - &master name: mc-master-rdma0 @@ -133,6 +118,7 @@ override_1p1d_mcstore_c56: &mcstore_1p1d start: before_workers critical: true inherit_discovery_env: false + # numa_exec.py binds the process's CPUs and memory to that NUMA node. command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26051, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26080, --metrics_port=26090, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] env: &store_env # A Mooncake process that sees a GPU probes every buffer through HIP. @@ -168,8 +154,7 @@ override_1p1d_mcstore_c56: &mcstore_1p1d command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] env: <<: *store_env - # Map the segment from the reserved 2 MiB pages, never 4 KiB ones the - # NIC cannot register in 64 GiB regions. + # Back the segment with the reserved 2 MiB pages; the NIC cannot register 64 GiB regions of 4 KiB pages. MC_STORE_USE_HUGEPAGE: "1" MC_MAX_MR_SIZE: "68719476736" MC_TCP_BIND_ADDRESS: "{node_ip}" @@ -223,12 +208,6 @@ override_1p1d_mcstore_c56: &mcstore_1p1d cudagraph-capture-sizes: '[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' num-speculative-tokens: 3 -override_1p1d_mcstore_c72: *mcstore_1p1d - -override_1p1d_mcstore_c96: *mcstore_1p1d - -override_1p1d_mcstore_c112: *mcstore_1p1d - override_1p2d_lmcache_c56: resources: &spread # A node per decode; srtctl otherwise packs two TP4 decodes onto one node. diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 39a8d7bb57..0ab8611b48 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1484,19 +1484,6 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: - { tp: 4, kv-offloading: none, conc-list: [2, 4, 8, 10], spec-decoding: mtp, srt-recipe: agentic.yaml } - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } -# GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM prefill/decode -# disaggregation behind AToMesh, with Mooncake RDMA moving KV from prefill to -# decode and MTP drafts. The prefill is one CPP4 worker: PP4 x TP1 on four GPUs. -# (1) 1P1D on one node, a DCP4 decode on GPUs 4-7, MTP3, concurrency 56-112, -# with the prefill's prefix tier in a Mooncake Store of four per-NIC pools -# on 2 MiB HugeTLB pages. -# (2) LMCache in the prefill: 1P2D with two DCP4 decodes, MTP4, concurrency 56; -# 1P3D with three TP4 decodes, MTP5, concurrency 12-24. Each decode has a -# node of its own. -# dram-utilization is over the prefill's four GPUs, half of the node's capped -# 3 TB: 1.0 gives the Store's 4 x 349 GiB (1499 GB), and 0.733 gives LMCache's -# 4 x 256 GiB (1099 GB). srtctl sizes the PP4 prefill as -tp 1 through -# runners/srt-slurm/patches/atom-tp-from-pp.patch. glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: image: rocm/atom-dev:nightly_202610131249@sha256:d5b3fcd99bd44e3489740d1b026037f49f2e3f6160dc70b70c3fd23a125a4905 model: amd/GLM-5.2-MXFP4 @@ -1511,36 +1498,17 @@ glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: srt-recipe-dir: glm5.2/atom/mi355x-fp4/agentx scenarios: agentic-coding: + # The Store's 4 x 349 GiB owners: all 1499 GB of the 4-GPU prefill's share of the 3 TB cap. - dram-utilization: 1.0 search-space: - spec-decoding: "mtp" - conc-list: [ 56 ] - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.13" } - srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c56 - prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } - decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } - - spec-decoding: "mtp" - conc-list: [ 72 ] - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.13" } - srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c72 - prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } - decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } - - spec-decoding: "mtp" - conc-list: [ 96 ] - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.13" } - srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c96 - prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } - decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } - - spec-decoding: "mtp" - conc-list: [ 112 ] + conc-list: [ 56, 72, 96, 112 ] kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13" } - srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c112 + srt-recipe: disagg-variants.yaml:override_1p1d_mcstore prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + # LMCache's 4 x 256 GiB stages: 1099 GB of that share. - dram-utilization: 0.733 search-space: - spec-decoding: "mtp" diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3b8861632c..b7293859fe 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9725,8 +9725,8 @@ - agentic-coding description: - "Add GLM-5.2 MXFP4 on MI355X with ATOM prefill/decode disaggregation behind AToMesh, Mooncake RDMA from prefill to decode and MTP, on rocm/atom-dev:nightly_202610131249 (digest-pinned). The prefill is one CPP4 worker (PP4 x TP1 on four GPUs)." - - "1P1D on one node at concurrency 56, 72, 96 and 112: a DCP4 decode on GPUs 4-7 with three draft tokens, and the prefill's prefix tier in a Mooncake Store (0.3.13) of four per-NIC pools, each one master and a 349 GiB owner on 2 MiB HugeTLB pages. host_setup reserves the pages before the job and teardown releases them." + - "1P1D on one node at concurrency 56, 72, 96 and 112: a DCP4 decode on GPUs 4-7 with three draft tokens, and the prefill's prefix tier in a Mooncake Store (0.3.13) of four per-NIC pools, each one master and a 349 GiB owner on 2 MiB HugeTLB pages. After the cluster host setup drops the page cache, host_setup reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its two owners' pages free; teardown releases them." - "LMCache in the prefill (256 GiB per stage): 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens), and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). Each decode has a node of its own." - - "host_setup drops the page cache and compacts free memory on every node before each point." + - "Workers load the checkpoint from the cluster's shared Hugging Face hub cache, which srtctl fills once before they start." - "Patch srt-slurm so its ATOM backend passes -tp as the worker's GPUs over its pipeline and prefill-context stages (NVIDIA/srt-slurm#604); unpatched, the PP4 prefill launches with -tp 4." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4045 diff --git a/inferencex-e2e/runners/srt-slurm/patches/atom-tp-from-pp.patch b/inferencex-e2e/runners/srt-slurm/patches/604-atom-tp-from-pp.patch similarity index 100% rename from inferencex-e2e/runners/srt-slurm/patches/atom-tp-from-pp.patch rename to inferencex-e2e/runners/srt-slurm/patches/604-atom-tp-from-pp.patch diff --git a/inferencex-e2e/runners/srt-slurm/patches/README.md b/inferencex-e2e/runners/srt-slurm/patches/README.md index f8b2e0ab4a..cba1bceb2b 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/README.md +++ b/inferencex-e2e/runners/srt-slurm/patches/README.md @@ -9,4 +9,4 @@ Each patch is a temporary fix for an open upstream PR. When the PR merges and th | Patch | Upstream PR | Fix | |-------|-------------|-----| | `573-participating-gpus.patch` | [NVIDIA/srt-slurm#573](https://github.com/NVIDIA/srt-slurm/pull/573) | Record only the GPUs workers occupy on a worker node; an exporter such as AMD's reports every GPU, which the collector otherwise flags as `unexpected_device`. | -| `atom-tp-from-pp.patch` | [NVIDIA/srt-slurm#604](https://github.com/NVIDIA/srt-slurm/pull/604) | ATOM's `-tp` is the worker's GPUs divided by its pipeline and prefill-context stages, so a 4-GPU PP4 prefill launches with `-tp 1` instead of `-tp 4` | +| `604-atom-tp-from-pp.patch` | [NVIDIA/srt-slurm#604](https://github.com/NVIDIA/srt-slurm/pull/604) | ATOM's `-tp` is the worker's GPUs divided by its pipeline and prefill-context stages, so a 4-GPU PP4 prefill launches with `-tp 1` instead of `-tp 4` | From f981a268a7043c4e0583984b92abd193f9859b00 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Wed, 14 Oct 2026 08:53:11 +0000 Subject: [PATCH 04/12] fix(agentx): read the GLM-5.2 checkpoint from each node's local hub cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mount /var/lib/hf-hub-cache at /mnt/hf_hub_cache and point HF_HUB_CACHE at it, as the single-node GLM-5.2 points do, so nodes that already hold amd/GLM-5.2-MXFP4 load it from local NVMe instead of downloading it into the shared cache. HF_HOME stays set so srtctl fills the first node's cache before workers start. 像单机 GLM-5.2 测试点一样，把 /var/lib/hf-hub-cache 挂载到 /mnt/hf_hub_cache，并让 HF_HUB_CACHE 指向它。已经有 amd/GLM-5.2-MXFP4 的 节点会直接从本地 NVMe 加载，不再下载到共享缓存。保留 HF_HOME，使 srtctl 在 worker 启动前先补齐第一个节点的缓存。 Co-Authored-By: Claude Opus 5.5 --- .../glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml | 9 ++++++--- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml index 8ec354facb..ef53bfa899 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -27,9 +27,9 @@ base: <<: &environment PYTHONUNBUFFERED: "1" PYTHONHASHSEED: "0" - # srtctl downloads an hf: checkpoint once into the shared hub cache only when HF_HOME is set. - HF_HOME: /hf_hub_cache - HF_HUB_CACHE: /hf_hub_cache/hub + # srtctl fills the first node's hub cache before workers start only when HF_HOME is set. + HF_HOME: /tmp/hf_home + HF_HUB_CACHE: /mnt/hf_hub_cache AITER_LOG_LEVEL: WARNING AITER_QUICK_REDUCE_QUANTIZATION: INT4 AITER_USE_FLYDSL_MOE_SORTING: "1" @@ -79,6 +79,9 @@ base: mem: "0" container-writable: "" container-remap-root: "" + extra_mount: + # The node-local hub cache the single-node GLM-5.2 points already fill. + - /var/lib/hf-hub-cache:/mnt/hf_hub_cache health_check: max_attempts: 1080 interval_seconds: 5 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index b7293859fe..2a42cf83cc 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9727,6 +9727,6 @@ - "Add GLM-5.2 MXFP4 on MI355X with ATOM prefill/decode disaggregation behind AToMesh, Mooncake RDMA from prefill to decode and MTP, on rocm/atom-dev:nightly_202610131249 (digest-pinned). The prefill is one CPP4 worker (PP4 x TP1 on four GPUs)." - "1P1D on one node at concurrency 56, 72, 96 and 112: a DCP4 decode on GPUs 4-7 with three draft tokens, and the prefill's prefix tier in a Mooncake Store (0.3.13) of four per-NIC pools, each one master and a 349 GiB owner on 2 MiB HugeTLB pages. After the cluster host setup drops the page cache, host_setup reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its two owners' pages free; teardown releases them." - "LMCache in the prefill (256 GiB per stage): 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens), and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). Each decode has a node of its own." - - "Workers load the checkpoint from the cluster's shared Hugging Face hub cache, which srtctl fills once before they start." + - "Workers read the checkpoint from each node's local Hugging Face hub cache, as the single-node GLM-5.2 points do; srtctl fills the first node's cache before they start." - "Patch srt-slurm so its ATOM backend passes -tp as the worker's GPUs over its pipeline and prefill-context stages (NVIDIA/srt-slurm#604); unpatched, the PP4 prefill launches with -tp 4." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4045 From 4aaf7c071e69d08c00af8aab17cd604411de7762 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Wed, 14 Oct 2026 09:14:26 +0000 Subject: [PATCH 05/12] fix(agentx): raise the Store owners' memlock limit before they register MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On mi355x-amds both Store points failed in the first owner: every ibv_reg_mr of its 349 GiB HugeTLB segment returned ENOMEM at once. The pyxis --container-remap-root user is root only inside its namespace, so the NIC's pin of the segment counts against RLIMIT_MEMLOCK. Run `ulimit -l unlimited` before each owner starts, as the DSv4 SGLang GB200 and GB300 recipes do for their Mooncake stores. 两个 Store 测试点都在 mi355x-amds 上的第一个 owner 处失败：它对 349 GiB HugeTLB 段的每次 ibv_reg_mr 都立刻返回 ENOMEM。pyxis 的 --container-remap-root 用户只在自己的命名空间内是 root，网卡锁定该段时会 计入 RLIMIT_MEMLOCK。现在每个 owner 启动前先执行 `ulimit -l unlimited`， 与 DSv4 SGLang GB200、GB300 配方对 Mooncake store 的做法一致。 Co-Authored-By: Claude Opus 5.5 --- .../glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml index ef53bfa899..52470a7ac9 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -161,6 +161,8 @@ override_1p1d_mcstore: MC_STORE_USE_HUGEPAGE: "1" MC_MAX_MR_SIZE: "68719476736" MC_TCP_BIND_ADDRESS: "{node_ip}" + # The remapped container root lacks host CAP_IPC_LOCK, so registering the segment counts against RLIMIT_MEMLOCK. + preamble: ulimit -l unlimited readiness: log: pattern: Starting real client service From 4f75d9416e3301c6fe788417702951d3f97430f3 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Wed, 14 Oct 2026 09:28:56 +0000 Subject: [PATCH 06/12] fix(agentx): role-aware, window-scoped ATOM cache rates with validity (cherry-pick #3964) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carry #3964 (2db01926, 0fda3f1e, 0842d829) so this PR's sweep prices the ATOM P/D cache rates with it: count only prefill endpoints, take the CPU rate from atom:prefix_cache_offload_tokens (reported for LMCache and the Mooncake Store alike), and slice counters to the profiling window. Applied as #3964's net diff onto current main; drop it once #3964 merges. 携带 #3964（2db01926、0fda3f1e、0842d829），使本 PR 的 sweep 用它来计算 ATOM P/D 的缓存命中率：只统计 prefill 端点，CPU 命中率取自 atom:prefix_cache_offload_tokens（LMCache 与 Mooncake Store 都会上报）， 并按 profiling 窗口切分计数。以 #3964 相对其基线的净改动应用到当前 main 上； #3964 合入后即可移除。 Co-Authored-By: Claude Opus 5.5 --- .../infx/results/agentic/__init__.py | 11 +- .../infx/results/agentic/artifacts.py | 3 + .../infx/results/agentic/backends/atom.py | 354 +++++++++++-- .../infx/results/agentic/backends/base.py | 1 + .../infx/results/agentic/server_metrics.py | 86 ++- .../infx/tests/bench/test_agentic_command.py | 98 +++- .../agentic/test_atom_cache_accounting.py | 493 ++++++++++++++++++ .../agentic/test_process_agentic_result.py | 338 ++++++++---- 8 files changed, 1247 insertions(+), 137 deletions(-) create mode 100644 inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py diff --git a/inferencex-e2e/infx/results/agentic/__init__.py b/inferencex-e2e/infx/results/agentic/__init__.py index 7cb92a82ff..a378329a28 100644 --- a/inferencex-e2e/infx/results/agentic/__init__.py +++ b/inferencex-e2e/infx/results/agentic/__init__.py @@ -9,7 +9,12 @@ from infx.results.topology import Parallelism, validate_parallelism from .request_metrics import compute_request_metrics -from .server_metrics import compute_server_metrics +from .server_metrics import ( + compute_server_metrics, + deployment_from_env, + endpoint_roles_from_env, + request_scope_from_records, +) def _env_int(env: Mapping[str, str], name: str, default: int = 0) -> int: @@ -167,6 +172,7 @@ def build_result( "records_dropped_total": 0, "records_warmup_dropped": 0, "records_error_dropped": 0, + "records_profiling_error_dropped": 0, "error_categories": {}, } @@ -216,6 +222,9 @@ def build_result( framework=framework, records=records, server_logs=server_logs, + endpoint_roles=endpoint_roles_from_env(env), + request_scope=request_scope_from_records(records, request_accounting), + deployment=deployment_from_env(env), ) if "total_tput_tps" in request_flat and num_gpus > 0: diff --git a/inferencex-e2e/infx/results/agentic/artifacts.py b/inferencex-e2e/infx/results/agentic/artifacts.py index a35ee829ae..ef072d274a 100644 --- a/inferencex-e2e/infx/results/agentic/artifacts.py +++ b/inferencex-e2e/infx/results/agentic/artifacts.py @@ -45,6 +45,7 @@ def load_records_with_accounting( "records_dropped_total": 0, "records_warmup_dropped": 0, "records_error_dropped": 0, + "records_profiling_error_dropped": 0, "error_categories": {}, } with open(path) as f: @@ -61,6 +62,8 @@ def load_records_with_accounting( accounting["records_warmup_dropped"] += 1 if error: accounting["records_error_dropped"] += 1 + if not is_warmup: + accounting["records_profiling_error_dropped"] += 1 category = _error_category(error) categories = accounting["error_categories"] categories[category] = categories.get(category, 0) + 1 diff --git a/inferencex-e2e/infx/results/agentic/backends/atom.py b/inferencex-e2e/infx/results/agentic/backends/atom.py index 947024b111..681d502f7c 100644 --- a/inferencex-e2e/infx/results/agentic/backends/atom.py +++ b/inferencex-e2e/infx/results/agentic/backends/atom.py @@ -2,12 +2,25 @@ from __future__ import annotations +import math +from collections.abc import Mapping from typing import Any -from infx.results.agentic.common import gauge_stat, normalize_fraction, rate, sum_stat +from infx.results.agentic.common import gauge_stat, normalize_fraction, series_stat, sum_stat from .base import ServerMetricsBackend, counter_int +# P/D decode workers re-count prefilled prompts in prefix_cache_full_tokens without reuse. +ACCOUNTED_ROLES = frozenset({"prefill", "unified"}) +FULL = "prefix_cache_full_tokens" +CACHED = "prefix_cache_cached_tokens" +OFFLOAD = "prefix_cache_offload_tokens" +LOADED = "lmcache_loaded_tokens" +REQUESTS = "prefix_cache_requests" +_COUNTERS = (FULL, CACHED, OFFLOAD, LOADED, REQUESTS) +VALIDITY_SCHEMA = "atom-cache-validity/1" +PRICING_STATUSES = frozenset({"VALID_COMPLETE", "VALID_ZERO"}) + def _atom_names(suffix: str) -> list[str]: """Accept direct worker metrics and Atomesh's colon-normalized form.""" @@ -19,8 +32,269 @@ def _atom_counter_names(stem: str) -> list[str]: return [*_atom_names(stem), *_atom_names(f"{stem}_total")] +def _series_by_endpoint( + section: Mapping[str, Any], stem: str +) -> tuple[dict[str, dict[str, Any]], list[str]]: + """One series per endpoint across all name aliases; a second one is a duplicate.""" + by_endpoint: dict[str, dict[str, Any]] = {} + duplicates: list[str] = [] + for name in _atom_counter_names(stem): + entry = section.get(name) + if not isinstance(entry, dict): + continue + for series in entry.get("series") or []: + url = series.get("endpoint_url") if isinstance(series, dict) else None + if not isinstance(url, str): + continue + if url in by_endpoint: + duplicates.append(url) + by_endpoint[url] = series + return by_endpoint, duplicates + + +def _slices(series: dict[str, Any]) -> list[tuple[int, int, float]] | None: + """Ordered, non-overlapping, finite, non-negative AIPerf timeslices, or None.""" + raw = series.get("timeslices") + if not isinstance(raw, list) or not raw: + return None + out: list[tuple[int, int, float]] = [] + for item in raw: + if not isinstance(item, dict): + return None + lo, hi, value = item.get("start_ns"), item.get("end_ns"), item.get("total") + if not isinstance(lo, int) or not isinstance(hi, int) or hi <= lo: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + if not math.isfinite(value) or value < 0: + return None + out.append((lo, hi, float(value))) + out.sort() + if any(out[i][0] < out[i - 1][1] for i in range(1, len(out))): + return None + return out + + +def _window_parts(slices: list[tuple[int, int, float]], start: int, end: int) -> dict[str, Any]: + """Split slice totals around [start, end]; flag sampling gaps inside the window.""" + parts = {"before": 0.0, "start_straddle": 0.0, "inside": 0.0, "end_straddle": 0.0} + covering = [s for s in slices if s[1] > start and s[0] < end] + gap = ( + not covering + or covering[0][0] > start + or covering[-1][1] < end + or any(covering[i][0] != covering[i - 1][1] for i in range(1, len(covering))) + ) + for lo, hi, value in slices: + if hi <= start: + parts["before"] += value + elif lo >= start and hi <= end: + parts["inside"] += value + else: + if lo < start < hi: + parts["start_straddle"] += value + if lo < end < hi and not lo < start < hi: + parts["end_straddle"] += value + parts["gap"] = gap + return parts + + +def account_cache( + metrics: dict[str, dict[str, Any]], + *, + framework: str, + endpoint_roles: Mapping[str, str] | None, + profiling_window: tuple[int, int] | None, + warmup_metrics: Mapping[str, Any] | None = None, + request_scope: Mapping[str, Any] | None = None, + disagg: bool | None = None, + kv_offloading: str | None = None, +) -> dict[str, Any]: + """Prefix-cache reuse on prefill (or unified) workers inside the profiling window. + + GPU = cached/full and CPU = offload/full on the same endpoints and timeslices. ATOM records + LMCache reuse in prefix_cache_offload_tokens, disjoint from the HBM `cached` counter, so + overall = GPU + CPU. lmcache_loaded_tokens is transfer volume and stays diagnostic only. + `validity` tells consumers whether the rates are a complete, same-scope actual observation + (`pricing_eligible`); warnings alone are not the guard. + """ + result: dict[str, Any] = dict.fromkeys( + ("gpu", "cpu", "overall", "full", "cached", "offload", "loaded", "requests") + ) + reasons: list[str] = [] + validity: dict[str, Any] = { + "schema": VALIDITY_SCHEMA, + "status": None, + "pricing_eligible": False, + "observed_scope": None, + "requested_scope": "profiling_successful_requests", + "cpu_component": None, + "roles_source": None, + "accounted_endpoints": [], + "window": None, + "request_scope": None, + "diagnostics": {}, + "reasons": reasons, + } + result.update(validity=validity, warnings=[]) + + def refuse(status: str, reason: str) -> dict[str, Any]: + validity["status"] = status + reasons.append(reason) + result["warnings"].append(f"atom cache accounting: {reason}") + return result + + series: dict[str, dict[str, dict[str, Any]]] = {} + for stem in _COUNTERS: + series[stem], duplicates = _series_by_endpoint(metrics, stem) + if duplicates: + return refuse("INVALID_COUNTER", f"duplicate {stem} series for {duplicates}") + observed = sorted(series[FULL]) + if not observed: + return refuse("MISSING_OBSERVATION", "no prefix_cache_full_tokens series") + + if endpoint_roles: + unmapped = [url for url in observed if url not in endpoint_roles] + accounted = sorted(u for u, role in endpoint_roles.items() if role in ACCOUNTED_ROLES) + missing = [url for url in accounted if url not in observed] + if unmapped or missing or not accounted: + return refuse( + "ROLE_OR_COVERAGE_UNRESOLVED", + "endpoint roles do not cover the scraped endpoints " + f"(unmapped={unmapped}, missing={missing})", + ) + validity["roles_source"] = "launcher_env" + elif disagg is False and len(observed) == 1: + accounted = observed + validity["roles_source"] = "single_endpoint_non_disagg" + else: + return refuse( + "ROLE_OR_COVERAGE_UNRESOLVED", + f"{framework or 'atom'}: no launcher endpoint roles and not a proven single unified " + "instance (SRT_PREFILL_ENDPOINTS/SRT_DECODE_ENDPOINTS or SRT_AGG_ENDPOINTS needed)", + ) + validity["accounted_endpoints"] = accounted + + for stem in (FULL, CACHED): + if any(url not in series[stem] for url in accounted): + return refuse("MISSING_OBSERVATION", f"{stem} missing on accounted endpoints") + cumulative = {} + for stem in _COUNTERS: + values = [series_stat(series[stem][u]) for u in accounted if u in series[stem]] + if values and len(values) == len(accounted) and None not in values: + cumulative[f"atom:{stem}"] = float(sum(values)) + validity["diagnostics"]["cumulative_stats_total"] = cumulative + + if profiling_window is None: + return refuse("WINDOW_UNRESOLVED", "no summary.phase_time_ranges.profiling window") + start, end = profiling_window + totals: dict[str, float | None] = {} + boundary: list[str] = [] + for stem in _COUNTERS: + present = [u for u in accounted if u in series[stem]] + if not present: + totals[stem] = None + continue + if len(present) != len(accounted): + totals[stem] = None + continue + inside = 0.0 + for url in present: + slices = _slices(series[stem][url]) + if slices is None: + return refuse("WINDOW_UNRESOLVED", f"{stem}@{url}: timeslices missing or invalid") + parts = _window_parts(slices, start, end) + if parts["gap"]: + return refuse("WINDOW_UNRESOLVED", f"{stem}@{url}: sampling gap in the window") + inside += parts["inside"] + if stem in (LOADED, REQUESTS): + continue + if parts["end_straddle"] > 0: + boundary.append(f"{stem}@{url}: activity in the slice straddling profiling end") + if parts["start_straddle"] > 0: + warm_series, _ = _series_by_endpoint(warmup_metrics or {}, stem) + warm = series_stat(warm_series[url]) if url in warm_series else None + if warm is None or parts["before"] + parts["start_straddle"] != warm: + boundary.append( + f"{stem}@{url}: activity in the slice straddling profiling start is not " + "proven warmup-only by warmup_metrics" + ) + totals[stem] = inside + validity["observed_scope"] = "profiling_window_interior_timeslices" + validity["window"] = {"start_ns": start, "end_ns": end, "boundary_issues": boundary} + + full, cached = totals[FULL], totals[CACHED] + offload, loaded, requests = totals[OFFLOAD], totals[LOADED], totals[REQUESTS] + if full is None or cached is None: + return refuse("MISSING_OBSERVATION", "cached/full counters missing on accounted endpoints") + if offload is not None: + validity["cpu_component"] = "MEASURED" + elif (kv_offloading or "").lower() == "none": + validity["cpu_component"] = "NO_CPU_TIER_CONFIRMED" + offload = 0.0 + else: + validity["cpu_component"] = "MISSING" + if not math.isfinite(full) or full <= 0: + return refuse("INVALID_COUNTER", "no prompt tokens in the accounting window") + if not math.isfinite(cached) or not 0 <= cached <= full: + return refuse( + "INVALID_COUNTER", f"cached tokens ({cached:.0f}) outside [0, full ({full:.0f})]" + ) + if offload is not None and ( + not math.isfinite(offload) or offload < 0 or not cached + offload <= full + ): + return refuse( + "INVALID_COUNTER", + f"cached+offload ({cached + offload:.0f}) exceeds full tokens ({full:.0f})", + ) + result.update(full=full, cached=cached, offload=offload, loaded=loaded, requests=requests) + result["gpu"] = cached / full + if offload is not None: + result.update(cpu=offload / full, overall=(cached + offload) / full) + + scope = dict(request_scope or {}) + client_requests = scope.get("profiling_requests") + client_tokens = scope.get("profiling_prompt_tokens") + conserved = ( + requests is not None + and client_tokens is not None + and requests == client_requests + and full == client_tokens + and not scope.get("error_records") + and not scope.get("cancelled_records") + ) + validity["request_scope"] = { + "server_window_admissions": requests, + "server_window_prompt_tokens": full, + **scope, + "conserved": conserved, + } + + if boundary: + reasons.extend(boundary) + validity["status"] = "WINDOW_UNRESOLVED" + elif validity["cpu_component"] == "MISSING": + reasons.append("CPU reuse counter missing and no confirmed absence of a CPU tier") + validity["status"] = "PARTIAL_COMPONENT" + elif not conserved: + reasons.append( + "server window admissions/prompt tokens do not match the successful profiling " + f"requests ({requests} vs {client_requests}; {full} vs {client_tokens})" + ) + validity["status"] = "REQUEST_SCOPE_UNRESOLVED" + else: + validity["status"] = "VALID_ZERO" if cached + offload == 0 else "VALID_COMPLETE" + validity["pricing_eligible"] = validity["status"] in PRICING_STATUSES + if not validity["pricing_eligible"]: + result["warnings"].append( + f"atom cache accounting: {validity['status']}; rates are telemetry, not pricing input" + ) + return result + + class AtomBackend(ServerMetricsBackend): name = "atom" + accepts_context = True def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: metric_names = set(metrics) @@ -33,6 +307,8 @@ def populate( metrics: dict[str, dict[str, Any]], flat: dict[str, Any], nested: dict[str, Any], + *, + context: Mapping[str, Any] | None = None, ) -> None: prompt_total = sum_stat( metrics, @@ -47,29 +323,26 @@ def populate( flat["total_prompt_tokens"] = counter_int(prompt_total) flat["total_generation_tokens"] = counter_int(generation_total) - cached_tokens = sum_stat( - metrics, - _atom_counter_names("prefix_cache_cached_tokens"), - preferred_keys=("total", "sum", "max", "avg"), - ) - full_tokens = sum_stat( - metrics, - _atom_counter_names("prefix_cache_full_tokens"), - preferred_keys=("total", "sum", "max", "avg"), - ) - cache_hit_rate = rate(cached_tokens, full_tokens) - external_tokens = sum_stat( + context = context or {} + accounting = account_cache( metrics, - _atom_counter_names("lmcache_loaded_tokens"), - preferred_keys=("total", "sum", "max", "avg"), + framework=str(context.get("framework") or ""), + endpoint_roles=context.get("endpoint_roles"), + profiling_window=context.get("profiling_window"), + warmup_metrics=context.get("warmup_metrics"), + request_scope=context.get("request_scope"), + disagg=context.get("disagg"), + kv_offloading=context.get("kv_offloading"), ) - external_hit_rate = rate(external_tokens, full_tokens) - overall_hit_rate = cache_hit_rate + sink = context.get("warnings") + if isinstance(sink, list): + sink.extend(accounting["warnings"]) - flat["server_gpu_cache_hit_rate"] = cache_hit_rate - flat["server_cpu_cache_hit_rate"] = external_hit_rate - flat["server_external_cache_hit_rate"] = external_hit_rate - flat["server_overall_cache_hit_rate"] = overall_hit_rate + flat["server_gpu_cache_hit_rate"] = accounting["gpu"] + flat["server_cpu_cache_hit_rate"] = accounting["cpu"] + # LocalCPUBackend is ATOM's only external tier, so external aliases the CPU reuse. + flat["server_external_cache_hit_rate"] = accounting["cpu"] + flat["server_overall_cache_hit_rate"] = accounting["overall"] flat["gpu_kv_cache_usage_pct"] = normalize_fraction( gauge_stat( metrics, @@ -81,16 +354,39 @@ def populate( nested["cache"].update( { - "gpu_cache_hit_rate": cache_hit_rate, - "cpu_cache_hit_rate": external_hit_rate, - "external_cache_hit_rate": external_hit_rate, - "overall_cache_hit_rate": overall_hit_rate, - "prefix_cache_hits": cached_tokens, - "prefix_cache_queries": full_tokens, - "external_prefix_cache_hits": external_tokens, - "external_prefix_cache_queries": full_tokens, + "gpu_cache_hit_rate": accounting["gpu"], + "cpu_cache_hit_rate": accounting["cpu"], + "external_cache_hit_rate": accounting["cpu"], + "overall_cache_hit_rate": accounting["overall"], + "prefix_cache_hits": accounting["cached"], + "prefix_cache_queries": accounting["full"], + "external_prefix_cache_hits": accounting["offload"], + "external_prefix_cache_queries": ( + accounting["full"] if accounting["offload"] is not None else None + ), } ) + nested["cache_validity"] = accounting["validity"] + by_source = nested["tokens"]["prompt_by_source"] + if accounting["full"] is not None and accounting["offload"] is not None: + by_source.update( + { + "gpu_cache_hit": accounting["cached"], + "cpu_or_external_cache_hit": accounting["offload"], + "computed": accounting["full"] - accounting["cached"] - accounting["offload"], + } + ) + by_source["raw"] = { + f"atom:{stem}": accounting[key] + for stem, key in ( + (FULL, "full"), + (CACHED, "cached"), + (OFFLOAD, "offload"), + (LOADED, "loaded"), + (REQUESTS, "requests"), + ) + if accounting[key] is not None + } nested["kv_cache"]["gpu_usage_pct"] = flat["gpu_kv_cache_usage_pct"] nested["tokens"].update( { diff --git a/inferencex-e2e/infx/results/agentic/backends/base.py b/inferencex-e2e/infx/results/agentic/backends/base.py index 427a4c1606..370897dd92 100644 --- a/inferencex-e2e/infx/results/agentic/backends/base.py +++ b/inferencex-e2e/infx/results/agentic/backends/base.py @@ -67,6 +67,7 @@ def empty_server_metrics(present: bool, metric_count: int) -> dict[str, Any]: "frontend_cached_tokens": None, "frontend_input_tokens": None, }, + "cache_validity": None, "kv_cache": { "gpu_usage_pct": None, "gpu_total_tokens": None, diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index d14962a61f..46cbc17684 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import Any from .backends import detect_backend @@ -12,6 +12,72 @@ empty_server_metrics, ) from .common import index_server_metrics +from .request_metrics import extract_per_record_ints + + +def request_scope_from_records( + records: list[dict[str, Any]], request_accounting: Mapping[str, Any] +) -> dict[str, Any]: + """Client-side profiling cohort in server token units (``usage_prompt_tokens``). + + ``profiling_prompt_tokens`` is None when any successful record lacks usage, so a partial + sum is never compared with server counters. ``error_records`` counts failed profiling + requests; accounting without that split falls back to the all-phase error total. + """ + prompt = extract_per_record_ints(records, "usage_prompt_tokens") + return { + "profiling_requests": len(records), + "profiling_prompt_tokens": sum(prompt) if len(prompt) == len(records) else None, + "error_records": request_accounting.get( + "records_profiling_error_dropped", request_accounting.get("records_error_dropped") + ), + "cancelled_records": sum( + 1 for record in records if (record.get("metadata") or {}).get("was_cancelled") + ), + } + + +def deployment_from_env(env: Mapping[str, str]) -> dict[str, Any]: + """Launcher-declared topology facts; absent or unrecognised values stay unknown (None).""" + disagg = {"true": True, "false": False}.get((env.get("DISAGG") or "").lower()) + return {"disagg": disagg, "kv_offloading": env.get("KV_OFFLOADING") or None} + + +def endpoint_roles_from_env(env: Mapping[str, str]) -> dict[str, str] | None: + """Map each scraped metrics URL to its launcher role, or None when roles are unknown. + + Mirrors ``infx.bench.agentic.replay._server_metrics_urls``: aggregated workers are + ``unified``; otherwise prefill endpoints precede decode endpoints. Caller-supplied + AIPERF_SERVER_METRICS_URLS and Dynamo frontends carry no roles, and an endpoint listed + under two roles is ambiguous. + """ + if env.get("AIPERF_SERVER_METRICS_URLS") or env.get("SRTCTL_FRONTEND_TYPE") == "dynamo": + return None + if env.get("SRT_AGG_ENDPOINTS"): + groups = [("unified", env["SRT_AGG_ENDPOINTS"])] + else: + groups = [ + ("prefill", env.get("SRT_PREFILL_ENDPOINTS", "")), + ("decode", env.get("SRT_DECODE_ENDPOINTS", "")), + ] + roles: dict[str, str] = {} + for role, endpoints in groups: + for endpoint in endpoints.split(","): + if endpoint and roles.setdefault(f"http://{endpoint}/metrics", role) != role: + return None + return roles or None + + +def _profiling_window(server_metrics: dict[str, Any]) -> tuple[int, int] | None: + summary = server_metrics.get("summary") + ranges = summary.get("phase_time_ranges") if isinstance(summary, dict) else None + profiling = ranges.get("profiling") if isinstance(ranges, dict) else None + if not isinstance(profiling, dict): + return None + start, end = profiling.get("start_ns"), profiling.get("end_ns") + if isinstance(start, int) and isinstance(end, int) and start < end: + return start, end + return None def compute_server_metrics( @@ -20,6 +86,9 @@ def compute_server_metrics( framework: str, records: list[dict[str, Any]], server_logs: Iterable[str | None] = (), + endpoint_roles: Mapping[str, str] | None = None, + request_scope: Mapping[str, Any] | None = None, + deployment: Mapping[str, Any] | None = None, ) -> tuple[dict[str, Any], dict[str, Any], list[str]]: metrics = index_server_metrics(server_metrics) flat = SERVER_CACHE_FLAT_FIELDS.copy() @@ -46,7 +115,20 @@ def compute_server_metrics( f"framework={framework.lower()!r}, metric_names=[{preview}]" ) nested["adapter"] = adapter.name - adapter.populate(metrics, flat, nested) + if getattr(adapter, "accepts_context", False): + warmup = server_metrics.get("warmup_metrics") + context = { + "framework": framework, + "endpoint_roles": endpoint_roles, + "profiling_window": _profiling_window(server_metrics), + "warmup_metrics": warmup if isinstance(warmup, dict) else None, + "request_scope": request_scope, + **dict(deployment or {}), + "warnings": warnings, + } + adapter.populate(metrics, flat, nested, context=context) + else: + adapter.populate(metrics, flat, nested) if framework.lower() == "dynamo-sglang" and adapter.name == "sglang": nested["kv_cache"]["gpu_total_tokens"] = None diff --git a/inferencex-e2e/infx/tests/bench/test_agentic_command.py b/inferencex-e2e/infx/tests/bench/test_agentic_command.py index 260809124b..b37558fb83 100644 --- a/inferencex-e2e/infx/tests/bench/test_agentic_command.py +++ b/inferencex-e2e/infx/tests/bench/test_agentic_command.py @@ -170,7 +170,9 @@ def test_standalone_srt_slurm_point_needs_no_kv_offload_declaration(tmp_path): WINDOW = { - "IS_MULTINODE": "true", "ENABLE_AGENTX_POWER": "1", "REQUIRE_POWER": "0", + "IS_MULTINODE": "true", + "ENABLE_AGENTX_POWER": "1", + "REQUIRE_POWER": "0", "SRT_MEASUREMENT_WINDOW_DIR": "/w", } MISSING = {"IS_MULTINODE": "true", "ENABLE_AGENTX_POWER": "1", "REQUIRE_POWER": "1"} @@ -439,3 +441,97 @@ def frontend(method: str, path: str) -> tuple[int, object]: assert "fake aiperf: profiled 1 request" in result.stdout aggregate = json.loads((tmp_path / "agentx.json").read_text()) assert (aggregate["conc"], aggregate["num_requests_total"]) == (8, 1) + + +FAKE_ATOM_AIPERF_SCRIPT = ( + FAKE_AIPERF_SCRIPT + + r""" +record["metrics"]["usage_prompt_tokens"] = {"value": 100, "unit": "tokens"} +(out / "profile_export.jsonl").write_text(json.dumps(record) + "\n") +urls = argv[argv.index("--server-metrics") + 1 :] +urls = urls[: next((i for i, url in enumerate(urls) if url.startswith("--")), len(urls))] +prefill = "http://" + os.environ["SRT_PREFILL_ENDPOINTS"] + "/metrics" +edges = [(0, 1000), (1000, 2500), (2500, 4000)] +counters = { + "prefix_cache_full_tokens": (0, 40, 60), + "prefix_cache_cached_tokens": (0, 20, 40), + "prefix_cache_offload_tokens": (0, 5, 5), + "prefix_cache_requests": (0, 1, 0), +} +def series(url, values): + slices = [{"start_ns": lo, "end_ns": hi, "total": v} for (lo, hi), v in zip(edges, values)] + return {"endpoint_url": url, "stats": {"total": sum(values)}, "timeslices": slices} +metrics = { + f"atom:{name}": { + "type": "counter", + "series": [series(url, values if url == prefill else (0, 50, 50)) for url in urls], + } + for name, values in counters.items() +} +summary = {"phase_time_ranges": {"profiling": {"start_ns": 1000, "end_ns": 4000}}} +(out / "server_metrics_export.json").write_text(json.dumps({"summary": summary, "metrics": metrics})) +""" +) + + +def test_srt_shim_hands_launcher_endpoint_roles_to_the_atom_producer(tmp_path, http_server): + tools = tmp_path / "tools" + tools.mkdir() + executable(tools / "uv", FAKE_UV) + (tools / "python3").symlink_to(sys.executable) + script = tmp_path / "fake_aiperf.py" + script.write_text(FAKE_ATOM_AIPERF_SCRIPT) + + def frontend(method: str, path: str) -> tuple[int, object]: + if path == "/v1/models": + return 200, {"data": [{"id": "served/model"}]} + return (405 if path == "/v1/chat/completions" else 200), {} + + workers = { + f"{side}_{key}": value + for side in ("PREFILL", "DECODE") + for key, value in { + "NUM_WORKERS": "1", + "TP": "8", + "PP_SIZE": "1", + "DCP_SIZE": "1", + "PCP_SIZE": "1", + "EP": "1", + "DP_ATTN": "false", + }.items() + } + env = _point( + tmp_path, + PATH=f"{tools}:/usr/bin:/bin", + HOME=str(tmp_path), + FAKE_AIPERF_SCRIPT=str(script), + REPLAY_ARGV=str(tmp_path / "replay-argv"), + HF_HUB_CACHE=str(tmp_path / "hf"), + AIPERF_RUNTIME_DIR=str(tmp_path / "runtime"), + EVAL_ONLY="true", + AIPERF_SERVER_URL=http_server(frontend), + SERVED_MODEL_NAME="served/model", + EVAL_ENDPOINT_READY_TIMEOUT_SECONDS="30", + EVAL_MODEL_STABILIZATION_SECONDS="0", + FRAMEWORK="atom-disagg", + DISAGG="true", + SRT_PREFILL_ENDPOINTS="10.0.0.1:6100", + SRT_DECODE_ENDPOINTS="10.0.0.2:6100", + **workers, + ) + + result = subprocess.run( + ["bash", str(SHIM)], env=env, capture_output=True, text=True, timeout=120, check=False + ) + + assert result.returncode == 0, result.stderr + argv = shlex.split((tmp_path / "replay-argv").read_text()) + scraped = argv[argv.index("--server-metrics") + 1 : argv.index("--server-metrics") + 3] + assert scraped == ["http://10.0.0.1:6100/metrics", "http://10.0.0.2:6100/metrics"] + server_metrics = json.loads((tmp_path / "agentx.json").read_text())["server_metrics"] + validity = server_metrics["cache_validity"] + assert validity["roles_source"] == "launcher_env" + assert validity["accounted_endpoints"] == ["http://10.0.0.1:6100/metrics"] + assert (validity["status"], validity["pricing_eligible"]) == ("VALID_COMPLETE", True) + cache = server_metrics["cache"] + assert (cache["gpu_cache_hit_rate"], cache["cpu_cache_hit_rate"]) == (0.6, 0.1) diff --git a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py new file mode 100644 index 0000000000..b82c25f1d3 --- /dev/null +++ b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py @@ -0,0 +1,493 @@ +from __future__ import annotations + +import json +import math +from pathlib import Path + +import pytest + +from infx.bench.agentic.replay import _server_metrics_urls +from infx.results.agentic.server_metrics import compute_server_metrics, endpoint_roles_from_env +from infx.tests.results.agentic.test_process_agentic_result import ( + _assert_stable_server_metrics_schema, + _run_processor, + _write_fixture, +) + +PREFILL = "http://10.0.0.1:6100/metrics" +DECODE = "http://10.0.0.2:6100/metrics" +ROLES = {PREFILL: "prefill", DECODE: "decode"} +WINDOW = (1_000, 4_000) +# warmup | profiling | profiling | tail: no slice straddles the window edges. +EDGES = ((0, 1_000), (1_000, 2_500), (2_500, 4_000), (4_000, 5_000)) +ZERO = (0.0, 0.0, 0.0, 0.0) +SCOPE = {"profiling_requests": 5, "profiling_prompt_tokens": 1000, "error_records": 0} +PD = {"disagg": True, "kv_offloading": "dram"} +RATE_KEYS = ( + "gpu_cache_hit_rate", + "cpu_cache_hit_rate", + "external_cache_hit_rate", + "overall_cache_hit_rate", +) +NO_RATES = (None, None, None, None) + + +def _series(url: str, values: tuple, edges: tuple = EDGES) -> dict: + slices = [{"start_ns": lo, "end_ns": hi, "total": v} for (lo, hi), v in zip(edges, values)] + return {"endpoint_url": url, "stats": {"total": sum(values)}, "timeslices": slices} + + +def _export(counters: dict[str, dict[str, tuple]], edges: tuple = EDGES, window=WINDOW) -> dict: + metrics = { + f"atom:{stem}": {"type": "counter", "series": [_series(u, v, edges) for u, v in ep.items()]} + for stem, ep in counters.items() + } + export: dict = {"metrics": metrics} + if window is not None: + profiling = {"start_ns": window[0], "end_ns": window[1]} + export["summary"] = {"phase_time_ranges": {"profiling": profiling}} + return export + + +def _pd(offload: tuple | None = (0, 50, 50, 0), **override: tuple) -> dict: + """Interior prefill: full 1000, cached 600, offload 100, loaded 150, 5 admissions.""" + prefill = { + "prefix_cache_full_tokens": (0, 400, 600, 0), + "prefix_cache_cached_tokens": (0, 200, 400, 0), + "prefix_cache_offload_tokens": offload, + "lmcache_loaded_tokens": (0, 70, 80, 0), + "prefix_cache_requests": (0, 3, 2, 0), + } | override + decode = {"prefix_cache_full_tokens": (0, 500, 650, 0), "prefix_cache_requests": (0, 3, 2, 0)} + return _export( + { + stem: {PREFILL: values, DECODE: decode.get(stem, ZERO)} + for stem, values in prefill.items() + if values is not None + } + ) + + +def _run(export: dict, *, roles=ROLES, scope=SCOPE, deployment=PD, framework="atom-disagg"): + _, nested, warnings = compute_server_metrics( + export, + framework=framework, + records=[], + endpoint_roles=roles, + request_scope=scope, + deployment=deployment, + ) + return nested, warnings + + +def _rates(nested: dict) -> tuple: + return tuple(nested["cache"][key] for key in RATE_KEYS) + + +def test_complete_pd_observation_is_eligible_and_ignores_decode_and_load_volume() -> None: + nested, warnings = _run(_pd()) + validity = nested["cache_validity"] + + assert _rates(nested) == (0.6, 0.1, 0.1, pytest.approx(0.7)) + assert nested["cache"]["prefix_cache_queries"] == 1000 + assert nested["tokens"]["prompt_by_source"]["raw"]["atom:lmcache_loaded_tokens"] == 150 + assert validity["status"] == "VALID_COMPLETE" + assert validity["pricing_eligible"] is True + assert validity["observed_scope"] == "profiling_window_interior_timeslices" + assert validity["accounted_endpoints"] == [PREFILL] + assert validity["request_scope"]["conserved"] is True + assert warnings == [] + + +@pytest.mark.parametrize( + ("kv_offloading", "status", "cpu", "eligible"), + [ + ("dram", "PARTIAL_COMPONENT", None, False), + (None, "PARTIAL_COMPONENT", None, False), + ("none", "VALID_COMPLETE", 0.0, True), + ], +) +def test_missing_cpu_counter_is_partial_unless_no_cpu_tier_is_configured( + kv_offloading: str | None, status: str, cpu: float | None, eligible: bool +) -> None: + nested, _ = _run(_pd(offload=None), deployment={"disagg": True, "kv_offloading": kv_offloading}) + + assert nested["cache"]["gpu_cache_hit_rate"] == 0.6 + assert nested["cache"]["cpu_cache_hit_rate"] == cpu + assert nested["cache_validity"]["status"] == status + assert nested["cache_validity"]["pricing_eligible"] is eligible + + +@pytest.mark.parametrize( + ("offload", "kv_offloading"), + [(None, "none"), ((0, 0, 0, 0), "dram")], +) +def test_cached_above_full_is_refused_on_every_cpu_branch(offload, kv_offloading) -> None: + deployment = {"disagg": True, "kv_offloading": kv_offloading} + export = _pd(offload=offload, prefix_cache_cached_tokens=(0, 500, 700, 0)) + + nested, warnings = _run(export, deployment=deployment) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "INVALID_COUNTER" + assert nested["cache_validity"]["pricing_eligible"] is False + assert any("cached" in w for w in warnings) + + +def test_missing_counter_contract_zero_matches_an_explicit_zero() -> None: + no_cpu_tier = {"disagg": True, "kv_offloading": "none"} + + explicit, _ = _run(_pd(offload=(0, 0, 0, 0)), deployment=no_cpu_tier) + missing, _ = _run(_pd(offload=None), deployment=no_cpu_tier) + + assert _rates(explicit) == _rates(missing) == (0.6, 0.0, 0.0, 0.6) + assert explicit["cache_validity"]["cpu_component"] == "MEASURED" + assert missing["cache_validity"]["cpu_component"] == "NO_CPU_TIER_CONFIRMED" + assert explicit["cache_validity"]["pricing_eligible"] is True + assert missing["cache_validity"]["pricing_eligible"] is True + + +@pytest.mark.parametrize( + ("cached", "offload", "status", "rates"), + [ + ((0, 0, 0, 0), (0, 0, 0, 0), "VALID_ZERO", (0.0, 0.0, 0.0, 0.0)), + ((0, 0, 0, 0), (0, 100, 150, 0), "VALID_COMPLETE", (0.0, 0.25, 0.25, 0.25)), + ], +) +def test_zero_and_cpu_only_reuse_are_eligible_measurements(cached, offload, status, rates) -> None: + nested, _ = _run(_pd(offload=offload, prefix_cache_cached_tokens=cached)) + + assert _rates(nested) == rates + assert nested["cache_validity"]["status"] == status + assert nested["cache_validity"]["pricing_eligible"] is True + + +@pytest.mark.parametrize( + ("roles", "deployment", "reason"), + [ + (None, PD, "no launcher endpoint roles"), + (None, {"disagg": None}, "no launcher endpoint roles"), + ({DECODE: "decode"}, PD, f"unmapped=['{PREFILL}']"), + ({**ROLES, "http://10.0.0.3:6100/metrics": "prefill"}, PD, "missing=['http://10.0.0.3"), + ({PREFILL: "decode", DECODE: "decode"}, PD, "unmapped=[], missing=[]"), + ], +) +def test_unresolved_roles_publish_no_rates(roles, deployment, reason) -> None: + nested, warnings = _run(_pd(), roles=roles, deployment=deployment) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "ROLE_OR_COVERAGE_UNRESOLVED" + assert nested["cache_validity"]["pricing_eligible"] is False + assert any(reason in w for w in warnings) + + +def test_unified_fallback_needs_a_declared_non_disagg_single_endpoint() -> None: + single = _export({stem: {PREFILL: v} for stem, v in _pd_values().items()}) + two = _export({stem: {PREFILL: v, DECODE: v} for stem, v in _pd_values().items()}) + unified = {"disagg": False, "kv_offloading": "dram"} + + one, _ = _run(single, roles=None, deployment=unified, framework="atom") + many, _ = _run(two, roles=None, deployment=unified, framework="atom") + + assert one["cache_validity"]["roles_source"] == "single_endpoint_non_disagg" + assert one["cache_validity"]["pricing_eligible"] is True + assert many["cache_validity"]["status"] == "ROLE_OR_COVERAGE_UNRESOLVED" + + +def _pd_values() -> dict[str, tuple]: + return { + "prefix_cache_full_tokens": (0, 400, 600, 0), + "prefix_cache_cached_tokens": (0, 200, 400, 0), + "prefix_cache_offload_tokens": (0, 50, 50, 0), + "prefix_cache_requests": (0, 3, 2, 0), + } + + +@pytest.mark.parametrize( + ("override", "status", "reason"), + [ + ({"prefix_cache_cached_tokens": (0, -1, 400, 0)}, "WINDOW_UNRESOLVED", "invalid"), + ({"prefix_cache_full_tokens": (0, math.nan, 600, 0)}, "WINDOW_UNRESOLVED", "invalid"), + ( + {"prefix_cache_full_tokens": ZERO, "prefix_cache_cached_tokens": ZERO}, + "INVALID_COUNTER", + "no prompt tokens", + ), + ({"prefix_cache_cached_tokens": (0, 400, 550, 0)}, "INVALID_COUNTER", "exceeds full"), + ], +) +def test_invalid_counters_are_refused_not_clamped(override, status, reason) -> None: + nested, warnings = _run(_pd(**override)) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == status + assert any(reason in w for w in warnings) + + +def test_missing_timeslices_never_fall_back_to_stats_total() -> None: + export = _pd() + for entry in export["metrics"].values(): + for series in entry["series"]: + del series["timeslices"] + + nested, _ = _run(export) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "WINDOW_UNRESOLVED" + cumulative = nested["cache_validity"]["diagnostics"]["cumulative_stats_total"] + assert cumulative["atom:prefix_cache_full_tokens"] == 1000 + + +def test_missing_profiling_window_is_unresolved() -> None: + export = _pd() + del export["summary"] + + nested, _ = _run(export) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "WINDOW_UNRESOLVED" + + +# Window (1_200, 4_000): (1_000, 1_400) straddles the start, (3_600, 4_200) the end. +STRADDLE = ((0, 1_000), (1_000, 1_400), (1_400, 2_600), (2_600, 3_600), (3_600, 4_200)) + + +def _unified(full: tuple, cached: tuple, warmup: dict | None = None) -> dict: + export = _export( + { + "prefix_cache_full_tokens": {PREFILL: full}, + "prefix_cache_cached_tokens": {PREFILL: cached}, + "prefix_cache_offload_tokens": {PREFILL: (0, 0, 0, 0, 0)}, + "prefix_cache_requests": {PREFILL: (0, 0, 3, 2, 0)}, + }, + edges=STRADDLE, + window=(1_200, 4_000), + ) + if warmup is not None: + export["warmup_metrics"] = { + f"atom:{stem}": {"series": [{"endpoint_url": PREFILL, "stats": {"total": total}}]} + for stem, total in warmup.items() + } + return export + + +@pytest.mark.parametrize( + ("full", "cached", "warmup", "status"), + [ + ((0, 0, 400, 600, 0), (0, 0, 200, 400, 0), None, "VALID_COMPLETE"), + ((0, 0, 400, 600, 9), (0, 0, 200, 400, 0), None, "WINDOW_UNRESOLVED"), + ((300, 90, 400, 600, 0), (100, 30, 200, 400, 0), None, "WINDOW_UNRESOLVED"), + ( + (300, 90, 400, 600, 0), + (100, 30, 200, 400, 0), + {"prefix_cache_full_tokens": 390, "prefix_cache_cached_tokens": 130}, + "VALID_COMPLETE", + ), + ( + (300, 90, 400, 600, 0), + (100, 30, 200, 400, 0), + {"prefix_cache_full_tokens": 300, "prefix_cache_cached_tokens": 100}, + "WINDOW_UNRESOLVED", + ), + ], +) +def test_boundary_slice_activity_must_be_proven_warmup_only(full, cached, warmup, status) -> None: + unified = {"disagg": False, "kv_offloading": "dram"} + nested, _ = _run(_unified(full, cached, warmup), roles=None, deployment=unified) + + assert nested["cache_validity"]["status"] == status + assert nested["cache_validity"]["pricing_eligible"] is (status == "VALID_COMPLETE") + assert nested["cache"]["gpu_cache_hit_rate"] == 0.6 + + +def test_sampling_gap_inside_the_window_is_unresolved() -> None: + gappy = ((0, 1_000), (1_000, 2_000), (2_200, 4_000), (4_000, 5_000)) + export = _export( + {stem: {PREFILL: values} for stem, values in _pd_values().items()}, edges=gappy + ) + + nested, warnings = _run(export, roles=None, deployment={"disagg": False}) + + assert nested["cache_validity"]["status"] == "WINDOW_UNRESOLVED" + assert any("sampling gap" in w for w in warnings) + + +def test_duplicate_alias_series_for_one_endpoint_is_invalid() -> None: + export = _pd() + export["metrics"]["atom_prefix_cache_cached_tokens_total"] = export["metrics"][ + "atom:prefix_cache_cached_tokens" + ] + + nested, _ = _run(export) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "INVALID_COUNTER" + + +@pytest.mark.parametrize( + "scope", + [ + {**SCOPE, "profiling_requests": 4, "profiling_prompt_tokens": 900}, + {**SCOPE, "error_records": 1}, + {**SCOPE, "profiling_prompt_tokens": None}, + None, + ], +) +def test_request_scope_mismatch_keeps_window_telemetry_but_blocks_pricing(scope) -> None: + nested, _ = _run(_pd(), scope=scope) + + assert _rates(nested) == (0.6, 0.1, 0.1, pytest.approx(0.7)) + assert nested["cache_validity"]["status"] == "REQUEST_SCOPE_UNRESOLVED" + assert nested["cache_validity"]["pricing_eligible"] is False + + +def test_unrelated_router_endpoint_is_not_a_missing_prefill() -> None: + export = _pd() + router = "http://10.0.0.9:8000/metrics" + export["metrics"]["atomesh:http_requests_total"] = { + "type": "counter", + "series": [_series(router, (0, 5, 5, 0))], + } + + nested, _ = _run(export, roles={**ROLES, router: "router"}) + + assert nested["cache_validity"]["status"] == "VALID_COMPLETE" + + +@pytest.mark.parametrize( + ("env", "expected"), + [ + ( + {"SRT_PREFILL_ENDPOINTS": "10.0.0.1:6100", "SRT_DECODE_ENDPOINTS": "10.0.0.2:6100,"}, + ROLES, + ), + ( + {"SRT_AGG_ENDPOINTS": "a:1,b:1", "SRT_PREFILL_ENDPOINTS": "c:1"}, + {"http://a:1/metrics": "unified", "http://b:1/metrics": "unified"}, + ), + ], +) +def test_endpoint_roles_cover_launcher_scrape_urls( + env: dict[str, str], expected: dict[str, str] +) -> None: + roles = endpoint_roles_from_env(env) + + assert roles == expected + assert set(roles) == set(_server_metrics_urls(env)) + + +@pytest.mark.parametrize( + "env", + [ + {"AIPERF_SERVER_METRICS_URLS": "http://x/metrics", "SRT_PREFILL_ENDPOINTS": "a:1"}, + {"SRTCTL_FRONTEND_TYPE": "dynamo", "SRT_PREFILL_ENDPOINTS": "a:1"}, + {"SRT_PREFILL_ENDPOINTS": "a:1", "SRT_DECODE_ENDPOINTS": "a:1"}, + {}, + ], +) +def test_endpoint_roles_unknown_without_launcher_metadata(env: dict[str, str]) -> None: + assert endpoint_roles_from_env(env) is None + + +DISAGG_ENV = { + "FRAMEWORK": "atom-disagg", + "IS_MULTINODE": "true", + "DISAGG": "true", + **{ + f"{side}_{key}": value + for side in ("PREFILL", "DECODE") + for key, value in { + "NUM_WORKERS": "1", + "TP": "8", + "PP_SIZE": "1", + "DCP_SIZE": "1", + "PCP_SIZE": "1", + "EP": "1", + "DP_ATTN": "false", + "HARDWARE": "mi355x", + }.items() + }, +} +ROLE_ENV = {"SRT_PREFILL_ENDPOINTS": "10.0.0.1:6100", "SRT_DECODE_ENDPOINTS": "10.0.0.2:6100"} + + +def _atom_result_dir(tmp_path: Path) -> tuple[Path, list[dict]]: + result_dir = _write_fixture(tmp_path) + artifacts = result_dir / "aiperf_artifacts" + (artifacts / "server_metrics_export.json").write_text(json.dumps(_pd())) + jsonl = artifacts / "profile_export.jsonl" + records = [json.loads(line) for line in jsonl.read_text().splitlines() if line.strip()] + for record in records: + record["metrics"]["usage_prompt_tokens"] = {"value": 1000 // len(records), "unit": "tokens"} + return result_dir, records + + +def _write_records(result_dir: Path, records: list[dict]) -> None: + jsonl = result_dir / "aiperf_artifacts" / "profile_export.jsonl" + jsonl.write_text("".join(json.dumps(record) + "\n" for record in records)) + + +def _failed(phase: str | None) -> dict: + metadata = {} if phase is None else {"benchmark_phase": phase} + return {"metadata": metadata, "error": {"type": "Timeout", "message": "request timed out"}} + + +def test_processor_entry_point_carries_validity_through_the_serializer(tmp_path: Path) -> None: + result_dir, records = _atom_result_dir(tmp_path) + _write_records(result_dir, records) + + agg = _run_processor(result_dir, tmp_path / "out", env_overrides={**DISAGG_ENV, **ROLE_ENV}) + unresolved = _run_processor(result_dir, tmp_path / "unresolved", env_overrides=DISAGG_ENV) + + _assert_stable_server_metrics_schema(agg) + assert len(records) == 5 + assert _rates(agg["server_metrics"]) == (0.6, 0.1, 0.1, 0.7) + assert agg["server_metrics"]["cache_validity"]["status"] == "VALID_COMPLETE" + assert agg["server_metrics"]["cache_validity"]["pricing_eligible"] is True + assert _rates(unresolved["server_metrics"]) == NO_RATES + assert unresolved["server_metrics"]["cache_validity"]["pricing_eligible"] is False + assert ( + agg["server_metrics"]["tokens"]["prompt_total"] + == (unresolved["server_metrics"]["tokens"]["prompt_total"]) + ) + + +@pytest.mark.parametrize( + ("failed_phases", "legacy", "status", "profiling_errors"), + [ + ((), False, "VALID_COMPLETE", 0), + (("warmup",), False, "VALID_COMPLETE", 0), + (("profiling",), False, "REQUEST_SCOPE_UNRESOLVED", 1), + (("warmup", "profiling"), False, "REQUEST_SCOPE_UNRESOLVED", 1), + ((), True, "VALID_COMPLETE", 0), + ((None,), True, "REQUEST_SCOPE_UNRESOLVED", 1), + ], + ids=["clean", "warmup-error", "profiling-error", "both-phases", "legacy-clean", "legacy-error"], +) +def test_only_profiling_errors_from_the_loader_block_pricing( + tmp_path: Path, + failed_phases: tuple[str | None, ...], + legacy: bool, + status: str, + profiling_errors: int, +) -> None: + result_dir, records = _atom_result_dir(tmp_path) + if legacy: + for record in records: + del record["metadata"]["benchmark_phase"] + _write_records(result_dir, [_failed(phase) for phase in failed_phases] + records) + + agg = _run_processor(result_dir, tmp_path / "out", env_overrides={**DISAGG_ENV, **ROLE_ENV}) + validity = agg["server_metrics"]["cache_validity"] + accounting = agg["request_accounting"] + + assert _rates(agg["server_metrics"]) == (0.6, 0.1, 0.1, 0.7) + assert validity["status"] == status + assert validity["pricing_eligible"] is (status == "VALID_COMPLETE") + assert validity["request_scope"]["error_records"] == profiling_errors + assert accounting["records_profiled"] == 5 + assert accounting["records_error_dropped"] == len(failed_phases) + assert accounting["records_profiling_error_dropped"] == profiling_errors + assert accounting["records_warmup_dropped"] == failed_phases.count("warmup") + expected_categories = {"Timeout": len(failed_phases)} if failed_phases else {} + assert accounting["error_categories"] == expected_categories diff --git a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py index b76964b2ff..e420a19f5e 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py @@ -67,6 +67,7 @@ "records_dropped_total", "records_warmup_dropped", "records_error_dropped", + "records_profiling_error_dropped", "error_categories", } @@ -79,6 +80,7 @@ "kv_offload", "tokens", "sources", + "cache_validity", } SERVER_CACHE_KEYS = { "gpu_cache_hit_rate", @@ -336,8 +338,7 @@ def _run_processor( timeout=30, ) assert proc.returncode == 0, ( - f"processor exited {proc.returncode}\n" - f"stdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" + f"processor exited {proc.returncode}\nstdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" ) out = output_dir / "agg_test.json" assert out.exists(), f"missing output {out}; stdout:\n{proc.stdout}" @@ -393,27 +394,54 @@ def test_processor_omits_component_metadata_when_absent(tmp_path: Path): assert "kv_p2p_transfer" not in agg -@pytest.mark.parametrize("parser", [ - _optional_component_metadata, _optional_kv_offload_backend_metadata, -]) +@pytest.mark.parametrize( + "parser", + [ + _optional_component_metadata, + _optional_kv_offload_backend_metadata, + ], +) @pytest.mark.parametrize("raw", [None, "", "null"]) def test_optional_metadata_accepts_unset_values(parser, raw): env = {} if raw is None else {"TEST_METADATA": raw} assert parser(env, "TEST_METADATA") is None -@pytest.mark.parametrize(("parser", "raw", "message"), [ - (_optional_component_metadata, "{", "must contain valid JSON"), - (_optional_component_metadata, "[]", "must contain exactly 'name' and 'version'"), - (_optional_component_metadata, '{"name":"router"}', "must contain exactly 'name' and 'version'"), - (_optional_component_metadata, '{"name":"router","version":0}', "name and version must be non-empty strings"), - (_optional_kv_offload_backend_metadata, "{", "must contain valid JSON"), - (_optional_kv_offload_backend_metadata, "[]", "may contain only 'name' and 'version'"), - (_optional_kv_offload_backend_metadata, '{"name":"cache","extra":1}', "may contain only 'name' and 'version'"), - (_optional_kv_offload_backend_metadata, "{}", "must contain 'name' and optional 'version'"), - (_optional_kv_offload_backend_metadata, '{"version":"1"}', "must contain 'name' and optional 'version'"), - (_optional_kv_offload_backend_metadata, '{"name":"cache","version":""}', "values must be non-empty strings"), -]) +@pytest.mark.parametrize( + ("parser", "raw", "message"), + [ + (_optional_component_metadata, "{", "must contain valid JSON"), + (_optional_component_metadata, "[]", "must contain exactly 'name' and 'version'"), + ( + _optional_component_metadata, + '{"name":"router"}', + "must contain exactly 'name' and 'version'", + ), + ( + _optional_component_metadata, + '{"name":"router","version":0}', + "name and version must be non-empty strings", + ), + (_optional_kv_offload_backend_metadata, "{", "must contain valid JSON"), + (_optional_kv_offload_backend_metadata, "[]", "may contain only 'name' and 'version'"), + ( + _optional_kv_offload_backend_metadata, + '{"name":"cache","extra":1}', + "may contain only 'name' and 'version'", + ), + (_optional_kv_offload_backend_metadata, "{}", "must contain 'name' and optional 'version'"), + ( + _optional_kv_offload_backend_metadata, + '{"version":"1"}', + "must contain 'name' and optional 'version'", + ), + ( + _optional_kv_offload_backend_metadata, + '{"name":"cache","version":""}', + "values must be non-empty strings", + ), + ], +) def test_optional_metadata_preserves_cli_errors(parser, raw, message): with pytest.raises(SystemExit) as error: parser({"TEST_METADATA": raw}, "TEST_METADATA") @@ -795,49 +823,82 @@ def test_gpu_shape_defaults_empty_parallelism_to_one(env): assert _gpu_shape(env) == ({"pp": 1, "dcp_size": 1, "pcp_size": 1}, 1, 1, 1, "false") -@pytest.mark.parametrize("decode_workers,expected_gpus,expected_decode", [ - ("3", 78, (2, 11, 1, 9, 5)), - ("0", 48, (0, 0, 1, 1, 1)), -]) +@pytest.mark.parametrize( + "decode_workers,expected_gpus,expected_decode", + [ + ("3", 78, (2, 11, 1, 9, 5)), + ("0", 48, (0, 0, 1, 1, 1)), + ], +) def test_gpu_shape_counts_workers_and_normalizes_absent_decode( - decode_workers, expected_gpus, expected_decode, + decode_workers, + expected_gpus, + expected_decode, ): env = { - "IS_MULTINODE": "true", "PREFILL_NUM_WORKERS": "2", "PREFILL_TP": "3", - "PREFILL_PP_SIZE": "2", "PREFILL_PCP_SIZE": "4", "PREFILL_DCP_SIZE": "7", - "PREFILL_EP": "8", "PREFILL_DP_ATTN": "YES", "DECODE_NUM_WORKERS": decode_workers, - "DECODE_TP": "2", "DECODE_EP": "11", "DECODE_PP_SIZE": "1", - "DECODE_DCP_SIZE": "9", "DECODE_PCP_SIZE": "5", "DECODE_DP_ATTN": "false", - "PREFILL_GPUS": "999", "DECODE_GPUS": "999", + "IS_MULTINODE": "true", + "PREFILL_NUM_WORKERS": "2", + "PREFILL_TP": "3", + "PREFILL_PP_SIZE": "2", + "PREFILL_PCP_SIZE": "4", + "PREFILL_DCP_SIZE": "7", + "PREFILL_EP": "8", + "PREFILL_DP_ATTN": "YES", + "DECODE_NUM_WORKERS": decode_workers, + "DECODE_TP": "2", + "DECODE_EP": "11", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "9", + "DECODE_PCP_SIZE": "5", + "DECODE_DP_ATTN": "false", + "PREFILL_GPUS": "999", + "DECODE_GPUS": "999", } fields, num_gpus, tp, ep, attention = _gpu_shape(env) assert num_gpus == expected_gpus assert fields["num_prefill_gpu"] == 48 assert fields["num_decode_gpu"] == (30 if decode_workers == "3" else 0) - assert tuple(fields[key] for key in ("decode_tp", "decode_ep", "decode_pp", - "decode_dcp_size", "decode_pcp_size")) == expected_decode + assert ( + tuple( + fields[key] + for key in ("decode_tp", "decode_ep", "decode_pp", "decode_dcp_size", "decode_pcp_size") + ) + == expected_decode + ) assert (tp, ep, attention) == ((5, 11, "true") if decode_workers == "3" else (3, 8, "true")) assert fields["prefill_dp_attention"] == "YES" assert fields["decode_dp_attention"] == "false" -@pytest.mark.parametrize("env,error_type,message", [ - ({"IS_MULTINODE": "true", "TP": "bad"}, ValueError, "invalid literal for int"), - ({"IS_MULTINODE": "true", "PREFILL_PP_SIZE": "0", "PREFILL_HARDWARE": "gpu"}, - SystemExit, "Multinode PP, DCP, and PCP sizes must be positive integers."), - ({"IS_MULTINODE": "true", "DECODE_NUM_WORKERS": "0", "DECODE_PCP_SIZE": "0"}, - SystemExit, "Multinode PP, DCP, and PCP sizes must be positive integers."), -]) +@pytest.mark.parametrize( + "env,error_type,message", + [ + ({"IS_MULTINODE": "true", "TP": "bad"}, ValueError, "invalid literal for int"), + ( + {"IS_MULTINODE": "true", "PREFILL_PP_SIZE": "0", "PREFILL_HARDWARE": "gpu"}, + SystemExit, + "Multinode PP, DCP, and PCP sizes must be positive integers.", + ), + ( + {"IS_MULTINODE": "true", "DECODE_NUM_WORKERS": "0", "DECODE_PCP_SIZE": "0"}, + SystemExit, + "Multinode PP, DCP, and PCP sizes must be positive integers.", + ), + ], +) def test_gpu_shape_preserves_parsing_and_validation_order(env, error_type, message): with pytest.raises(error_type, match=message): _gpu_shape(env) -@pytest.mark.parametrize("error,category", [ - ({"type": "HTTPStatusError", "message": "500 server error"}, "HTTPStatusError"), - (" \t\r\n", "unknown"), - ({"message": " \t\r\n"}, "unknown"), -]) +@pytest.mark.parametrize( + "error,category", + [ + ({"type": "HTTPStatusError", "message": "500 server error"}, "HTTPStatusError"), + (" \t\r\n", "unknown"), + ({"message": " \t\r\n"}, "unknown"), + ], +) def test_processor_surfaces_request_accounting(tmp_path: Path, error, category): result_dir = tmp_path / "results" artifact = result_dir / "aiperf_artifacts" @@ -895,6 +956,7 @@ def test_processor_surfaces_request_accounting(tmp_path: Path, error, category): "records_dropped_total": 2, "records_warmup_dropped": 1, "records_error_dropped": 1, + "records_profiling_error_dropped": 1, "error_categories": {category: 1}, } assert agg["server_metrics"]["tokens"]["requests_completed"] == 1 @@ -1336,9 +1398,7 @@ def test_processor_normalizes_sglang_server_metrics(tmp_path: Path, framework: s }, "sglang:realtime_tokens": { "type": "counter", - "series": [ - {"labels": {"mode": "prefill_compute"}, "stats": {"total": 500.0}} - ], + "series": [{"labels": {"mode": "prefill_compute"}, "stats": {"total": 500.0}}], }, } } @@ -1383,9 +1443,7 @@ def test_processor_normalizes_sglang_server_metrics(tmp_path: Path, framework: s if framework == "dynamo-sglang": assert agg["server_metrics"]["kv_cache"]["gpu_total_tokens"] is None assert agg["kv_cache_pool_tokens"] is None - assert any( - "rank replicas are not normalized" in warning for warning in agg["warnings"] - ) + assert any("rank replicas are not normalized" in warning for warning in agg["warnings"]) def test_processor_normalizes_trtllm_server_metrics(tmp_path: Path): @@ -1669,9 +1727,7 @@ def test_processor_uses_aiperf_theoretical_cache_metric(tmp_path: Path): assert proc.returncode == 0, proc.stderr agg = json.loads((tmp_path / "out" / "agg_test.json").read_text()) assert agg["request_metrics"]["cache"]["theoretical_cache_hit_rate"] == pytest.approx(0.25) - assert agg["request_metrics"]["tokens"]["output_expected"]["mean"] == pytest.approx( - 55.0 - ) + assert agg["request_metrics"]["tokens"]["output_expected"]["mean"] == pytest.approx(55.0) @pytest.mark.parametrize("cache_state", ["matching", "missing", "ambiguous", "no_identity"]) @@ -1740,29 +1796,61 @@ def test_processor_supports_per_run_subdir_layout(tmp_path: Path): def test_builder_uses_explicit_inputs_without_environment_or_file_access(monkeypatch): records = [ - _make_record(conv_id="root", turn_index=0, isl=100, osl=50, - ttft_ms=30, e2e_ms=1000, itl_ms=10, - start_ns=1_000_000_000, end_ns=2_000_000_000), - _make_record(conv_id="root::sa:child", turn_index=1, isl=200, osl=100, - ttft_ms=50, e2e_ms=2000, itl_ms=20, - start_ns=2_000_000_000, end_ns=4_000_000_000), + _make_record( + conv_id="root", + turn_index=0, + isl=100, + osl=50, + ttft_ms=30, + e2e_ms=1000, + itl_ms=10, + start_ns=1_000_000_000, + end_ns=2_000_000_000, + ), + _make_record( + conv_id="root::sa:child", + turn_index=1, + isl=200, + osl=100, + ttft_ms=50, + e2e_ms=2000, + itl_ms=20, + start_ns=2_000_000_000, + end_ns=4_000_000_000, + ), ] aggregate = {"metadata": {"dataset": {"hf_dataset_name": "example/traces"}}} - server_metrics = {"metrics": {"vllm:prefix_cache_hits": { - "series": [{"stats": {"total": 10}}], - }, "vllm:prefix_cache_queries": {"series": [{"stats": {"total": 40}}]}}} - traces = [{"id": "root", "requests": [ - {"type": "n", "out": 11}, {"type": "tool", "out": 999}, - {"type": "s", "output_length": 29}, - ]}] + server_metrics = { + "metrics": { + "vllm:prefix_cache_hits": { + "series": [{"stats": {"total": 10}}], + }, + "vllm:prefix_cache_queries": {"series": [{"stats": {"total": 40}}]}, + } + } + traces = [ + { + "id": "root", + "requests": [ + {"type": "n", "out": 11}, + {"type": "tool", "out": 999}, + {"type": "s", "output_length": 29}, + ], + } + ] logs = [ "INFO (EngineCore_DP0 pid=10) GPU KV cache size: 800 tokens\n" "INFO (EngineCore_DP0 pid=10) GPU KV cache size: 800 tokens", "INFO (EngineCore_DP0 pid=11) GPU KV cache size: 1,200 tokens", ] - accounting = {"records_total": 4, "records_profiled": 2, "records_dropped_total": 2, - "records_warmup_dropped": 1, "records_error_dropped": 1, - "error_categories": {"Timeout": 1}} + accounting = { + "records_total": 4, + "records_profiled": 2, + "records_dropped_total": 2, + "records_warmup_dropped": 1, + "records_error_dropped": 1, + "error_categories": {"Timeout": 1}, + } env = {"KV_OFFLOADING": "none", "TP": "4", "FRAMEWORK": "vllm"} before = deepcopy((records, aggregate, server_metrics, traces, logs, accounting, env)) @@ -1777,13 +1865,20 @@ class NoEnvironment(dict): isolated.setattr("builtins.open", forbidden) isolated.setattr(Path, "open", forbidden) result = build_result( - records, aggregate, server_metrics, MappingProxyType(env), - request_accounting=accounting, traces=iter(traces), server_logs=iter(logs), + records, + aggregate, + server_metrics, + MappingProxyType(env), + request_accounting=accounting, + traces=iter(traces), + server_logs=iter(logs), ) second = build_result(records, aggregate, {}, {**env, "TP": "2"}) assert result["request_metrics"]["throughput"]["per_gpu"] == { - "total_tput_tps": 37.5, "output_tput_tps": 12.5, "input_tput_tps": 25.0, + "total_tput_tps": 37.5, + "output_tput_tps": 12.5, + "input_tput_tps": 25.0, } assert second["request_metrics"]["throughput"]["per_gpu"]["total_tput_tps"] == 75.0 assert result["request_metrics"]["tokens"]["output_expected"]["mean"] == 20 @@ -1796,18 +1891,23 @@ class NoEnvironment(dict): assert (records, aggregate, server_metrics, traces, logs, accounting, env) == before -@pytest.mark.parametrize("framework,metric_names", [ - ("atom", ["atom:prompt_tokens"]), - ("trtllm", ["trtllm_kv_cache_max_blocks"]), - ("dynamo-sglang", ["sglang:num_requests_total"]), -]) +@pytest.mark.parametrize( + "framework,metric_names", + [ + ("atom", ["atom:prompt_tokens"]), + ("trtllm", ["trtllm_kv_cache_max_blocks"]), + ("dynamo-sglang", ["sglang:num_requests_total"]), + ], +) def test_builder_does_not_consume_logs_when_backend_does_not_use_them(framework, metric_names): def unavailable_logs(): raise OSError("unreadable optional log") yield result = build_result( - [], {}, {"metrics": {name: {"series": []} for name in metric_names}}, + [], + {}, + {"metrics": {name: {"series": []} for name in metric_names}}, {"KV_OFFLOADING": "none", "FRAMEWORK": framework}, server_logs=unavailable_logs(), ) @@ -1819,30 +1919,47 @@ def unavailable_logs(): ] -@pytest.mark.parametrize("override,message", [ - ({"KV_OFFLOADING": ""}, "Missing required environment variable: KV_OFFLOADING"), - ({"PP_SIZE": "0"}, "PP_SIZE, DCP_SIZE, and PCP_SIZE must be positive integers."), - ({"ROUTER_METADATA": "{"}, "ROUTER_METADATA must contain valid JSON"), - ({}, "invalid literal for int() with base 10: 'bad-output'"), -]) +@pytest.mark.parametrize( + "override,message", + [ + ({"KV_OFFLOADING": ""}, "Missing required environment variable: KV_OFFLOADING"), + ({"PP_SIZE": "0"}, "PP_SIZE, DCP_SIZE, and PCP_SIZE must be positive integers."), + ({"ROUTER_METADATA": "{"}, "ROUTER_METADATA must contain valid JSON"), + ({}, "invalid literal for int() with base 10: 'bad-output'"), + ], +) def test_processor_preserves_validation_before_optional_trace_reads(tmp_path, override, message): result_dir = _write_fixture(tmp_path) cache = tmp_path / "hf" snapshot = cache / "datasets--semianalysisai--cc-traces-weka-062126" / "snapshots" / "one" snapshot.mkdir(parents=True) - (snapshot / "traces.json").write_text(json.dumps({ - "id": "trace-A", "requests": [{"type": "n", "out": "bad-output"}], - })) + (snapshot / "traces.json").write_text( + json.dumps( + { + "id": "trace-A", + "requests": [{"type": "n", "out": "bad-output"}], + } + ) + ) out = tmp_path / "out" env = { - "PATH": os.environ.get("PATH", ""), "PYTHONPATH": str(REPO_ROOT), - "RESULT_DIR": str(result_dir), "AGENTIC_OUTPUT_DIR": str(out), - "RESULT_FILENAME": "agg_test", "KV_OFFLOADING": "none", - "FRAMEWORK": "vllm", "HF_HUB_CACHE": str(cache), **override, + "PATH": os.environ.get("PATH", ""), + "PYTHONPATH": str(REPO_ROOT), + "RESULT_DIR": str(result_dir), + "AGENTIC_OUTPUT_DIR": str(out), + "RESULT_FILENAME": "agg_test", + "KV_OFFLOADING": "none", + "FRAMEWORK": "vllm", + "HF_HUB_CACHE": str(cache), + **override, } proc = subprocess.run( [sys.executable, "-m", "infx.results.agentic.process_agentic_result"], - cwd=REPO_ROOT, env=env, text=True, capture_output=True, timeout=30, + cwd=REPO_ROOT, + env=env, + text=True, + capture_output=True, + timeout=30, ) assert proc.returncode == 1 assert proc.stderr.rstrip().endswith(message) @@ -1858,9 +1975,14 @@ def test_processor_trace_cache_precedence_and_last_duplicate(tmp_path, cache_env snapshot.mkdir(parents=True) trace = {"id": "trace-A", "requests": [{"type": "n", "out": 17}] * 3} (snapshot / "a.jsonl").write_text("{bad json\n" + json.dumps(trace) + "\n") - (snapshot / "z.json").write_text(json.dumps({ - **trace, "requests": [{"type": "s", "output_length": 23}] * 3, - })) + (snapshot / "z.json").write_text( + json.dumps( + { + **trace, + "requests": [{"type": "s", "output_length": 23}] * 3, + } + ) + ) (snapshot / "zz.json").write_text(json.dumps({"id": "trace-A", "requests": []})) env = {"HF_HUB_CACHE": "", "HUGGINGFACE_HUB_CACHE": "", "HF_HOME": ""} env[cache_env] = str(cache) @@ -1880,13 +2002,20 @@ def test_processor_trace_cache_precedence_and_last_duplicate(tmp_path, cache_env assert expected["std"] == 0 -@pytest.mark.parametrize("offsets,expected_mean,expected_p95,windows", [ - ([0, 500_000_000, 1_000_000_000, 1_500_000_000, 2_000_000_000], 2, 2, 2), - ([0, 1_000_000_000, 1_000_000_000, 2_000_000_000], 1.5, 1.95, 2), - ([0, 250_000_000, 500_000_000], 6, None, 0), -]) -def test_qps_windows_exclude_right_boundary_and_preserve_duplicates(offsets, expected_mean, expected_p95, windows): - records = [{"metadata": {"request_end_ns": 1_000_000_000 + offset}} for offset in reversed(offsets)] +@pytest.mark.parametrize( + "offsets,expected_mean,expected_p95,windows", + [ + ([0, 500_000_000, 1_000_000_000, 1_500_000_000, 2_000_000_000], 2, 2, 2), + ([0, 1_000_000_000, 1_000_000_000, 2_000_000_000], 1.5, 1.95, 2), + ([0, 250_000_000, 500_000_000], 6, None, 0), + ], +) +def test_qps_windows_exclude_right_boundary_and_preserve_duplicates( + offsets, expected_mean, expected_p95, windows +): + records = [ + {"metadata": {"request_end_ns": 1_000_000_000 + offset}} for offset in reversed(offsets) + ] flat, nested = compute_qps_stats(records) assert nested["samples"] == windows assert flat["mean_qps"] == pytest.approx(expected_mean) @@ -1898,10 +2027,11 @@ def test_qps_windows_exclude_right_boundary_and_preserve_duplicates(offsets, exp @pytest.mark.parametrize("invalid_turn", [-1, -3, 2]) def test_expected_output_tokens_ignore_out_of_range_trace_turns(invalid_turn): - records = [{"metadata": {"conversation_id": "root", "turn_index": index}} - for index in (0, 1, invalid_turn)] - traces = [{"id": "root", "requests": [{"type": "n", "out": 11}, - {"type": "s", "out": 29}]}] + records = [ + {"metadata": {"conversation_id": "root", "turn_index": index}} + for index in (0, 1, invalid_turn) + ] + traces = [{"id": "root", "requests": [{"type": "n", "out": 11}, {"type": "s", "out": 29}]}] _, metrics = compute_request_metrics(records, traces=traces) assert metrics["tokens"]["output_expected"]["mean"] == 20 assert metrics["tokens"]["output_expected"]["std"] == 9 From 4b2c5ea714d3fbe08a8f7e7e1e7948a8c52cb8b3 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Wed, 14 Oct 2026 11:31:11 +0000 Subject: [PATCH 07/12] fix(agentx): mount the node-local hub cache at $HF_HOME/hub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit srtctl's model pre-download runs without its bash wrapper, so it sees only the HF_HOME it exports itself, not the role's HF_HUB_CACHE. On the c72 Store point it therefore pulled all 438 GB into the container's /tmp/hf_home/hub while the workers loaded the copy already in /var/lib/hf-hub-cache. Mount that cache at $HF_HOME/hub so the pre-download checks, and if needed fills, the same directory the workers read. srtctl 的模型预下载不经过 bash 包装，只能看到它自己 export 的 HF_HOME， 看不到角色环境里的 HF_HUB_CACHE。因此在 Store c72 测试点上，它把整整 438 GB 下载到了容器内的 /tmp/hf_home/hub，而 worker 实际加载的是 /var/lib/hf-hub-cache 中已有的副本。现在把该缓存挂载到 $HF_HOME/hub， 使预下载检查（必要时填充）的正是 worker 读取的同一目录。 Co-Authored-By: Claude Opus 5.5 --- .../glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml index 52470a7ac9..809c06be2b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -27,9 +27,9 @@ base: <<: &environment PYTHONUNBUFFERED: "1" PYTHONHASHSEED: "0" - # srtctl fills the first node's hub cache before workers start only when HF_HOME is set. - HF_HOME: /tmp/hf_home - HF_HUB_CACHE: /mnt/hf_hub_cache + # srtctl's pre-download sees only HF_HOME, so the node-local cache is mounted at $HF_HOME/hub. + HF_HOME: /mnt/hf_home + HF_HUB_CACHE: /mnt/hf_home/hub AITER_LOG_LEVEL: WARNING AITER_QUICK_REDUCE_QUANTIZATION: INT4 AITER_USE_FLYDSL_MOE_SORTING: "1" @@ -81,7 +81,7 @@ base: container-remap-root: "" extra_mount: # The node-local hub cache the single-node GLM-5.2 points already fill. - - /var/lib/hf-hub-cache:/mnt/hf_hub_cache + - /var/lib/hf-hub-cache:/mnt/hf_home/hub health_check: max_attempts: 1080 interval_seconds: 5 From e3f27cd4a205ab6a9bcbdf32fd3a44d407e24bac Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Sat, 17 Oct 2026 08:24:03 +0000 Subject: [PATCH 08/12] feat(agentx): run every GLM-5.2 ATOM PD point on the Mooncake Store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LMCache prefills sometimes hung for good in the startup barrier after registering their 1 TiB pinned pool (3 of 8 starts on mi355x-amds); Store prefills never have. On the test runs the Store matched or beat LMCache: 1P2D c56 24,090 vs 22,379 tok/s/GPU (TTFT p50 1.33 vs 1.62 s), 1P3D c24 10,173 vs 10,051. Replace the LMCache variants with Store variants that share the 1P1D Store masters, prefill env and connector; reserve the owners' hugepages only on the head node, where srtctl starts the prefill. #4211 budgets the colocated 1P1D decode's DRAM too, so the 1P1D owners grow from 349 to 448 GiB (1924 GB, 962 GB per NUMA node); c112 went from 36,779 to 40,499 tok/s/GPU (TTFT p50 19.3 -> 14.0 s). 1P2D and 1P3D owners stay at 349 GiB, the prefill node's 1499 GB. Add 1P1D c128. LMCache prefill 注册 1 TiB pinned 池后有时在启动 barrier 永久卡住 （mi355x-amds 上 8 次启动卡 3 次），Store prefill 从未出现。测试中 Store 不逊于 LMCache：1P2D c56 24,090 对 22,379 tok/s/GPU（TTFT p50 1.33 对 1.62 s），1P3D c24 10,173 对 10,051。用共享 1P1D Store master、prefill 环境和 connector 的 Store variant 取代 LMCache variant；只在 srtctl 启动 prefill 的 head 节点上预留 owner 的 hugepage。 #4211 也为同节点的 1P1D decode 分配内存预算，因此 1P1D owner 从 349 GiB 扩到 448 GiB（共 1924 GB，每个 NUMA 节点 962 GB）；c112 从 36,779 升到 40,499 tok/s/GPU（TTFT p50 19.3 -> 14.0 s）。1P2D 和 1P3D owner 保持 349 GiB，即 prefill 节点的 1499 GB。新增 1P1D c128。 Co-Authored-By: Claude Opus 5.5 --- .../mi355x-fp4/agentx/disagg-variants.yaml | 119 ++++++++++++------ inferencex-e2e/configs/amd-master.yaml | 23 ++-- inferencex-e2e/perf-changelog.yaml | 4 +- 3 files changed, 92 insertions(+), 54 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml index 809c06be2b..103e6ba08d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -99,16 +99,18 @@ override_1p1d_mcstore: commands: # A recipe host_setup replaces the cluster default, which also drops the page cache under DRAM KV offloading. - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" - # Owners take 714752 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. + # Owners take 917504 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. + # Only the prefill node holds owners; srtctl starts the prefill on the head node, which launches this srun. - >- - if [[ "$(hostname -s)" == "$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)" ]]; then + if hostname -I | tr ' ' '\n' | grep -qxF "${SLURM_LAUNCH_NODE_IPADDR:?srun did not set SLURM_LAUNCH_NODE_IPADDR}"; then + echo "[hugepage] $(hostname -s) is the head node; reserving the owners' pages"; held=$(awk '/^HugePages_Total:/ {total = $2} /^HugePages_Free:/ {free = $2} /^HugePages_Rsvd:/ {rsvd = $2} END {print total - free + rsvd}' /proc/meminfo); - HUGEPAGES_TARGET=$(( held + 714752 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 + HUGEPAGES_TARGET=$(( held + 917504 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" || exit 1; for node in 0 1; do free=$(cat /sys/devices/system/node/node$node/hugepages/hugepages-2048kB/free_hugepages); - echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 357376"; - [[ $free -ge 357376 ]] || exit 1; + echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 458752"; + [[ $free -ge 458752 ]] || exit 1; done; fi teardown: - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" @@ -133,19 +135,22 @@ override_1p1d_mcstore: readiness: port: 26051 timeout_seconds: 300 - - <<: *master + - &master_rdma1 + <<: *master name: mc-master-rdma1 command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26151, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26180, --metrics_port=26190, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] readiness: port: 26151 timeout_seconds: 300 - - <<: *master + - &master_rdma2 + <<: *master name: mc-master-rdma2 command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26251, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26280, --metrics_port=26290, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] readiness: port: 26251 timeout_seconds: 300 - - <<: *master + - &master_rdma3 + <<: *master name: mc-master-rdma3 command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26351, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26380, --metrics_port=26390, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] readiness: @@ -154,7 +159,7 @@ override_1p1d_mcstore: - &owner <<: *master name: mc-owner-rdma0 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] env: <<: *store_env # Back the segment with the reserved 2 MiB pages; the NIC cannot register 64 GiB regions of 4 KiB pages. @@ -170,16 +175,16 @@ override_1p1d_mcstore: interval_seconds: 5 - <<: *owner name: mc-owner-rdma1 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26151, --metadata_server=http://127.0.0.1:26180/metadata, --protocol=rdma, --device_names=rdma1, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26151, --metadata_server=http://127.0.0.1:26180/metadata, --protocol=rdma, --device_names=rdma1, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] - <<: *owner name: mc-owner-rdma2 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26054, --master_server_address=127.0.0.1:26251, --metadata_server=http://127.0.0.1:26280/metadata, --protocol=rdma, --device_names=rdma2, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26054, --master_server_address=127.0.0.1:26251, --metadata_server=http://127.0.0.1:26280/metadata, --protocol=rdma, --device_names=rdma2, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] - <<: *owner name: mc-owner-rdma3 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26055, --master_server_address=127.0.0.1:26351, --metadata_server=http://127.0.0.1:26380/metadata, --protocol=rdma, --device_names=rdma3, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26055, --master_server_address=127.0.0.1:26351, --metadata_server=http://127.0.0.1:26380/metadata, --protocol=rdma, --device_names=rdma3, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] roles: prefill: - env: + env: &store_prefill_env # More than one load worker hung the PP prefill pipeline at C96. OFFLOAD_LOAD_WORKERS: "1" MC_NUM_QP_PER_EP: "1" @@ -188,7 +193,7 @@ override_1p1d_mcstore: args: num-speculative-tokens: 3 # Each stage's Store client uses its GPU's NIC and that NIC's pool. - extra-kv-connectors: + extra-kv-connectors: &store_connector - kv_connector: mooncake_store kv_role: offload kv_connector_extra_config: @@ -213,7 +218,7 @@ override_1p1d_mcstore: cudagraph-capture-sizes: '[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' num-speculative-tokens: 3 -override_1p2d_lmcache_c56: +override_1p2d_mcstore_c56: resources: &spread # A node per decode; srtctl otherwise packs two TP4 decodes onto one node. spread_workers: true @@ -226,44 +231,78 @@ override_1p2d_lmcache_c56: balance-abs-threshold: 2 balance-rel-threshold: 2.0 eviction-interval: 300 + host_setup: &prefill_budget_host_setup + nodes: all + timeout_seconds: 1800 + commands: + # A recipe host_setup replaces the cluster default, which also drops the page cache under DRAM KV offloading. + - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" + # Owners take 714752 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. + # Only the prefill node holds owners; srtctl starts the prefill on the head node, which launches this srun. + - >- + if hostname -I | tr ' ' '\n' | grep -qxF "${SLURM_LAUNCH_NODE_IPADDR:?srun did not set SLURM_LAUNCH_NODE_IPADDR}"; then + echo "[hugepage] $(hostname -s) is the head node; reserving the owners' pages"; + held=$(awk '/^HugePages_Total:/ {total = $2} /^HugePages_Free:/ {free = $2} /^HugePages_Rsvd:/ {rsvd = $2} END {print total - free + rsvd}' /proc/meminfo); + HUGEPAGES_TARGET=$(( held + 714752 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 + bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" || exit 1; + for node in 0 1; do + free=$(cat /sys/devices/system/node/node$node/hugepages/hugepages-2048kB/free_hugepages); + echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 357376"; + [[ $free -ge 357376 ]] || exit 1; + done; fi + teardown: + - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" + services: &prefill_budget_services + - *master + - *master_rdma1 + - *master_rdma2 + - *master_rdma3 + - <<: *owner + name: mc-owner-rdma0 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + - <<: *owner + name: mc-owner-rdma1 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26151, --metadata_server=http://127.0.0.1:26180/metadata, --protocol=rdma, --device_names=rdma1, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + - <<: *owner + name: mc-owner-rdma2 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26054, --master_server_address=127.0.0.1:26251, --metadata_server=http://127.0.0.1:26280/metadata, --protocol=rdma, --device_names=rdma2, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + - <<: *owner + name: mc-owner-rdma3 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26055, --master_server_address=127.0.0.1:26351, --metadata_server=http://127.0.0.1:26380/metadata, --protocol=rdma, --device_names=rdma3, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] roles: prefill: - env: &lmcache_env - LMCACHE_LOCAL_CPU: "True" - LMCACHE_NUMA_MODE: auto - LMCACHE_MAX_LOCAL_CPU_SIZE: "256" - LMCACHE_CHUNK_SIZE: "256" - # Every PP stage runs a lookup server, so each stage refreshes its own LRU. - LMCACHE_LOOKUP_SERVER_WORKER_IDS: "0,1,2,3" + env: *store_prefill_env args: num-speculative-tokens: 4 - extra-kv-connectors: &lmcache_connector - - kv_connector: lmcache_offload - kv_role: offload + extra-kv-connectors: *store_connector decode: nodes: 2 workers: 2 env: AITER_REUSE_IDENTICAL_COMM_GROUPS: "1" + MC_NUM_QP_PER_EP: "1" args: <<: *dcp_decode num-speculative-tokens: 4 -override_1p3d_lmcache_c12: +override_1p3d_mcstore_c12: resources: *spread frontend: args: *cache_aware + host_setup: *prefill_budget_host_setup + services: *prefill_budget_services roles: prefill: - env: *lmcache_env + env: *store_prefill_env args: num-speculative-tokens: 5 - extra-kv-connectors: *lmcache_connector + extra-kv-connectors: *store_connector decode: nodes: 3 workers: 3 - env: + env: &tp4_decode_env PYTHONNOUSERSITE: "1" + MC_NUM_QP_PER_EP: "1" args: num-speculative-tokens: 5 gpu-memory-utilization: 0.95 @@ -272,21 +311,22 @@ override_1p3d_lmcache_c12: cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24]' disable-uvicorn-access-log: true -override_1p3d_lmcache_c16: +override_1p3d_mcstore_c16: resources: *spread frontend: args: *cache_aware + host_setup: *prefill_budget_host_setup + services: *prefill_budget_services roles: prefill: - env: *lmcache_env + env: *store_prefill_env args: num-speculative-tokens: 5 - extra-kv-connectors: *lmcache_connector + extra-kv-connectors: *store_connector decode: nodes: 3 workers: 3 - env: - PYTHONNOUSERSITE: "1" + env: *tp4_decode_env args: num-speculative-tokens: 5 gpu-memory-utilization: 0.95 @@ -295,21 +335,22 @@ override_1p3d_lmcache_c16: cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32]' disable-uvicorn-access-log: true -override_1p3d_lmcache_c24: +override_1p3d_mcstore_c24: resources: *spread frontend: args: *cache_aware + host_setup: *prefill_budget_host_setup + services: *prefill_budget_services roles: prefill: - env: *lmcache_env + env: *store_prefill_env args: num-speculative-tokens: 5 - extra-kv-connectors: *lmcache_connector + extra-kv-connectors: *store_connector decode: nodes: 3 workers: 3 - env: - PYTHONNOUSERSITE: "1" + env: *tp4_decode_env args: num-speculative-tokens: 5 gpu-memory-utilization: 0.95 diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 7ddfc5e96c..33ba5ae0f0 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1500,45 +1500,42 @@ glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: srt-recipe-dir: glm5.2/atom/mi355x-fp4/agentx scenarios: agentic-coding: - # The Store's 4 x 349 GiB owners: all 1499 GB of the 4-GPU prefill's share of the 3 TB cap. + # 1.0 gives each 4-GPU server 1499 GB: 1P1D's 4 x 448 GiB owners hold 962 GB per NUMA node of its prefill's and colocated decode's shares; 1P2D and 1P3D owners hold the prefill's 1499 GB. - dram-utilization: 1.0 search-space: - spec-decoding: "mtp" - conc-list: [ 56, 72, 96, 112 ] + conc-list: [ 56, 72, 96, 112, 128 ] kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13" } srt-recipe: disagg-variants.yaml:override_1p1d_mcstore prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } - # LMCache's 4 x 256 GiB stages: 1099 GB of that share. - - dram-utilization: 0.733 - search-space: - spec-decoding: "mtp" conc-list: [ 56 ] kv-offloading: dram - kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } - srt-recipe: disagg-variants.yaml:override_1p2d_lmcache_c56 + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p2d_mcstore_c56 prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } decode: { num-worker: 2, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } - spec-decoding: "mtp" conc-list: [ 12 ] kv-offloading: dram - kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } - srt-recipe: disagg-variants.yaml:override_1p3d_lmcache_c12 + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p3d_mcstore_c12 prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } - spec-decoding: "mtp" conc-list: [ 16 ] kv-offloading: dram - kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } - srt-recipe: disagg-variants.yaml:override_1p3d_lmcache_c16 + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p3d_mcstore_c16 prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } - spec-decoding: "mtp" conc-list: [ 24 ] kv-offloading: dram - kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } - srt-recipe: disagg-variants.yaml:override_1p3d_lmcache_c24 + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p3d_mcstore_c24 prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 98cfe55680..62bac6da66 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10079,8 +10079,8 @@ - agentic-coding description: - "Add GLM-5.2 MXFP4 on MI355X with ATOM prefill/decode disaggregation behind AToMesh, Mooncake RDMA from prefill to decode and MTP, on rocm/atom-dev:nightly_202610131249 (digest-pinned). The prefill is one CPP4 worker (PP4 x TP1 on four GPUs)." - - "1P1D on one node at concurrency 56, 72, 96 and 112: a DCP4 decode on GPUs 4-7 with three draft tokens, and the prefill's prefix tier in a Mooncake Store (0.3.13) of four per-NIC pools, each one master and a 349 GiB owner on 2 MiB HugeTLB pages. After the cluster host setup drops the page cache, host_setup reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its two owners' pages free; teardown releases them." - - "LMCache in the prefill (256 GiB per stage): 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens), and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). Each decode has a node of its own." + - "Every point keeps its prefill's prefix tier in a Mooncake Store (0.3.13) on the prefill node: four per-NIC pools, each one master and an owner on 2 MiB HugeTLB pages. 1P1D owners hold 448 GiB each (1924 GB, 962 GB per NUMA node, within the prefill's and the colocated decode's 1499 GB DRAM shares); 1P2D and 1P3D owners hold 349 GiB each, the prefill's 1499 GB. After the cluster host setup drops the page cache, host_setup on the head node, where srtctl starts the prefill, reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its two owners' pages free; teardown releases them." + - "1P1D on one node at concurrency 56, 72, 96, 112 and 128 with a DCP4 decode on GPUs 4-7 and three draft tokens; 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens); and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). In 1P2D and 1P3D each decode has a node of its own." - "Workers read the checkpoint from each node's local Hugging Face hub cache, as the single-node GLM-5.2 points do; srtctl fills the first node's cache before they start." - "Patch srt-slurm so its ATOM backend passes -tp as the worker's GPUs over its pipeline and prefill-context stages (NVIDIA/srt-slurm#604); unpatched, the PP4 prefill launches with -tp 4." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4045 From 1a153fd161a274cd676423505f766dd14ddfe978 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Sat, 17 Oct 2026 11:20:13 +0000 Subject: [PATCH 09/12] feat(agentx): give the GLM-5.2 1P1D points one shared Store pool, as in ATOM's CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ATOM's CI runs these 1P1D points at 46.1K tok/s/GPU (c112) with one master and two 768 GiB owners that all four PP stages share. Per-NIC pools here reached 36.8K at 4 x 349 GiB and 40.5K at 4 x 448 GiB: each stage only reads its own pool, and a prefix hit needs every stage to keep its chunk. Give 1P1D one master and two 768 GiB owners, one per NUMA node, each registered on rdma0-3, and set MC_ENABLE_DEST_DEVICE_AFFINITY on the prefill so each stage reaches an owner through the NIC of its own name on this rail-isolated fabric. 1P2D and 1P3D keep their tested per-NIC pools. ATOM CI 在这些 1P1D 点上（c112）跑到 46.1K tok/s/GPU，用的是一个 master 和两个 768 GiB owner，四个 PP stage 共享。这里按网卡分池时，4 x 349 GiB 为 36.8K，4 x 448 GiB 为 40.5K：每个 stage 只能读自己的池，而前缀命中 要求每个 stage 都保留各自的块。1P1D 改为一个 master 和两个 768 GiB owner （每个 NUMA 节点一个），各自注册在 rdma0-3 上，并在 prefill 上设置 MC_ENABLE_DEST_DEVICE_AFFINITY，使每个 stage 在这套按 rail 隔离的网络中 通过同名网卡访问 owner。1P2D 和 1P3D 保留已验证的按网卡分池。 Co-Authored-By: Claude Opus 5.5 --- .../mi355x-fp4/agentx/disagg-variants.yaml | 114 +++++++++--------- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 59 insertions(+), 59 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml index 103e6ba08d..21c171b2b7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -99,24 +99,24 @@ override_1p1d_mcstore: commands: # A recipe host_setup replaces the cluster default, which also drops the page cache under DRAM KV offloading. - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" - # Owners take 917504 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. + # Owners take 786432 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. # Only the prefill node holds owners; srtctl starts the prefill on the head node, which launches this srun. - >- if hostname -I | tr ' ' '\n' | grep -qxF "${SLURM_LAUNCH_NODE_IPADDR:?srun did not set SLURM_LAUNCH_NODE_IPADDR}"; then echo "[hugepage] $(hostname -s) is the head node; reserving the owners' pages"; held=$(awk '/^HugePages_Total:/ {total = $2} /^HugePages_Free:/ {free = $2} /^HugePages_Rsvd:/ {rsvd = $2} END {print total - free + rsvd}' /proc/meminfo); - HUGEPAGES_TARGET=$(( held + 917504 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 + HUGEPAGES_TARGET=$(( held + 786432 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" || exit 1; for node in 0 1; do free=$(cat /sys/devices/system/node/node$node/hugepages/hugepages-2048kB/free_hugepages); - echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 458752"; - [[ $free -ge 458752 ]] || exit 1; + echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 393216"; + [[ $free -ge 393216 ]] || exit 1; done; fi teardown: - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" services: - &master - name: mc-master-rdma0 + name: mc-master type: generic placement: node: prefill @@ -135,31 +135,11 @@ override_1p1d_mcstore: readiness: port: 26051 timeout_seconds: 300 - - &master_rdma1 - <<: *master - name: mc-master-rdma1 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26151, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26180, --metrics_port=26190, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] - readiness: - port: 26151 - timeout_seconds: 300 - - &master_rdma2 - <<: *master - name: mc-master-rdma2 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26251, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26280, --metrics_port=26290, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] - readiness: - port: 26251 - timeout_seconds: 300 - - &master_rdma3 - <<: *master - name: mc-master-rdma3 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26351, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26380, --metrics_port=26390, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] - readiness: - port: 26351 - timeout_seconds: 300 - &owner <<: *master - name: mc-owner-rdma0 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] + name: mc-owner-numa0 + # Each owner registers on rdma0-3, so every stage reaches it on its own rail. + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, "--device_names=rdma0,rdma1,rdma2,rdma3", --global_segment_size=824633720832, --local_buffer_size=0, --threads=4] env: <<: *store_env # Back the segment with the reserved 2 MiB pages; the NIC cannot register 64 GiB regions of 4 KiB pages. @@ -174,35 +154,29 @@ override_1p1d_mcstore: timeout_seconds: 1800 interval_seconds: 5 - <<: *owner - name: mc-owner-rdma1 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26151, --metadata_server=http://127.0.0.1:26180/metadata, --protocol=rdma, --device_names=rdma1, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] - - <<: *owner - name: mc-owner-rdma2 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26054, --master_server_address=127.0.0.1:26251, --metadata_server=http://127.0.0.1:26280/metadata, --protocol=rdma, --device_names=rdma2, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] - - <<: *owner - name: mc-owner-rdma3 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26055, --master_server_address=127.0.0.1:26351, --metadata_server=http://127.0.0.1:26380/metadata, --protocol=rdma, --device_names=rdma3, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] + name: mc-owner-numa1 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, "--device_names=rdma0,rdma1,rdma2,rdma3", --global_segment_size=824633720832, --local_buffer_size=0, --threads=4] roles: prefill: - env: &store_prefill_env - # More than one load worker hung the PP prefill pipeline at C96. - OFFLOAD_LOAD_WORKERS: "1" - MC_NUM_QP_PER_EP: "1" - # Registers the worker's HBM transfer pool in 1 GiB regions. - MC_MAX_MR_SIZE: "1073741824" + env: + <<: &store_prefill_env + # More than one load worker hung the PP prefill pipeline at C96. + OFFLOAD_LOAD_WORKERS: "1" + MC_NUM_QP_PER_EP: "1" + # Registers the worker's HBM transfer pool in 1 GiB regions. + MC_MAX_MR_SIZE: "1073741824" + # Every owner sits on rdma0-3 and this fabric is rail-isolated, so each stage must reach an owner through the NIC of its own name. + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" args: num-speculative-tokens: 3 - # Each stage's Store client uses its GPU's NIC and that NIC's pool. - extra-kv-connectors: &store_connector + # One pool shared by all four stages, as in ATOM's CI; each client still uses its GPU's NIC. + extra-kv-connectors: - kv_connector: mooncake_store kv_role: offload kv_connector_extra_config: mooncake_store.protocol: rdma - mooncake_store.pools: - rdma0: {master: "127.0.0.1:26051", metadata: "http://127.0.0.1:26080/metadata"} - rdma1: {master: "127.0.0.1:26151", metadata: "http://127.0.0.1:26180/metadata"} - rdma2: {master: "127.0.0.1:26251", metadata: "http://127.0.0.1:26280/metadata"} - rdma3: {master: "127.0.0.1:26351", metadata: "http://127.0.0.1:26380/metadata"} + mooncake_store.master: "127.0.0.1:26051" + mooncake_store.metadata: "http://127.0.0.1:26080/metadata" decode: nodes: colocate env: @@ -253,10 +227,26 @@ override_1p2d_mcstore_c56: teardown: - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" services: &prefill_budget_services - - *master - - *master_rdma1 - - *master_rdma2 - - *master_rdma3 + - <<: *master + name: mc-master-rdma0 + - <<: *master + name: mc-master-rdma1 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26151, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26180, --metrics_port=26190, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26151 + timeout_seconds: 300 + - <<: *master + name: mc-master-rdma2 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26251, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26280, --metrics_port=26290, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26251 + timeout_seconds: 300 + - <<: *master + name: mc-master-rdma3 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26351, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26380, --metrics_port=26390, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26351 + timeout_seconds: 300 - <<: *owner name: mc-owner-rdma0 command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] @@ -274,7 +264,17 @@ override_1p2d_mcstore_c56: env: *store_prefill_env args: num-speculative-tokens: 4 - extra-kv-connectors: *store_connector + # Each stage's Store client uses its GPU's NIC and that NIC's pool. + extra-kv-connectors: &per_nic_store_connector + - kv_connector: mooncake_store + kv_role: offload + kv_connector_extra_config: + mooncake_store.protocol: rdma + mooncake_store.pools: + rdma0: {master: "127.0.0.1:26051", metadata: "http://127.0.0.1:26080/metadata"} + rdma1: {master: "127.0.0.1:26151", metadata: "http://127.0.0.1:26180/metadata"} + rdma2: {master: "127.0.0.1:26251", metadata: "http://127.0.0.1:26280/metadata"} + rdma3: {master: "127.0.0.1:26351", metadata: "http://127.0.0.1:26380/metadata"} decode: nodes: 2 workers: 2 @@ -296,7 +296,7 @@ override_1p3d_mcstore_c12: env: *store_prefill_env args: num-speculative-tokens: 5 - extra-kv-connectors: *store_connector + extra-kv-connectors: *per_nic_store_connector decode: nodes: 3 workers: 3 @@ -322,7 +322,7 @@ override_1p3d_mcstore_c16: env: *store_prefill_env args: num-speculative-tokens: 5 - extra-kv-connectors: *store_connector + extra-kv-connectors: *per_nic_store_connector decode: nodes: 3 workers: 3 @@ -346,7 +346,7 @@ override_1p3d_mcstore_c24: env: *store_prefill_env args: num-speculative-tokens: 5 - extra-kv-connectors: *store_connector + extra-kv-connectors: *per_nic_store_connector decode: nodes: 3 workers: 3 diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 33ba5ae0f0..7ad0141164 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1500,7 +1500,7 @@ glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: srt-recipe-dir: glm5.2/atom/mi355x-fp4/agentx scenarios: agentic-coding: - # 1.0 gives each 4-GPU server 1499 GB: 1P1D's 4 x 448 GiB owners hold 962 GB per NUMA node of its prefill's and colocated decode's shares; 1P2D and 1P3D owners hold the prefill's 1499 GB. + # 1.0 gives each 4-GPU server 1499 GB: 1P1D's 2 x 768 GiB owners hold 825 GB per NUMA node of its prefill's and colocated decode's shares; 1P2D and 1P3D owners hold the prefill's 1499 GB. - dram-utilization: 1.0 search-space: - spec-decoding: "mtp" diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 62bac6da66..89809ae5ab 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10079,7 +10079,7 @@ - agentic-coding description: - "Add GLM-5.2 MXFP4 on MI355X with ATOM prefill/decode disaggregation behind AToMesh, Mooncake RDMA from prefill to decode and MTP, on rocm/atom-dev:nightly_202610131249 (digest-pinned). The prefill is one CPP4 worker (PP4 x TP1 on four GPUs)." - - "Every point keeps its prefill's prefix tier in a Mooncake Store (0.3.13) on the prefill node: four per-NIC pools, each one master and an owner on 2 MiB HugeTLB pages. 1P1D owners hold 448 GiB each (1924 GB, 962 GB per NUMA node, within the prefill's and the colocated decode's 1499 GB DRAM shares); 1P2D and 1P3D owners hold 349 GiB each, the prefill's 1499 GB. After the cluster host setup drops the page cache, host_setup on the head node, where srtctl starts the prefill, reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its two owners' pages free; teardown releases them." + - "Every point keeps its prefill's prefix tier in a Mooncake Store (0.3.13) on the prefill node, with owners on 2 MiB HugeTLB pages. 1P1D, as in ATOM's CI, has one pool shared by all four PP stages: one master and two 768 GiB owners, one per NUMA node (1649 GB, within the prefill's and the colocated decode's 1499 GB DRAM shares), each registered on rdma0-3, with MC_ENABLE_DEST_DEVICE_AFFINITY so each stage reaches them through the NIC of its own name on this rail-isolated fabric. 1P2D and 1P3D keep a pool per prefill NIC, each one master and a 349 GiB owner, the prefill's 1499 GB. After the cluster host setup drops the page cache, host_setup on the head node, where srtctl starts the prefill, reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its owners' pages free; teardown releases them." - "1P1D on one node at concurrency 56, 72, 96, 112 and 128 with a DCP4 decode on GPUs 4-7 and three draft tokens; 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens); and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). In 1P2D and 1P3D each decode has a node of its own." - "Workers read the checkpoint from each node's local Hugging Face hub cache, as the single-node GLM-5.2 points do; srtctl fills the first node's cache before they start." - "Patch srt-slurm so its ATOM backend passes -tp as the worker's GPUs over its pipeline and prefill-context stages (NVIDIA/srt-slurm#604); unpatched, the PP4 prefill launches with -tp 4." From 9740fde54d406f27ac2b9b38c369a710663c3c69 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Thu, 22 Oct 2026 07:44:46 +0000 Subject: [PATCH 10/12] fix: account ATOMesh cache reuse on prefill workers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 仅汇总 prefill/unified 缓存计数，CPU reuse 使用 offload counter， overall 合并 GPU 和 CPU reuse；缺少增量或角色时保留未知值。 --- .../infx/results/agentic/__init__.py | 1 + .../infx/results/agentic/backends/atom.py | 111 ++++++++-- .../infx/results/agentic/server_metrics.py | 37 +++- .../agentic/test_atom_cache_counters.py | 195 ++++++++++++++++++ .../agentic/test_process_agentic_result.py | 49 +++++ 5 files changed, 374 insertions(+), 19 deletions(-) create mode 100644 inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py diff --git a/inferencex-e2e/infx/results/agentic/__init__.py b/inferencex-e2e/infx/results/agentic/__init__.py index 5cf7469691..92a78f8584 100644 --- a/inferencex-e2e/infx/results/agentic/__init__.py +++ b/inferencex-e2e/infx/results/agentic/__init__.py @@ -218,6 +218,7 @@ def build_result( framework=framework, records=records, server_logs=server_logs, + env=env, ) if "total_tput_tps" in request_flat and num_gpus > 0: diff --git a/inferencex-e2e/infx/results/agentic/backends/atom.py b/inferencex-e2e/infx/results/agentic/backends/atom.py index 947024b111..7c6071adff 100644 --- a/inferencex-e2e/infx/results/agentic/backends/atom.py +++ b/inferencex-e2e/infx/results/agentic/backends/atom.py @@ -2,12 +2,24 @@ from __future__ import annotations +import logging +import math +from collections.abc import Mapping from typing import Any -from infx.results.agentic.common import gauge_stat, normalize_fraction, rate, sum_stat +from infx.results.agentic.common import ( + gauge_stat, + metric_series, + normalize_fraction, + rate, + series_stat, + sum_stat, +) from .base import ServerMetricsBackend, counter_int +logger = logging.getLogger(__name__) + def _atom_names(suffix: str) -> list[str]: """Accept direct worker metrics and Atomesh's colon-normalized form.""" @@ -19,6 +31,73 @@ def _atom_counter_names(stem: str) -> list[str]: return [*_atom_names(stem), *_atom_names(f"{stem}_total")] +def _counter_total(metrics: dict[str, dict[str, Any]], stem: str, endpoint: str) -> float | None: + names = _atom_counter_names(f"prefix_cache_{stem}_tokens") + series = [s for s in metric_series(metrics, names) if s.get("endpoint_url") == endpoint] + if not series or any( + (value := series_stat(s, preferred_keys=("total",))) is None + or not math.isfinite(value) + or value < 0 + for s in series + ): + logger.warning("ATOM %s cache counter missing or invalid at %s", stem, endpoint) + return None + total = sum_stat( + metrics, + names, + preferred_keys=("total",), + series_filter=lambda s: s.get("endpoint_url") == endpoint, + ) + return total if total is not None and math.isfinite(total) else None + + +def _cache_counts( + metrics: dict[str, dict[str, Any]], + endpoint_roles: Mapping[str, str] | None, + disaggregated: bool | None, +) -> tuple[float | None, float | None, float | None]: + names = [ + name + for stem in ("full", "cached", "offload") + for name in _atom_counter_names(f"prefix_cache_{stem}_tokens") + ] + endpoints = [s.get("endpoint_url") for s in metric_series(metrics, names)] + if any(not isinstance(endpoint, str) or not endpoint for endpoint in endpoints): + return None, None, None + observed = set(endpoints) + if endpoint_roles is None: + accounted = observed if disaggregated is False and len(observed) == 1 else set() + else: + # Decode counts the prompt again without serving its prefill cache reuse. + accounted = {url for url, role in endpoint_roles.items() if role in {"prefill", "unified"}} + if observed - endpoint_roles.keys(): + accounted = set() + if not accounted: + logger.warning("ATOM cache endpoint roles missing or ambiguous") + return None, None, None + full_total = cached_total = offload_total = 0.0 + cpu_complete = True + for endpoint in accounted: + full, cached, offload = ( + _counter_total(metrics, stem, endpoint) for stem in ("full", "cached", "offload") + ) + if full is None or cached is None or cached > full: + return None, None, None + full_total += full + cached_total += cached + if offload is None or cached + offload > full: + cpu_complete = False + else: + offload_total += offload + if full_total <= 0 or not all(math.isfinite(v) for v in (full_total, cached_total)): + return None, None, None + return ( + full_total, + cached_total, + offload_total if cpu_complete and math.isfinite(offload_total) else None, + ) + + class AtomBackend(ServerMetricsBackend): name = "atom" @@ -33,6 +112,9 @@ def populate( metrics: dict[str, dict[str, Any]], flat: dict[str, Any], nested: dict[str, Any], + *, + endpoint_roles: Mapping[str, str] | None = None, + disaggregated: bool | None = None, ) -> None: prompt_total = sum_stat( metrics, @@ -47,24 +129,17 @@ def populate( flat["total_prompt_tokens"] = counter_int(prompt_total) flat["total_generation_tokens"] = counter_int(generation_total) - cached_tokens = sum_stat( - metrics, - _atom_counter_names("prefix_cache_cached_tokens"), - preferred_keys=("total", "sum", "max", "avg"), - ) - full_tokens = sum_stat( - metrics, - _atom_counter_names("prefix_cache_full_tokens"), - preferred_keys=("total", "sum", "max", "avg"), + full_tokens, cached_tokens, external_tokens = _cache_counts( + metrics, endpoint_roles, disaggregated ) cache_hit_rate = rate(cached_tokens, full_tokens) - external_tokens = sum_stat( - metrics, - _atom_counter_names("lmcache_loaded_tokens"), - preferred_keys=("total", "sum", "max", "avg"), - ) + # Offload is CPU reuse beyond the GPU prefix; loaded tokens measure transfer volume. external_hit_rate = rate(external_tokens, full_tokens) - overall_hit_rate = cache_hit_rate + overall_hit_rate = ( + rate(cached_tokens + external_tokens, full_tokens) + if cached_tokens is not None and external_tokens is not None + else None + ) flat["server_gpu_cache_hit_rate"] = cache_hit_rate flat["server_cpu_cache_hit_rate"] = external_hit_rate @@ -88,7 +163,9 @@ def populate( "prefix_cache_hits": cached_tokens, "prefix_cache_queries": full_tokens, "external_prefix_cache_hits": external_tokens, - "external_prefix_cache_queries": full_tokens, + "external_prefix_cache_queries": ( + full_tokens if external_tokens is not None else None + ), } ) nested["kv_cache"]["gpu_usage_pct"] = flat["gpu_kv_cache_usage_pct"] diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index d14962a61f..e100011642 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -2,10 +2,11 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import Any from .backends import detect_backend +from .backends.atom import AtomBackend from .backends.base import ( SERVER_CACHE_FLAT_FIELDS, apply_profile_totals, @@ -14,12 +15,34 @@ from .common import index_server_metrics +def endpoint_roles_from_env(env: Mapping[str, str]) -> dict[str, str] | None: + if env.get("AIPERF_SERVER_METRICS_URLS") or env.get("SRTCTL_FRONTEND_TYPE") == "dynamo": + return {} + groups = ( + [("unified", env["SRT_AGG_ENDPOINTS"])] + if env.get("SRT_AGG_ENDPOINTS") + else [ + ("prefill", env.get("SRT_PREFILL_ENDPOINTS", "")), + ("decode", env.get("SRT_DECODE_ENDPOINTS", "")), + ] + ) + roles: dict[str, str] = {} + for role, endpoints in groups: + for endpoint in endpoints.split(","): + if endpoint: + url = f"http://{endpoint}/metrics" + if roles.setdefault(url, role) != role: + return {} + return roles or None + + def compute_server_metrics( server_metrics: dict[str, Any], *, framework: str, records: list[dict[str, Any]], server_logs: Iterable[str | None] = (), + env: Mapping[str, str] | None = None, ) -> tuple[dict[str, Any], dict[str, Any], list[str]]: metrics = index_server_metrics(server_metrics) flat = SERVER_CACHE_FLAT_FIELDS.copy() @@ -46,7 +69,17 @@ def compute_server_metrics( f"framework={framework.lower()!r}, metric_names=[{preview}]" ) nested["adapter"] = adapter.name - adapter.populate(metrics, flat, nested) + if isinstance(adapter, AtomBackend): + source_env = env or {} + adapter.populate( + metrics, + flat, + nested, + endpoint_roles=endpoint_roles_from_env(source_env), + disaggregated={"true": True, "false": False}.get(source_env.get("DISAGG", "").lower()), + ) + else: + adapter.populate(metrics, flat, nested) if framework.lower() == "dynamo-sglang" and adapter.name == "sglang": nested["kv_cache"]["gpu_total_tokens"] = None diff --git a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py new file mode 100644 index 0000000000..2d0e6f433d --- /dev/null +++ b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import math + +import pytest + +from infx.results.agentic.server_metrics import compute_server_metrics + + +def _export(full: dict, cached: dict, offload: dict) -> dict: + return { + "metrics": { + f"atom:prefix_cache_{stem}_tokens": { + "series": [ + {"endpoint_url": f"http://{endpoint}/metrics", "stats": {"total": value}} + for endpoint, value in values.items() + ] + } + for stem, values in (("full", full), ("cached", cached), ("offload", offload)) + } + } + + +def _compute(export: dict, env: dict) -> tuple[dict, dict]: + flat, nested, _ = compute_server_metrics(export, framework="mooncake-atom", records=[], env=env) + return flat, nested + + +def _assert_rates( + flat: dict, nested: dict, gpu: float | None, cpu: float | None, overall: float | None +): + for name, expected in (("gpu", gpu), ("cpu", cpu), ("external", cpu), ("overall", overall)): + actual = flat[f"server_{name}_cache_hit_rate"] + assert actual == (pytest.approx(expected) if expected is not None else None) + assert nested["cache"][f"{name}_cache_hit_rate"] == actual + + +@pytest.mark.parametrize("decode_cached", [800, math.nan]) +def test_prefill_counters_exclude_decode_and_transfer_volume(decode_cached: float) -> None: + export = _export({"p": 1000, "d": 1000}, {"p": 600, "d": decode_cached}, {"p": 100, "d": -1}) + export["metrics"]["atom:lmcache_loaded_tokens"] = { + "series": [{"endpoint_url": "http://p/metrics", "stats": {"total": 900}}] + } + flat, nested = _compute( + export, {"SRT_PREFILL_ENDPOINTS": "p", "SRT_DECODE_ENDPOINTS": "d", "DISAGG": "true"} + ) + _assert_rates(flat, nested, 0.6, 0.1, 0.7) + assert nested["cache"]["prefix_cache_queries"] == 1000 + assert nested["cache"]["external_prefix_cache_hits"] == 100 + assert nested["cache"]["external_prefix_cache_queries"] == 1000 + + +@pytest.mark.parametrize( + ("env", "offload", "expected"), + [ + ({"SRT_AGG_ENDPOINTS": "worker", "SRT_PREFILL_ENDPOINTS": "unused"}, 50, 0.5), + ({"DISAGG": "false"}, 0, 0), + ], +) +def test_unified_zero_gpu_and_offload_counter_alias( + env: dict, offload: int, expected: float +) -> None: + export = _export({"worker": 100}, {"worker": 0}, {"worker": offload}) + export["metrics"] = { + name.replace("atom:", "atom_") + "_total": value + for name, value in export["metrics"].items() + } + flat, nested = _compute(export, env) + _assert_rates(flat, nested, 0, expected, expected) + + +def test_multiple_prefills_and_labelled_series_are_token_weighted() -> None: + export = _export( + {"p": 100, "q": 900, "idle": 0}, + {"p": 80, "q": 20, "idle": 0}, + {"p": 10, "q": 180, "idle": 0}, + ) + for stem, values in (("full", (400, 500)), ("cached", (5, 15)), ("offload", (80, 100))): + series = export["metrics"][f"atom:prefix_cache_{stem}_tokens"]["series"] + series[:] = [s for s in series if s["endpoint_url"] != "http://q/metrics"] + series.extend( + { + "endpoint_url": "http://q/metrics", + "labels": {"engine": str(i)}, + "stats": {"total": v}, + } + for i, v in enumerate(values) + ) + flat, nested = _compute(export, {"SRT_PREFILL_ENDPOINTS": "p,q,idle"}) + _assert_rates(flat, nested, 0.1, 0.19, 0.29) + + +@pytest.mark.parametrize( + "env", + [ + {}, + {"DISAGG": "true"}, + {"SRT_PREFILL_ENDPOINTS": "p", "SRT_DECODE_ENDPOINTS": "p,d"}, + {"SRT_PREFILL_ENDPOINTS": "p,missing", "SRT_DECODE_ENDPOINTS": "d"}, + {"SRT_PREFILL_ENDPOINTS": "p"}, + { + "SRT_PREFILL_ENDPOINTS": "p", + "SRT_DECODE_ENDPOINTS": "d", + "AIPERF_SERVER_METRICS_URLS": "custom", + }, + { + "SRT_PREFILL_ENDPOINTS": "p", + "SRT_DECODE_ENDPOINTS": "d", + "SRTCTL_FRONTEND_TYPE": "dynamo", + }, + ], +) +def test_unknown_conflicting_or_incomplete_roles_do_not_mix_endpoints(env: dict) -> None: + flat, nested = _compute( + _export({"p": 100, "d": 100}, {"p": 60, "d": 0}, {"p": 10, "d": 0}), env + ) + _assert_rates(flat, nested, None, None, None) + + +def test_single_endpoint_without_explicit_unified_topology_stays_unknown() -> None: + flat, nested = _compute(_export({"worker": 100}, {"worker": 60}, {"worker": 10}), {}) + _assert_rates(flat, nested, None, None, None) + + +@pytest.mark.parametrize( + ("offload", "cpu", "overall"), + [({"p": 0, "q": 0}, 0, 0.2), ({}, None, None), ({"p": 10}, None, None)], +) +def test_missing_or_partial_offload_is_distinct_from_zero( + offload: dict, cpu: float | None, overall: float | None +) -> None: + flat, nested = _compute( + _export({"p": 100, "q": 100}, {"p": 20, "q": 20}, offload), + {"SRT_PREFILL_ENDPOINTS": "p,q"}, + ) + _assert_rates(flat, nested, 0.2, cpu, overall) + assert nested["cache"]["external_prefix_cache_queries"] == (200 if cpu is not None else None) + + +@pytest.mark.parametrize( + ("stem", "value"), + [ + ("full", 0), + ("cached", -1), + ("cached", 101), + ("cached", math.nan), + ("cached", math.inf), + ("cached", None), + ], +) +def test_invalid_counted_counter_does_not_produce_rates(stem: str, value: object) -> None: + export = _export({"worker": 100}, {"worker": 20}, {"worker": 10}) + export["metrics"][f"atom:prefix_cache_{stem}_tokens"]["series"][0]["stats"]["total"] = value + flat, nested = _compute(export, {"DISAGG": "false"}) + _assert_rates(flat, nested, None, None, None) + + +def test_unreadable_series_does_not_get_silently_skipped() -> None: + export = _export({"worker": 100}, {"worker": 20}, {"worker": 10}) + export["metrics"]["atom:prefix_cache_cached_tokens"]["series"].append( + { + "endpoint_url": "http://worker/metrics", + "labels": {"engine": "other"}, + "stats": {"max": 5}, + } + ) + flat, nested = _compute(export, {"DISAGG": "false"}) + _assert_rates(flat, nested, None, None, None) + + +@pytest.mark.parametrize("offload", [30, math.nan]) +def test_invalid_worker_cpu_count_cannot_hide_in_another_workers_denominator( + offload: float, +) -> None: + flat, nested = _compute( + _export({"p": 100, "q": 900}, {"p": 80, "q": 20}, {"p": offload, "q": 0}), + {"SRT_PREFILL_ENDPOINTS": "p,q"}, + ) + _assert_rates(flat, nested, 0.1, None, None) + + +def test_unknown_cache_roles_preserve_other_atom_telemetry() -> None: + export = _export({"p": 100, "d": 100}, {"p": 60, "d": 0}, {"p": 10, "d": 0}) + export["metrics"].update( + { + "atom:prompt_tokens": {"series": [{"stats": {"total": 200}}]}, + "atom:generation_tokens": {"series": [{"stats": {"total": 30}}]}, + "atom:kv_cache_usage_ratio": {"series": [{"stats": {"max": 0.75}}]}, + } + ) + flat, nested = _compute(export, {}) + _assert_rates(flat, nested, None, None, None) + assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 200 + assert flat["total_generation_tokens"] == nested["tokens"]["generation_total"] == 30 + assert flat["gpu_kv_cache_usage_pct"] == nested["kv_cache"]["gpu_usage_pct"] == 0.75 diff --git a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py index e162f49685..29aeeeef94 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py @@ -365,6 +365,55 @@ def test_processor_records_run_attempt(tmp_path: Path): assert "run_attempt" not in local +def test_processor_uses_launcher_roles_for_atom_cache_counts(tmp_path: Path) -> None: + result_dir = _write_fixture(tmp_path) + metrics = { + f"atom:prefix_cache_{stem}_tokens": { + "series": [ + {"endpoint_url": f"http://{endpoint}/metrics", "stats": {"total": value}} + for endpoint, value in values.items() + ] + } + for stem, values in ( + ("full", {"prefill:8000": 1000, "decode:8000": 1000}), + ("cached", {"prefill:8000": 600, "decode:8000": 0}), + ("offload", {"prefill:8000": 100, "decode:8000": 0}), + ) + } + (result_dir / "aiperf_artifacts" / "server_metrics_export.json").write_text( + json.dumps({"metrics": metrics}) + ) + agg = _run_processor( + result_dir, + tmp_path / "out", + { + "FRAMEWORK": "mooncake-atom", + "DISAGG": "true", + "IS_MULTINODE": "true", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "4", + "PREFILL_EP": "1", + "DECODE_NUM_WORKERS": "1", + "DECODE_TP": "4", + "DECODE_EP": "1", + "SRT_PREFILL_ENDPOINTS": "prefill:8000", + "SRT_DECODE_ENDPOINTS": "decode:8000", + "SRT_AGG_ENDPOINTS": "", + "AIPERF_SERVER_METRICS_URLS": "", + "SRTCTL_FRONTEND_TYPE": "", + "KV_OFFLOADING": "dram", + "KV_OFFLOAD_BACKEND": "lmcache", + "KV_OFFLOAD_BACKEND_METADATA": '{"name":"lmcache","version":"0.4.5"}', + }, + ) + cache = agg["server_metrics"]["cache"] + assert cache["gpu_cache_hit_rate"] == 0.6 + assert cache["cpu_cache_hit_rate"] == cache["external_cache_hit_rate"] == 0.1 + assert cache["overall_cache_hit_rate"] == 0.7 + assert cache["prefix_cache_queries"] == cache["external_prefix_cache_queries"] == 1000 + assert agg["num_gpus"] == 8 + + def test_processor_preserves_dataset_provenance(tmp_path: Path): result_dir = _write_fixture(tmp_path) output_dir = tmp_path / "out" From 984280caef3ac62f0a498e8f539da9904ea0a939 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Thu, 22 Oct 2026 20:28:52 +0000 Subject: [PATCH 11/12] fix(results): preserve ATOM unified counters and prefill totals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 仅在确认关闭 CPU offload 且无计数器时补零，保留单端点自定义 metrics 的角色回退，并从输入总量中排除 decode 重复计数。补充相应行为回归。 --- .../infx/results/agentic/backends/atom.py | 30 ++++++++++++++-- .../infx/results/agentic/server_metrics.py | 9 ++++- .../agentic/test_atom_cache_counters.py | 34 +++++++++++++++++-- 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/backends/atom.py b/inferencex-e2e/infx/results/agentic/backends/atom.py index 7c6071adff..17929bb0c8 100644 --- a/inferencex-e2e/infx/results/agentic/backends/atom.py +++ b/inferencex-e2e/infx/results/agentic/backends/atom.py @@ -55,6 +55,7 @@ def _cache_counts( metrics: dict[str, dict[str, Any]], endpoint_roles: Mapping[str, str] | None, disaggregated: bool | None, + cpu_offload_disabled: bool = False, ) -> tuple[float | None, float | None, float | None]: names = [ name @@ -77,10 +78,16 @@ def _cache_counts( return None, None, None full_total = cached_total = offload_total = 0.0 cpu_complete = True + offload_absent = not any( + s.get("endpoint_url") in accounted + for s in metric_series(metrics, _atom_counter_names("prefix_cache_offload_tokens")) + ) for endpoint in accounted: full, cached, offload = ( _counter_total(metrics, stem, endpoint) for stem in ("full", "cached", "offload") ) + if cpu_offload_disabled and offload_absent: + offload = 0.0 if full is None or cached is None or cached > full: return None, None, None full_total += full @@ -115,11 +122,28 @@ def populate( *, endpoint_roles: Mapping[str, str] | None = None, disaggregated: bool | None = None, + cpu_offload_disabled: bool = False, ) -> None: + prompt_names = _atom_counter_names("prompt_tokens") + prompt_endpoints = {s.get("endpoint_url") for s in metric_series(metrics, prompt_names)} + if endpoint_roles is None: + accounted = ( + prompt_endpoints if disaggregated is False and len(prompt_endpoints) == 1 else set() + ) + else: + accounted = { + url for url, role in endpoint_roles.items() if role in {"prefill", "unified"} + } + if prompt_endpoints - endpoint_roles.keys(): + accounted = set() + accounted = {url for url in accounted if isinstance(url, str) and url} + if accounted - prompt_endpoints: + accounted = set() prompt_total = sum_stat( metrics, - _atom_counter_names("prompt_tokens"), - preferred_keys=("total", "sum", "max", "avg"), + prompt_names, + preferred_keys=("total",), + series_filter=lambda s: s.get("endpoint_url") in accounted, ) generation_total = sum_stat( metrics, @@ -130,7 +154,7 @@ def populate( flat["total_generation_tokens"] = counter_int(generation_total) full_tokens, cached_tokens, external_tokens = _cache_counts( - metrics, endpoint_roles, disaggregated + metrics, endpoint_roles, disaggregated, cpu_offload_disabled ) cache_hit_rate = rate(cached_tokens, full_tokens) # Offload is CPU reuse beyond the GPU prefix; loaded tokens measure transfer volume. diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index e100011642..2ef47da115 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -16,8 +16,10 @@ def endpoint_roles_from_env(env: Mapping[str, str]) -> dict[str, str] | None: - if env.get("AIPERF_SERVER_METRICS_URLS") or env.get("SRTCTL_FRONTEND_TYPE") == "dynamo": + if env.get("SRTCTL_FRONTEND_TYPE") == "dynamo": return {} + if env.get("AIPERF_SERVER_METRICS_URLS"): + return None groups = ( [("unified", env["SRT_AGG_ENDPOINTS"])] if env.get("SRT_AGG_ENDPOINTS") @@ -77,6 +79,11 @@ def compute_server_metrics( nested, endpoint_roles=endpoint_roles_from_env(source_env), disaggregated={"true": True, "false": False}.get(source_env.get("DISAGG", "").lower()), + cpu_offload_disabled=(source_env.get("KV_OFFLOADING", "").strip().lower() == "none") + and not ( + source_env.get("KV_OFFLOAD_BACKEND") + or source_env.get("KV_OFFLOAD_BACKEND_METADATA") + ), ) else: adapter.populate(metrics, flat, nested) diff --git a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py index 2d0e6f433d..40ea3d9b89 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py @@ -41,9 +41,16 @@ def test_prefill_counters_exclude_decode_and_transfer_volume(decode_cached: floa export["metrics"]["atom:lmcache_loaded_tokens"] = { "series": [{"endpoint_url": "http://p/metrics", "stats": {"total": 900}}] } + export["metrics"]["atom:prompt_tokens"] = { + "series": [ + {"endpoint_url": f"http://{endpoint}/metrics", "stats": {"total": 1000}} + for endpoint in ("p", "d") + ] + } flat, nested = _compute( export, {"SRT_PREFILL_ENDPOINTS": "p", "SRT_DECODE_ENDPOINTS": "d", "DISAGG": "true"} ) + assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 1000 _assert_rates(flat, nested, 0.6, 0.1, 0.7) assert nested["cache"]["prefix_cache_queries"] == 1000 assert nested["cache"]["external_prefix_cache_hits"] == 100 @@ -55,6 +62,7 @@ def test_prefill_counters_exclude_decode_and_transfer_volume(decode_cached: floa [ ({"SRT_AGG_ENDPOINTS": "worker", "SRT_PREFILL_ENDPOINTS": "unused"}, 50, 0.5), ({"DISAGG": "false"}, 0, 0), + ({"DISAGG": "false", "AIPERF_SERVER_METRICS_URLS": "http://worker/metrics"}, 50, 0.5), ], ) def test_unified_zero_gpu_and_offload_counter_alias( @@ -188,8 +196,30 @@ def test_unknown_cache_roles_preserve_other_atom_telemetry() -> None: "atom:kv_cache_usage_ratio": {"series": [{"stats": {"max": 0.75}}]}, } ) - flat, nested = _compute(export, {}) + flat, nested, _ = compute_server_metrics( + export, + framework="mooncake-atom", + records=[{"metrics": {"input_sequence_length": {"value": 123}}}], + env={}, + ) _assert_rates(flat, nested, None, None, None) - assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 200 + assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 123 assert flat["total_generation_tokens"] == nested["tokens"]["generation_total"] == 30 assert flat["gpu_kv_cache_usage_pct"] == nested["kv_cache"]["gpu_usage_pct"] == 0.75 + + +@pytest.mark.parametrize( + ("env", "cpu", "overall"), + [ + ({"KV_OFFLOADING": " NONE "}, 0, 0.6), + ({"KV_OFFLOADING": "dram"}, None, None), + ({"KV_OFFLOADING": "none", "KV_OFFLOAD_BACKEND": "mooncake"}, None, None), + ], +) +def test_missing_offload_is_zero_only_with_confirmed_disabled_tier( + env: dict, cpu: float | None, overall: float | None +) -> None: + flat, nested = _compute( + _export({"worker": 100}, {"worker": 60}, {}), {"DISAGG": "false", **env} + ) + _assert_rates(flat, nested, 0.6, cpu, overall) From c14c0fde87eae59e1bf6194098255bf7078c9953 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Sun, 25 Oct 2026 17:25:34 +0000 Subject: [PATCH 12/12] fix(results): validate ATOM cache counters per label domain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：按端点与完整标签域校验缓存分层计数；仅在 CPU offload 明确关闭且现有计数均为零时补齐缺失零值，拒绝无效或矛盾的 offload 数据。 --- .../infx/results/agentic/backends/atom.py | 76 ++++++++++++------- .../agentic/test_atom_cache_counters.py | 38 ++++++++-- 2 files changed, 78 insertions(+), 36 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/backends/atom.py b/inferencex-e2e/infx/results/agentic/backends/atom.py index 183ca99832..32543a005b 100644 --- a/inferencex-e2e/infx/results/agentic/backends/atom.py +++ b/inferencex-e2e/infx/results/agentic/backends/atom.py @@ -31,24 +31,31 @@ def _atom_counter_names(stem: str) -> list[str]: return [*_atom_names(stem), *_atom_names(f"{stem}_total")] -def _counter_total(metrics: dict[str, dict[str, Any]], stem: str, endpoint: str) -> float | None: +def _counter_values( + metrics: dict[str, dict[str, Any]], stem: str, endpoint: str +) -> dict[tuple, float] | None: names = _atom_counter_names(f"prefix_cache_{stem}_tokens") - series = [s for s in metric_series(metrics, names) if s.get("endpoint_url") == endpoint] - if not series or any( - (value := series_stat(s, preferred_keys=("total",))) is None - or not math.isfinite(value) - or value < 0 - for s in series - ): - logger.warning("ATOM %s cache counter missing or invalid at %s", stem, endpoint) - return None - total = sum_stat( - metrics, - names, - preferred_keys=("total",), - series_filter=lambda s: s.get("endpoint_url") == endpoint, - ) - return total if total is not None and math.isfinite(total) else None + values = {} + for series in metric_series(metrics, names): + if series.get("endpoint_url") != endpoint: + continue + labels = series.get("labels") + labels = {} if labels is None else labels + value = series_stat(series, preferred_keys=("total",)) + if ( + not isinstance(labels, dict) + or any(not isinstance(k, str) or not isinstance(v, str) for k, v in labels.items()) + or value is None + or not math.isfinite(value) + or value < 0 + ): + return None + key = tuple(sorted(labels.items())) + if key in values: + logger.warning("ATOM %s cache counter duplicated at %s", stem, endpoint) + return None + values[key] = value + return values def _cache_counts( @@ -78,24 +85,35 @@ def _cache_counts( return None, None, None full_total = cached_total = offload_total = 0.0 cpu_complete = True - offload_absent = not any( - s.get("endpoint_url") in accounted - for s in metric_series(metrics, _atom_counter_names("prefix_cache_offload_tokens")) - ) for endpoint in accounted: full, cached, offload = ( - _counter_total(metrics, stem, endpoint) for stem in ("full", "cached", "offload") + _counter_values(metrics, stem, endpoint) for stem in ("full", "cached", "offload") ) - if cpu_offload_disabled and offload_absent: - offload = 0.0 - if full is None or cached is None or cached > full: + if ( + not full + or cached is None + or full.keys() != cached.keys() + or any(cached[key] > full[key] for key in full) + ): return None, None, None - full_total += full - cached_total += cached - if offload is None or cached + offload > full: + if cpu_offload_disabled: + offload = ( + dict.fromkeys(full, 0.0) + if offload is not None + and offload.keys() <= full.keys() + and not any(offload.values()) + else None + ) + full_total += sum(full.values()) + cached_total += sum(cached.values()) + if ( + offload is None + or offload.keys() != full.keys() + or any(cached[key] + offload[key] > full[key] for key in full) + ): cpu_complete = False else: - offload_total += offload + offload_total += sum(offload.values()) if full_total <= 0 or not all(math.isfinite(v) for v in (full_total, cached_total)): return None, None, None return ( diff --git a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py index 40ea3d9b89..8356e5bd39 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py @@ -177,13 +177,17 @@ def test_unreadable_series_does_not_get_silently_skipped() -> None: @pytest.mark.parametrize("offload", [30, math.nan]) +@pytest.mark.parametrize("labelled", [False, True]) def test_invalid_worker_cpu_count_cannot_hide_in_another_workers_denominator( - offload: float, + offload: float, labelled: bool ) -> None: - flat, nested = _compute( - _export({"p": 100, "q": 900}, {"p": 80, "q": 20}, {"p": offload, "q": 0}), - {"SRT_PREFILL_ENDPOINTS": "p,q"}, - ) + export = _export({"p": 100, "q": 900}, {"p": 80, "q": 20}, {"p": offload, "q": 0}) + if labelled: + for metric in export["metrics"].values(): + for series in metric["series"]: + series["labels"] = {"engine": series["endpoint_url"]} + series["endpoint_url"] = "http://worker/metrics" + flat, nested = _compute(export, {"SRT_PREFILL_ENDPOINTS": "worker" if labelled else "p,q"}) _assert_rates(flat, nested, 0.1, None, None) @@ -216,10 +220,30 @@ def test_unknown_cache_roles_preserve_other_atom_telemetry() -> None: ({"KV_OFFLOADING": "none", "KV_OFFLOAD_BACKEND": "mooncake"}, None, None), ], ) +@pytest.mark.parametrize("offload", [{}, {"p": 0}]) def test_missing_offload_is_zero_only_with_confirmed_disabled_tier( - env: dict, cpu: float | None, overall: float | None + env: dict, cpu: float | None, overall: float | None, offload: dict ) -> None: flat, nested = _compute( - _export({"worker": 100}, {"worker": 60}, {}), {"DISAGG": "false", **env} + _export({"p": 100, "q": 100}, {"p": 60, "q": 60}, offload), + {"SRT_PREFILL_ENDPOINTS": "p,q", **env}, ) _assert_rates(flat, nested, 0.6, cpu, overall) + + +@pytest.mark.parametrize("offload", [10, math.nan, None]) +def test_disabled_cpu_tier_does_not_hide_invalid_present_offload(offload: object) -> None: + flat, nested = _compute( + _export({"p": 100, "q": 100}, {"p": 60, "q": 60}, {"p": offload}), + {"SRT_PREFILL_ENDPOINTS": "p,q", "KV_OFFLOADING": "none"}, + ) + _assert_rates(flat, nested, 0.6, None, None) + + +def test_disabled_cpu_tier_does_not_fill_an_unmatched_label_domain() -> None: + export = _export({"worker": 100}, {"worker": 60}, {"worker": 0}) + export["metrics"]["atom:prefix_cache_offload_tokens"]["series"][0]["labels"] = { + "engine": "other" + } + flat, nested = _compute(export, {"DISAGG": "false", "KV_OFFLOADING": "none"}) + _assert_rates(flat, nested, 0.6, None, None)