From 5a695ba78879a322dfb0e93e424ebb29d4b11bc2 Mon Sep 17 00:00:00 2001 From: Yi Date: Wed, 14 Oct 2026 00:49:03 +0800 Subject: [PATCH 1/9] perf(amd): DSv4.1-Flash ATOM DPA + prefill TBO from c32 on the 1013 nightly Move dsv41flash-fp4-mi355x-atom-agentic-dspark to rocm/atom-dev:nightly_202610131505 (ATOM 661ef7661, with ROCm/ATOM#2376 V4.1 DPA and prefill TBO). Run TP2 c32/64/128/256 and TP4 c32 with DP attention, prefill TBO, DP session affinity and the prefill delayer; drop TP2 c6. Co-Authored-By: Claude Opus 5.5 --- .../atom/mi355x-fp4-mtp/agentic.yaml | 94 ++++++++++++++----- inferencex-e2e/configs/amd-master.yaml | 11 ++- inferencex-e2e/perf-changelog.yaml | 10 ++ 3 files changed, 87 insertions(+), 28 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 39f78b50d4..ff4370edcb 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -1,20 +1,21 @@ # DeepSeek-V4.1-Flash AgentX on MI355X with ATOM DSpark (five draft tokens), # following ROCm/ATOM recipes/DeepSeek-V4.1-Flash-Agentic.md at the image's -# revision (a0527c9dbfb801e22cd29882454df6f6d21cdd91). TP2 and TP4, no expert -# parallelism; the KV cache is GPU-resident. +# revision (661ef7661eef0d804d2b52d81ed41d80aeaf9fdb). TP2 and TP4, no expert +# parallelism, DP attention with prefill TBO (ROCm/ATOM#2376) from concurrency +# 32; the KV cache is GPU-resident. base: schema: 2 name: dsv41flash-fp4-mi355x-atom-agentic model: path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: rocm/atom-dev:nightly_202610121516 + container: rocm/atom-dev:nightly_202610131505 precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:a60811a8467b43e3e50e9e9eb8cec34f68b7bca03f8c6dbd6a0ff813dde49d4c + container_image: rocm/atom-dev@sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884 enable_multiple_frontends: false observability: enabled: false @@ -66,7 +67,10 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' # One variant per point. Every point admits 128 sequences and captures every -# decode size through 32 plus 48, 64 and 128. +# decode size through 32 plus 48, 64 and 128. The _dpa points (concurrency 32 +# and above) run DP attention with prefill TBO, keep each AIPerf session on one +# DP rank and defer prefill so that at least 10 decode passes run between +# prefills. override_tp2_c1: roles: @@ -92,14 +96,6 @@ override_tp2_c4: env: CONC: '4' -override_tp2_c6: - roles: - agg: - gpus: 2 - benchmark: - env: - CONC: '6' - override_tp2_c8: roles: agg: @@ -116,25 +112,76 @@ override_tp2_c16: env: CONC: '16' -override_tp2_c32: +override_tp2_c32_dpa: roles: agg: gpus: 2 + args: &dpa_args + enable-dp-attention: true + enable-tbo: true + # ATOM_DP_LB_REQ_EQUIV weights in-flight requests under least_tokens. + dp-load-balance: least_tokens + env: &dpa_env + GPU_MAX_HW_QUEUES: '5' + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' benchmark: env: CONC: '32' + # Session affinity keys on the AIPerf session id. + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' -override_tp2_c64: +override_tp2_c64_dpa: roles: agg: gpus: 2 - # Run at least 4 decode passes after each prefill and batch the prefills - # that arrive meanwhile; at this load the batching also raises throughput. - env: - ATOM_PREFILL_DECODE_INTERVAL: '4' + args: *dpa_args + env: *dpa_env benchmark: env: CONC: '64' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_tp2_c128_dpa: + roles: + agg: + gpus: 2 + args: *dpa_args + env: *dpa_env + benchmark: + env: + CONC: '128' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_tp2_c256_dpa: + roles: + agg: + gpus: 2 + args: *dpa_args + env: *dpa_env + benchmark: + env: + CONC: '256' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + +# Plain-TP companions of the DP-attention points above, at the same concurrency. +override_tp2_c128: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '128' + +override_tp2_c256: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '256' override_tp4_c1: roles: @@ -152,14 +199,13 @@ override_tp4_c8: env: CONC: '8' -override_tp4_c32: +override_tp4_c32_dpa: roles: agg: gpus: 4 - # Run at least 16 decode passes after each prefill and batch the prefills - # that arrive meanwhile, so new turns stall decode less often. - env: - ATOM_PREFILL_DECODE_INTERVAL: '16' + args: *dpa_args + env: *dpa_env benchmark: env: CONC: '32' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 48a89330eb..af44ff47f4 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1518,10 +1518,11 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } -# DeepSeek-V4.1-Flash on ATOM, following the upstream TP2 / TP4 AgentX recipe. +# DeepSeek-V4.1-Flash on ATOM, following the upstream TP2 / TP4 AgentX recipe, +# with DP attention and prefill TBO from concurrency 32. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610121516 + image: rocm/atom-dev:nightly_202610131505 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds @@ -1531,8 +1532,10 @@ dsv41flash-fp4-mi355x-atom-agentic-dspark: scenarios: agentic-coding: - search-space: - - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 6, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 8, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 128, 256], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [32, 64, 128, 256], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 8], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } # The golden acceptance curve remains the provisional GLM-5.2-derived curve # documented in infx/golden_al_distribution/glm5.3_mtp.yaml. diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 17ff00f039..f77bbf063a 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9646,3 +9646,13 @@ - "ATOM's mono decode path (on by default) now takes any decode step of up to 48 rows, i.e. concurrency 1-8 with five DSpark drafts, not only DSpark verify steps. Steps of 12 or more rows that are a multiple of TP run its sublayers token-parallel, with reduce-scatters instead of all-reduces. The draft runner never runs token-parallel." - "No data-type or precision change to the DSpark draft. The checkpoint's bundled mtp.0-2 stages are stored like the target's layers: FP8 E4M3 weights with 32x32 UE8M0 block scales for the attention projections, shared experts and main_proj; MXFP4 routed experts; BF16 norms, router gate, Markov head and confidence head; F32 attention sinks and hc mixers. ATOM builds the draft with the target's native_quant_config() and make_v4_quant_config(), with no online quantization in this recipe, so it runs as shipped: FP8 activations on the FP8 GEMMs and the target's BF16 KV cache. Between 73defd9b8 and a0527c9db only the rows the draft reads under bounded replay change." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3928 + +- config-keys: + - dsv41flash-fp4-mi355x-atom-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610131505 (digest sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884; ATOM 0.1.8.dev44+g661ef7661, ROCm 7.2.4) and update model.container and the AToMesh frontend image to match. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention and prefill TBO), ROCm/ATOM#2543 (V4 FP4 graph replay with fixed-capacity metadata) and ROCm/ATOM#2556 (FP4 decode bounds drop the DP-padded empty tail)." + - "Run DP attention with prefill TBO (--enable-dp-attention --enable-tbo) from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on every point." + - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 1273f8b54f95727418970403bafbf151072f5d62 Mon Sep 17 00:00:00 2001 From: Yi Date: Wed, 14 Oct 2026 00:59:25 +0800 Subject: [PATCH 2/9] chore: set perf-changelog pr-link to #3965 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index f77bbf063a..32d7b4e684 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9655,4 +9655,4 @@ - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610131505 (digest sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884; ATOM 0.1.8.dev44+g661ef7661, ROCm 7.2.4) and update model.container and the AToMesh frontend image to match. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention and prefill TBO), ROCm/ATOM#2543 (V4 FP4 graph replay with fixed-capacity metadata) and ROCm/ATOM#2556 (FP4 decode bounds drop the DP-padded empty tail)." - "Run DP attention with prefill TBO (--enable-dp-attention --enable-tbo) from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on every point." - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From cd3daee00aed8492a7f8744020b62902f8238e5b Mon Sep 17 00:00:00 2001 From: Yi Date: Fri, 16 Oct 2026 01:06:22 +0800 Subject: [PATCH 3/9] perf(amd): move DSv4.1-Flash ATOM AgentX to the 1015 nightly Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 6 +++--- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index ff4370edcb..66531143da 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -1,6 +1,6 @@ # DeepSeek-V4.1-Flash AgentX on MI355X with ATOM DSpark (five draft tokens), # following ROCm/ATOM recipes/DeepSeek-V4.1-Flash-Agentic.md at the image's -# revision (661ef7661eef0d804d2b52d81ed41d80aeaf9fdb). TP2 and TP4, no expert +# revision (e6182e08b85f0e58bc15f7393ef57899220bfbaf). TP2 and TP4, no expert # parallelism, DP attention with prefill TBO (ROCm/ATOM#2376) from concurrency # 32; the KV cache is GPU-resident. base: @@ -8,14 +8,14 @@ base: name: dsv41flash-fp4-mi355x-atom-agentic model: path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: rocm/atom-dev:nightly_202610131505 + container: rocm/atom-dev:nightly_202610151507 precision: fp4 resources: gpu_type: mi355x gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884 + container_image: rocm/atom-dev@sha256:aa6754256dd68009240e54206265e5250395374640f07757f2fdeac6170097cf enable_multiple_frontends: false observability: enabled: false diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index af44ff47f4..db858ff525 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1522,7 +1522,7 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: # with DP attention and prefill TBO from concurrency 32. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610131505 + image: rocm/atom-dev:nightly_202610151507 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 32d7b4e684..24f853a332 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9652,7 +9652,7 @@ scenario-type: - agentic-coding description: - - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610131505 (digest sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884; ATOM 0.1.8.dev44+g661ef7661, ROCm 7.2.4) and update model.container and the AToMesh frontend image to match. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention and prefill TBO), ROCm/ATOM#2543 (V4 FP4 graph replay with fixed-capacity metadata) and ROCm/ATOM#2556 (FP4 decode bounds drop the DP-padded empty tail)." + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610151507 (digest sha256:aa6754256dd68009240e54206265e5250395374640f07757f2fdeac6170097cf; ATOM 0.1.8.dev63+ge6182e08b, ROCm 7.2.4) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention and prefill TBO), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention) and ROCm/ATOM#2588 (prompt hashing and the prefix walk off the GPU critical path)." - "Run DP attention with prefill TBO (--enable-dp-attention --enable-tbo) from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on every point." - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From 57f16ecf170f001cdd6f5d8987970fb981792f67 Mon Sep 17 00:00:00 2001 From: Yi Date: Thu, 22 Oct 2026 11:48:19 -0500 Subject: [PATCH 4/9] perf(amd): move DSv4.1-Flash ATOM AgentX to the 1022 nightly rocm/atom-dev:nightly_202610221638 (sha256:32c81e56e468...; ATOM 0.1.9.dev11+gce46f6877, ROCm 7.2.4). Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 2 +- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index eada9477a3..2d5be82958 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:aa6754256dd68009240e54206265e5250395374640f07757f2fdeac6170097cf + container_image: rocm/atom-dev@sha256:32c81e56e4680065b6ea3e0ca8363a00cd3ccb932d8990e83d799b8bbb58a4b3 enable_multiple_frontends: false observability: enabled: false diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 02d7e5ccc0..bd2a75c0de 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1536,7 +1536,7 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: # with DP attention and prefill TBO from concurrency 32. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610151507 + image: rocm/atom-dev:nightly_202610221638 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index e3b51f12d1..1f12102387 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10812,7 +10812,7 @@ scenario-type: - agentic-coding description: - - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610151507 (digest sha256:aa6754256dd68009240e54206265e5250395374640f07757f2fdeac6170097cf; ATOM 0.1.8.dev63+ge6182e08b, ROCm 7.2.4) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention and prefill TBO), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention) and ROCm/ATOM#2588 (prompt hashing and the prefix walk off the GPU critical path)." + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610221638 (digest sha256:32c81e56e4680065b6ea3e0ca8363a00cd3ccb932d8990e83d799b8bbb58a4b3; ATOM 0.1.9.dev11+gce46f6877, ROCm 7.2.4) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention and prefill TBO), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." - "Run DP attention with prefill TBO (--enable-dp-attention --enable-tbo) from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on every point." - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From 7a22fb85158e44fde35ec1f0dad03f4b0ba210e3 Mon Sep 17 00:00:00 2001 From: Yi Date: Sat, 24 Oct 2026 01:40:48 +0800 Subject: [PATCH 5/9] perf(amd): DSv4.1-Flash ATOM AgentX on nightly_202610231734 Use the ATOM 2c772a5 release image (ROCm/ATOM run 39986380362, sha256:01a19e8b...) instead of nightly_202610231458. Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 2 +- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 2f6bc4b048..71a1c4585a 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:05322b8651303be041b7adf2ee7d562815688ebe88ae2a11b3f8f2fc89adde61 + container_image: rocm/atom-dev@sha256:01a19e8b14ad151c7f90a7862319f43796d8261e34e2546ab5dfabcce5cda105 enable_multiple_frontends: false observability: enabled: false diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 18e674c96b..a910e86b50 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1557,7 +1557,7 @@ minimaxm31flash-fp4-mi355x-vllm-agentic-dspark: # with DP attention (no TBO) from concurrency 32. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610231458 + image: rocm/atom-dev:nightly_202610231734 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index e2ada942ca..c31643f7d8 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11003,7 +11003,7 @@ scenario-type: - agentic-coding description: - - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610231458 (digest sha256:05322b8651303be041b7adf2ee7d562815688ebe88ae2a11b3f8f2fc89adde61) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610231734 (ATOM 2c772a5, digest sha256:01a19e8b14ad151c7f90a7862319f43796d8261e34e2546ab5dfabcce5cda105) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on every point." - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From 67b40a6853fde6a14c750d843227418ca2414457 Mon Sep 17 00:00:00 2001 From: Yi Date: Sat, 24 Oct 2026 01:53:24 +0800 Subject: [PATCH 6/9] perf(amd): DSv4.1-Flash ATOM TP2 c256 plain TP admits 256 sequences max-num-seqs 128 queued half of the 256 sessions (TTFT 490 s); raise it to 256 and capture a 256 decode graph on that point. Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 10 ++++++++-- inferencex-e2e/perf-changelog.yaml | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 71a1c4585a..df4d4997b3 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -52,8 +52,9 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point. Every point admits 128 sequences and captures every -# decode size through 32 plus 48, 64 and 128. The _dpa points (concurrency 32 +# One variant per point. Every point admits 128 sequences (256 on plain-TP TP2 +# c256) and captures every decode size through 32 plus 48, 64 and 128 (and 256 +# there). The _dpa points (concurrency 32 # and above) run DP attention without TBO (TBO does not support SWA bounded # replay), keep each AIPerf session on one DP rank and defer prefill so that at # least 10 decode passes run between prefills. @@ -164,6 +165,11 @@ override_tp2_c256: roles: agg: gpus: 2 + # Admit every session: at 128 sequences half of the 256 queue behind the + # running batch. + args: + max-num-seqs: 256 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' benchmark: env: CONC: '256' diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 82e682cb0f..977520059e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11014,6 +11014,6 @@ - agentic-coding description: - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610231734 (ATOM 2c772a5, digest sha256:01a19e8b14ad151c7f90a7862319f43796d8261e34e2546ab5dfabcce5cda105) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." - - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on every point." - - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." + - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on these points." + - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points; plain-TP TP2 c256 raises max-num-seqs from 128 to 256 and adds 256 to cudagraph-capture-sizes so every session is admitted (at 128 the previous run queued half the sessions, TTFT 490 s). Other server settings are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From 94c8a190b8d4853528d255fdbb928ade28929f38 Mon Sep 17 00:00:00 2001 From: Yi Date: Sat, 24 Oct 2026 16:38:16 +0800 Subject: [PATCH 7/9] perf(amd): DSv4.1-Flash ATOM AgentX on nightly_202610240802, max-num-seqs as B300 vLLM Move to the ATOM 11a2d56 release image (ROCm/ATOM run 40072622970, sha256:42716923...) and set max-num-seqs per point (per DP rank) like the B300 vLLM AgentX recipe: 1 at c1, 2x concurrency for c2-c16, concurrency from c32 up; capture sizes stop at each point's max-num-seqs. Co-Authored-By: Claude Opus 5.5 --- .../atom/mi355x-fp4-mtp/agentic.yaml | 73 +++++++++++++++---- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 7 +- 3 files changed, 62 insertions(+), 20 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index df4d4997b3..7df0878eab 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:01a19e8b14ad151c7f90a7862319f43796d8261e34e2546ab5dfabcce5cda105 + container_image: rocm/atom-dev@sha256:4271692397830e1198b6aee1fe81c29c39f3704b0816f2746162f778cfa93048 enable_multiple_frontends: false observability: enabled: false @@ -23,7 +23,6 @@ base: kv_cache_dtype: bf16 index-cache-dtype: fp4 gpu-memory-utilization: 0.9 - max-num-seqs: 128 max-num-batched-tokens: 16384 attn-prefill-chunk-size: 16384 enable_prefix_caching: true @@ -31,7 +30,6 @@ base: state-checkpoint-interval-tokens: 8192 level: 3 cudagraph-mode: FULL - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' # The checkpoint's bundled DSpark head. Throughput runs get the golden # AL (dsv41flash_dspark thinking_on, K5) from infx/srt_slurm/synthetic_acceptance.py; # evals keep real acceptance. @@ -52,17 +50,22 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point. Every point admits 128 sequences (256 on plain-TP TP2 -# c256) and captures every decode size through 32 plus 48, 64 and 128 (and 256 -# there). The _dpa points (concurrency 32 -# and above) run DP attention without TBO (TBO does not support SWA bounded -# replay), keep each AIPerf session on one DP rank and defer prefill so that at -# least 10 decode passes run between prefills. +# One variant per point. max-num-seqs (per DP rank under DP attention) follows +# the B300 vLLM AgentX recipe: one sequence at concurrency 1, twice the +# concurrency from 2 to 16, and the concurrency from 32 up. Each point captures +# every decode size through min(32, max-num-seqs) plus 48, 64, 128 and 256 up to +# its max-num-seqs. The _dpa points (concurrency 32 and above) run DP attention +# without TBO (TBO does not support SWA bounded replay), keep each AIPerf +# session on one DP rank and defer prefill so that at least 10 decode passes run +# between prefills. override_tp2_c1: roles: agg: gpus: 2 + args: + max-num-seqs: 1 + cudagraph-capture-sizes: '[1]' benchmark: env: CONC: '1' @@ -71,6 +74,9 @@ override_tp2_c2: roles: agg: gpus: 2 + args: + max-num-seqs: 4 + cudagraph-capture-sizes: '[1,2,3,4]' benchmark: env: CONC: '2' @@ -79,6 +85,9 @@ override_tp2_c4: roles: agg: gpus: 2 + args: + max-num-seqs: 8 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8]' benchmark: env: CONC: '4' @@ -87,6 +96,9 @@ override_tp2_c8: roles: agg: gpus: 2 + args: + max-num-seqs: 16 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' benchmark: env: CONC: '8' @@ -95,6 +107,9 @@ override_tp2_c16: roles: agg: gpus: 2 + args: + max-num-seqs: 32 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' benchmark: env: CONC: '16' @@ -103,10 +118,12 @@ override_tp2_c32_dpa: roles: agg: gpus: 2 - args: &dpa_args + args: enable-dp-attention: true # ATOM_DP_LB_REQ_EQUIV weights in-flight requests under least_tokens. dp-load-balance: least_tokens + max-num-seqs: 32 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' env: &dpa_env GPU_MAX_HW_QUEUES: '5' ATOM_DP_SESSION_AFFINITY: '1' @@ -123,7 +140,11 @@ override_tp2_c64_dpa: roles: agg: gpus: 2 - args: *dpa_args + args: + enable-dp-attention: true + dp-load-balance: least_tokens + max-num-seqs: 64 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64]' env: *dpa_env benchmark: env: @@ -134,7 +155,11 @@ override_tp2_c128_dpa: roles: agg: gpus: 2 - args: *dpa_args + args: + enable-dp-attention: true + dp-load-balance: least_tokens + max-num-seqs: 128 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' env: *dpa_env benchmark: env: @@ -145,7 +170,11 @@ override_tp2_c256_dpa: roles: agg: gpus: 2 - args: *dpa_args + args: + enable-dp-attention: true + dp-load-balance: least_tokens + max-num-seqs: 256 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' env: *dpa_env benchmark: env: @@ -153,10 +182,14 @@ override_tp2_c256_dpa: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' # Plain-TP companions of the DP-attention points above, at the same concurrency. + override_tp2_c128: roles: agg: gpus: 2 + args: + max-num-seqs: 128 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' benchmark: env: CONC: '128' @@ -165,8 +198,6 @@ override_tp2_c256: roles: agg: gpus: 2 - # Admit every session: at 128 sequences half of the 256 queue behind the - # running batch. args: max-num-seqs: 256 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' @@ -178,6 +209,9 @@ override_tp4_c1: roles: agg: gpus: 4 + args: + max-num-seqs: 1 + cudagraph-capture-sizes: '[1]' benchmark: env: CONC: '1' @@ -186,6 +220,9 @@ override_tp4_c8: roles: agg: gpus: 4 + args: + max-num-seqs: 16 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' benchmark: env: CONC: '8' @@ -194,7 +231,11 @@ override_tp4_c32_dpa: roles: agg: gpus: 4 - args: *dpa_args + args: + enable-dp-attention: true + dp-load-balance: least_tokens + max-num-seqs: 32 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' env: *dpa_env benchmark: env: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index a910e86b50..60ea330b6b 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1557,7 +1557,7 @@ minimaxm31flash-fp4-mi355x-vllm-agentic-dspark: # with DP attention (no TBO) from concurrency 32. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610231734 + image: rocm/atom-dev:nightly_202610240802 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 977520059e..eb7fc8111c 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11013,7 +11013,8 @@ scenario-type: - agentic-coding description: - - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610231734 (ATOM 2c772a5, digest sha256:01a19e8b14ad151c7f90a7862319f43796d8261e34e2546ab5dfabcce5cda105) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." - - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32). max-num-seqs stays 128 on these points." - - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points; plain-TP TP2 c256 raises max-num-seqs from 128 to 256 and adds 256 to cudagraph-capture-sizes so every session is admitted (at 128 the previous run queued half the sessions, TTFT 490 s). Other server settings are unchanged." + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610240802 (ATOM 11a2d56, digest sha256:4271692397830e1198b6aee1fe81c29c39f3704b0816f2746162f778cfa93048) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." + - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32)." + - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." + - "Align max-num-seqs with the B300 vLLM AgentX recipe instead of 128 on every point (per DP rank under DP attention, as in vLLM): 1 at concurrency 1, twice the concurrency from 2 to 16 (TP2 c2/c4/c8/c16 -> 4/8/16/32, TP4 c8 -> 16), and the concurrency from 32 up (TP2 DP-attention c32/c64/c128/c256 -> 32/64/128/256, TP4 DP-attention c32 -> 32, plain-TP TP2 c128/c256 -> 128/256; at 128 the plain-TP c256 point queued half the sessions, TTFT 490 s). cudagraph-capture-sizes is cut to each point's max-num-seqs and adds 256 where it is 256." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From c4815ef5c57f7c9d2d8deed9f2fde0011b65ec9b Mon Sep 17 00:00:00 2001 From: Yi Date: Sat, 24 Oct 2026 17:24:42 +0800 Subject: [PATCH 8/9] perf(amd): DSv4.1-Flash ATOM c1 admits 2 sequences max-num-seqs is twice the concurrency at c1 too (TP2 and TP4). Co-Authored-By: Claude Opus 5.5 --- .../dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml | 12 ++++++------ inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 7df0878eab..9be4cd78d4 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -51,8 +51,8 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' # One variant per point. max-num-seqs (per DP rank under DP attention) follows -# the B300 vLLM AgentX recipe: one sequence at concurrency 1, twice the -# concurrency from 2 to 16, and the concurrency from 32 up. Each point captures +# the B300 vLLM AgentX recipe: twice the concurrency from 1 to 16 and the +# concurrency from 32 up. Each point captures # every decode size through min(32, max-num-seqs) plus 48, 64, 128 and 256 up to # its max-num-seqs. The _dpa points (concurrency 32 and above) run DP attention # without TBO (TBO does not support SWA bounded replay), keep each AIPerf @@ -64,8 +64,8 @@ override_tp2_c1: agg: gpus: 2 args: - max-num-seqs: 1 - cudagraph-capture-sizes: '[1]' + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' benchmark: env: CONC: '1' @@ -210,8 +210,8 @@ override_tp4_c1: agg: gpus: 4 args: - max-num-seqs: 1 - cudagraph-capture-sizes: '[1]' + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' benchmark: env: CONC: '1' diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index c4838e3541..a48fa7adff 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11080,5 +11080,5 @@ - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610240802 (ATOM 11a2d56, digest sha256:4271692397830e1198b6aee1fe81c29c39f3704b0816f2746162f778cfa93048) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32)." - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." - - "Align max-num-seqs with the B300 vLLM AgentX recipe instead of 128 on every point (per DP rank under DP attention, as in vLLM): 1 at concurrency 1, twice the concurrency from 2 to 16 (TP2 c2/c4/c8/c16 -> 4/8/16/32, TP4 c8 -> 16), and the concurrency from 32 up (TP2 DP-attention c32/c64/c128/c256 -> 32/64/128/256, TP4 DP-attention c32 -> 32, plain-TP TP2 c128/c256 -> 128/256; at 128 the plain-TP c256 point queued half the sessions, TTFT 490 s). cudagraph-capture-sizes is cut to each point's max-num-seqs and adds 256 where it is 256." + - "Align max-num-seqs with the B300 vLLM AgentX recipe instead of 128 on every point (per DP rank under DP attention, as in vLLM): twice the concurrency from 1 to 16 (TP2 c1/c2/c4/c8/c16 -> 2/4/8/16/32, TP4 c1/c8 -> 2/16), and the concurrency from 32 up (TP2 DP-attention c32/c64/c128/c256 -> 32/64/128/256, TP4 DP-attention c32 -> 32, plain-TP TP2 c128/c256 -> 128/256; at 128 the plain-TP c256 point queued half the sessions, TTFT 490 s). cudagraph-capture-sizes is cut to each point's max-num-seqs and adds 256 where it is 256." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965 From 9542f0d294a270eb92e7667e08c9c7deb9ad1b39 Mon Sep 17 00:00:00 2001 From: Yi Date: Sat, 24 Oct 2026 17:33:47 +0800 Subject: [PATCH 9/9] perf(amd): DSv4.1-Flash ATOM AgentX on ATOM's default max-num-seqs Drop every max-num-seqs setting; decode graphs go back to the base list (sizes through 32 plus 48, 64, 128), with 256 added on the c256 points. Co-Authored-By: Claude Opus 5.5 --- .../atom/mi355x-fp4-mtp/agentic.yaml | 49 +++---------------- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 8 insertions(+), 43 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 9be4cd78d4..176c830812 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -30,6 +30,7 @@ base: state-checkpoint-interval-tokens: 8192 level: 3 cudagraph-mode: FULL + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' # The checkpoint's bundled DSpark head. Throughput runs get the golden # AL (dsv41flash_dspark thinking_on, K5) from infx/srt_slurm/synthetic_acceptance.py; # evals keep real acceptance. @@ -50,22 +51,17 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point. max-num-seqs (per DP rank under DP attention) follows -# the B300 vLLM AgentX recipe: twice the concurrency from 1 to 16 and the -# concurrency from 32 up. Each point captures -# every decode size through min(32, max-num-seqs) plus 48, 64, 128 and 256 up to -# its max-num-seqs. The _dpa points (concurrency 32 and above) run DP attention -# without TBO (TBO does not support SWA bounded replay), keep each AIPerf -# session on one DP rank and defer prefill so that at least 10 decode passes run -# between prefills. +# One variant per point. max-num-seqs is ATOM's default on every point; decode +# graphs cover every size through 32 plus 48, 64 and 128 (and 256 on the c256 +# points). The _dpa points (concurrency 32 and above) run DP attention without +# TBO (TBO does not support SWA bounded replay), keep each AIPerf session on +# one DP rank and defer prefill so that at least 10 decode passes run between +# prefills. override_tp2_c1: roles: agg: gpus: 2 - args: - max-num-seqs: 2 - cudagraph-capture-sizes: '[1,2]' benchmark: env: CONC: '1' @@ -74,9 +70,6 @@ override_tp2_c2: roles: agg: gpus: 2 - args: - max-num-seqs: 4 - cudagraph-capture-sizes: '[1,2,3,4]' benchmark: env: CONC: '2' @@ -85,9 +78,6 @@ override_tp2_c4: roles: agg: gpus: 2 - args: - max-num-seqs: 8 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8]' benchmark: env: CONC: '4' @@ -96,9 +86,6 @@ override_tp2_c8: roles: agg: gpus: 2 - args: - max-num-seqs: 16 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' benchmark: env: CONC: '8' @@ -107,9 +94,6 @@ override_tp2_c16: roles: agg: gpus: 2 - args: - max-num-seqs: 32 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' benchmark: env: CONC: '16' @@ -122,8 +106,6 @@ override_tp2_c32_dpa: enable-dp-attention: true # ATOM_DP_LB_REQ_EQUIV weights in-flight requests under least_tokens. dp-load-balance: least_tokens - max-num-seqs: 32 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' env: &dpa_env GPU_MAX_HW_QUEUES: '5' ATOM_DP_SESSION_AFFINITY: '1' @@ -143,8 +125,6 @@ override_tp2_c64_dpa: args: enable-dp-attention: true dp-load-balance: least_tokens - max-num-seqs: 64 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64]' env: *dpa_env benchmark: env: @@ -158,8 +138,6 @@ override_tp2_c128_dpa: args: enable-dp-attention: true dp-load-balance: least_tokens - max-num-seqs: 128 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' env: *dpa_env benchmark: env: @@ -173,7 +151,6 @@ override_tp2_c256_dpa: args: enable-dp-attention: true dp-load-balance: least_tokens - max-num-seqs: 256 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' env: *dpa_env benchmark: @@ -187,9 +164,6 @@ override_tp2_c128: roles: agg: gpus: 2 - args: - max-num-seqs: 128 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' benchmark: env: CONC: '128' @@ -199,7 +173,6 @@ override_tp2_c256: agg: gpus: 2 args: - max-num-seqs: 256 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' benchmark: env: @@ -209,9 +182,6 @@ override_tp4_c1: roles: agg: gpus: 4 - args: - max-num-seqs: 2 - cudagraph-capture-sizes: '[1,2]' benchmark: env: CONC: '1' @@ -220,9 +190,6 @@ override_tp4_c8: roles: agg: gpus: 4 - args: - max-num-seqs: 16 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' benchmark: env: CONC: '8' @@ -234,8 +201,6 @@ override_tp4_c32_dpa: args: enable-dp-attention: true dp-load-balance: least_tokens - max-num-seqs: 32 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' env: *dpa_env benchmark: env: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index a48fa7adff..9641067e48 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11080,5 +11080,5 @@ - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610240802 (ATOM 11a2d56, digest sha256:4271692397830e1198b6aee1fe81c29c39f3704b0816f2746162f778cfa93048) and update model.container and the AToMesh frontend image to match. Upstream recipes/DeepSeek-V4.1-Flash-Agentic.md is unchanged. The image adds ROCm/ATOM#2376 (DeepSeek-V4.1 DP attention), ROCm/ATOM#2543 and ROCm/ATOM#2556 (FP4 fixes for DP-padded batches), ROCm/ATOM#2551 (DP-attention prefill steps balanced by estimated chunk cost), ROCm/ATOM#2569 (decoder SWA bounded replay under DP attention), ROCm/ATOM#2588 and ROCm/ATOM#2599 (prompt hashing and the prefix walk off the GPU critical path), ROCm/ATOM#2429 (DP-attention comm-fused MoE reduce-scatter) and ROCm/ATOM#2670 (reused speculative output buffers)." - "Run DP attention (--enable-dp-attention) without TBO from concurrency 32: TP2 at concurrency 32, 64, 128 and 256 (128 and 256 are new) and TP4 at concurrency 32. TBO is not enabled because it does not support SWA bounded replay. These points set GPU_MAX_HW_QUEUES=5, ATOM_DP_SESSION_AFFINITY=1 with AIPerf sending the session id, dp-load-balance least_tokens with ATOM_DP_LB_REQ_EQUIV=512, and ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=10 (replacing the interval of 4 at TP2 c64 and 16 at TP4 c32)." - "Drop TP2 concurrency 6. TP2 runs [1, 2, 4, 8, 16] and TP4 runs [1, 8] on plain TP; TP2 also runs concurrency 128 and 256 on plain TP, without any of the DP settings, next to the DP-attention points. Other server settings are unchanged." - - "Align max-num-seqs with the B300 vLLM AgentX recipe instead of 128 on every point (per DP rank under DP attention, as in vLLM): twice the concurrency from 1 to 16 (TP2 c1/c2/c4/c8/c16 -> 2/4/8/16/32, TP4 c1/c8 -> 2/16), and the concurrency from 32 up (TP2 DP-attention c32/c64/c128/c256 -> 32/64/128/256, TP4 DP-attention c32 -> 32, plain-TP TP2 c128/c256 -> 128/256; at 128 the plain-TP c256 point queued half the sessions, TTFT 490 s). cudagraph-capture-sizes is cut to each point's max-num-seqs and adds 256 where it is 256." + - "Drop the max-num-seqs 128 cap: every point runs ATOM's default max-num-seqs (at 128 the plain-TP TP2 c256 point queued half the sessions, TTFT 490 s). Decode graphs stay at every size through 32 plus 48, 64 and 128; the TP2 c256 points (plain TP and DP attention) also capture 256." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965