diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml index 881ce3cb68..2d937e42e0 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:a60811a8467b43e3e50e9e9eb8cec34f68b7bca03f8c6dbd6a0ff813dde49d4c + container_image: rocm/atom-dev@sha256:96a9614276518c2619e88800dec5f276bc76a95691331b4abb2210184e1431af enable_multiple_frontends: false observability: enabled: false @@ -20,10 +20,9 @@ base: args: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true - kv_cache_dtype: bf16 + kv_cache_dtype: fp8 index-cache-dtype: fp4 gpu-memory-utilization: 0.9 - max-num-seqs: 128 max-num-batched-tokens: 16384 attn-prefill-chunk-size: 16384 enable_prefix_caching: true @@ -50,13 +49,26 @@ base: AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' -# One variant per point. Every point admits 128 sequences and captures every -# decode size through 32 plus 48, 64 and 128. +# One variant per point, on an MXFP8 KV cache. Plain TP everywhere, plus DEP2 +# (DP attention and expert parallelism over the TP group) from concurrency 64. +# max-num-seqs is ATOM's default except at concurrency 1; decode graphs cover +# every size through 32 plus 48, 64 and 128 (and 256 on the c256 points). +# +# Concurrency 1 follows the C1 recipe of ROCm/ATOM#2659: the prefill delayer +# with a decode interval no step reaches, so a new turn's prefill waits for the +# running decode instead of interleaving with it, and max-num-seqs 2 (DSpark's +# block-drafter warmup needs two sequences) with decode graphs at 1 and 2. override_tp2_c1: roles: agg: gpus: 2 + args: &c1_args + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' + env: &c1_env + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '1000000' benchmark: env: CONC: '1' @@ -77,14 +89,6 @@ override_tp2_c4: env: CONC: '4' -override_tp2_c6: - roles: - agg: - gpus: 2 - benchmark: - env: - CONC: '6' - override_tp2_c8: roles: agg: @@ -113,18 +117,34 @@ override_tp2_c64: roles: agg: gpus: 2 - # Run at least 4 decode passes after each prefill and batch the prefills - # that arrive meanwhile; at this load the batching also raises throughput. - env: - ATOM_PREFILL_DECODE_INTERVAL: '4' benchmark: env: CONC: '64' +override_tp2_c128: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '128' + +override_tp2_c256: + roles: + agg: + gpus: 2 + args: + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' + benchmark: + env: + CONC: '256' + override_tp4_c1: roles: agg: gpus: 4 + args: *c1_args + env: *c1_env benchmark: env: CONC: '1' @@ -137,14 +157,57 @@ override_tp4_c8: env: CONC: '8' -override_tp4_c32: +# DEP2: the DSv4.1 DEP2 cell of ROCm/ATOM#2784 (DP attention without TBO, which +# does not support SWA bounded replay, plus expert parallelism). Each AIPerf +# session stays on one DP rank and at least 10 decode passes run between +# prefills. + +override_tp2_c64_dep: roles: agg: - gpus: 4 - # Run at least 16 decode passes after each prefill and batch the prefills - # that arrive meanwhile, so new turns stall decode less often. - env: - ATOM_PREFILL_DECODE_INTERVAL: '16' + gpus: 2 + args: + enable-dp-attention: true + enable-expert-parallel: true + # ATOM_DP_LB_REQ_EQUIV weights in-flight requests under least_tokens. + dp-load-balance: least_tokens + env: &dep_env + GPU_MAX_HW_QUEUES: '5' + ATOM_DP_SESSION_AFFINITY: '1' + ATOM_DP_LB_REQ_EQUIV: '512' + ATOM_ENABLE_PREFILL_DELAYER: '1' + ATOM_PREFILL_DECODE_INTERVAL: '10' benchmark: env: - CONC: '32' + CONC: '64' + # Session affinity keys on the AIPerf session id. + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_tp2_c128_dep: + roles: + agg: + gpus: 2 + args: + enable-dp-attention: true + enable-expert-parallel: true + dp-load-balance: least_tokens + env: *dep_env + benchmark: + env: + CONC: '128' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_tp2_c256_dep: + roles: + agg: + gpus: 2 + args: + enable-dp-attention: true + enable-expert-parallel: true + dp-load-balance: least_tokens + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128,256]' + env: *dep_env + benchmark: + env: + CONC: '256' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index ed18bec2d6..657b66f271 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1628,10 +1628,12 @@ minimaxm31flash-fp4-mi355x-sglang-agentic-dspark: # Low-Latency: TP4, GPU-resident. - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 48, 64], srt-recipe: agentic.yaml } -# DeepSeek-V4.1-Flash on ATOM, following the upstream TP2 / TP4 AgentX recipe. +# DeepSeek-V4.1-Flash on ATOM with an MXFP8 KV cache, following the upstream +# TP2 / TP4 AgentX recipe, with DEP2 (DP attention, no TBO, plus expert +# parallelism) from concurrency 64. # Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. dsv41flash-fp4-mi355x-atom-agentic-dspark: - image: rocm/atom-dev:nightly_202610121516 + image: rocm/atom-dev:nightly_202610281457 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds @@ -1642,8 +1644,9 @@ dsv41flash-fp4-mi355x-atom-agentic-dspark: scenarios: agentic-coding: - search-space: - - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 6, 8, 16, 32, 64], srt-recipe: agentic.yaml } - - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 8, 32], srt-recipe: agentic.yaml } + - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256], srt-recipe: agentic.yaml } + - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [64, 128, 256], srt-recipe: agentic.yaml } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 8], srt-recipe: agentic.yaml } # The golden acceptance curve remains the provisional GLM-5.2-derived curve # documented in infx/golden_al_distribution/glm5.3_mtp.yaml. diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 920eaaaf82..8c4d66ca09 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11499,3 +11499,15 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4509 scenario-type: [agentic-coding] append-only: true + +- config-keys: + - dsv41flash-fp4-mi355x-atom-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202610121516 to nightly_202610281457 (digest sha256:96a9614276518c2619e88800dec5f276bc76a95691331b4abb2210184e1431af): nightly_202610271459 rebuilt on ATOM 646da00fe and AITER 11a9b4f5c. AITER includes ROCm/aiter#7085, which keeps the MX quant-sort and multi-phase MoE sort in bounds under num_local_tokens (plain TP2 hit an intermittent illegal memory access at concurrency 16 and above on nightly_202610261558), plus #7020 and #7068." + - "Switch the KV cache from BF16 to MXFP8 (kv_cache_dtype fp8, ROCm/ATOM#2725), as ATOM's own DSv4.1 AgentX CI runs it. The index cache stays FP4." + - "Run two shapes. Plain TP: TP2 at concurrency [1, 2, 4, 8, 16, 32, 64, 128, 256] and TP4 at [1, 8]. DEP2 (DP attention without TBO plus expert parallelism, the DEP2 cell of ROCm/ATOM#2784) at concurrency [64, 128, 256]: dp-load-balance least_tokens, ATOM_DP_SESSION_AFFINITY=1, ATOM_DP_LB_REQ_EQUIV=512, ATOM_ENABLE_PREFILL_DELAYER=1, ATOM_PREFILL_DECODE_INTERVAL=10, GPU_MAX_HW_QUEUES=5." + - "Drop the DP-attention points without expert parallelism (TP2 concurrency 32-256, TP4 concurrency 32); TP2 concurrency 32 and 64 move to plain TP." + - "Concurrency 1 (TP2 and TP4) follows the C1 recipe of ROCm/ATOM#2659: ATOM_ENABLE_PREFILL_DELAYER=1 with ATOM_PREFILL_DECODE_INTERVAL=1000000, max-num-seqs 2 and decode graphs at [1, 2]. Every other point runs ATOM's default max-num-seqs; the c256 points also capture 256." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3965