From 5885a6d078f90c2451fdca26ad5c6419e7b36447 Mon Sep 17 00:00:00 2001 From: yhl-amd Date: Mon, 26 Oct 2026 11:31:12 +0800 Subject: [PATCH 1/5] dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: TP4 low-concurrency points - conc 1: 1P1D TP4/TP4 with six DSpark draft tokens. - conc 16: 1P1D TP4 prefill + TP8 decode with six DSpark draft tokens; the TP4 prefill adds the lmcache_offload host KV tier (dram-utilization 0.50). - gpu-memory-utilization, image and conc 64/128/256 unchanged. Co-Authored-By: Claude Opus 5.5 --- .../agentx/disagg-lmcache-dspark.yaml | 36 ++++++++++++++- inferencex-e2e/configs/amd-master.yaml | 46 ++++++++++--------- inferencex-e2e/perf-changelog.yaml | 9 ++++ 3 files changed, 67 insertions(+), 24 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml index aa985aec5b..05c64cff58 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml @@ -84,8 +84,11 @@ base: # One variant per point; each point runs in its own allocation. Admission is # 2x CONC on both roles. Decode captures every batch up to min(64, 2x CONC) on # TP, or CONC/4 per rank under DP attention. DP attention adds TBO on prefill. +# Concurrency 1 runs 1P1D TP4/TP4 and concurrency 16 runs 1P1D TP4/TP8, both +# with six DSpark draft tokens; concurrency 16 adds the LMCache host KV tier on +# the TP4 prefill. -override_tp8_c1: +override_tp4_c1: frontend: args: prefill-policy: round_robin @@ -93,14 +96,18 @@ override_tp8_c1: atom-pd-rank-mapping-policy: none roles: prefill: + gpus: 4 args: + num-speculative-tokens: 6 max-num-seqs: 2 decode: + gpus: 4 args: + num-speculative-tokens: 6 max-num-seqs: 2 cudagraph-capture-sizes: '[1,2]' -override_tp8_c16: +override_tp4tp8_c16_lmcache: frontend: args: prefill-policy: round_robin @@ -108,10 +115,35 @@ override_tp8_c16: atom-pd-rank-mapping-policy: none roles: prefill: + gpus: 4 + env: + PYTHONHASHSEED: '0' + OFFLOAD_COPY_WORKERS: '1' + OFFLOAD_MIN_LOAD_TOKENS: '8192' + OFFLOAD_SLOT_STAGING_SLOTS: '4' args: + num-speculative-tokens: 6 max-num-seqs: 32 + extra-kv-connectors: + - kv_connector: lmcache_offload + kv_role: offload + offload_layout: hybrid + max_pending_saves: 8 + slot_sidecar_staging_slots: 4 + lmcache.local_cpu: true + # total-cpu-dram-gb 1499 (dram-utilization 0.5) over the four prefill ranks. + lmcache.max_local_cpu_size: 374 + lmcache.local_disk: null + lmcache.max_local_disk_size: 0 + lmcache.remote_url: null + lmcache.chunk_size: 256 + lmcache.cache_policy: LRU + lmcache.lookup_server_worker_ids: [] + lmcache.store_location: LocalCPUBackend + lmcache.retrieve_locations: [LocalCPUBackend] decode: args: + num-speculative-tokens: 6 max-num-seqs: 32 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 6bba904af0..86ef6511f2 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1325,36 +1325,22 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: scenarios: agentic-coding: # srt-slurm recipe benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml - # (ATOM recipes/DeepSeek-V4-Agentic-PD-Max.md). Concurrency 1/16 run plain - # TP8; 64/128 add DP attention with prefill TBO and the cache-aware router. + # (ATOM recipes/DeepSeek-V4-Agentic-PD-Max.md). Concurrency 1 runs TP4/TP4; + # 64/128 add DP attention with prefill TBO and the cache-aware router. - dram-utilization: 0.80 search-space: - spec-decoding: "draft_model" conc-list: [ 1 ] kv-offloading: none - srt-recipe: disagg-lmcache-dspark.yaml:override_tp8_c1 + srt-recipe: disagg-lmcache-dspark.yaml:override_tp4_c1 prefill: num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "draft_model" - conc-list: [ 16 ] - kv-offloading: none - srt-recipe: disagg-lmcache-dspark.yaml:override_tp8_c16 - prefill: - num-worker: 1 - tp: 8 + tp: 4 ep: 1 dp-attn: false decode: num-worker: 1 - tp: 8 + tp: 4 ep: 1 dp-attn: false - spec-decoding: "draft_model" @@ -1385,11 +1371,27 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: tp: 8 ep: 1 dp-attn: true - # DP attention plus ATOM's in-process LMCache CPU offload on prefill - # (lmcache_offload); decode is plain Mooncake. dram-utilization 0.50 as - # measured in run 35825955863. + # ATOM's in-process LMCache CPU offload on prefill (lmcache_offload); decode + # is plain Mooncake. Concurrency 16 runs a TP4 prefill and a TP8 decode; + # concurrency 256 adds DP attention. dram-utilization 0.50 as measured in + # run 35825955863. - dram-utilization: 0.50 search-space: + - spec-decoding: "draft_model" + conc-list: [ 16 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache } + srt-recipe: disagg-lmcache-dspark.yaml:override_tp4tp8_c16_lmcache + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false - spec-decoding: "draft_model" conc-list: [ 256 ] kv-offloading: dram diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 518bc9b1ad..dd011a5213 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11210,3 +11210,12 @@ - "Add TP2 points with DRAM KV offload, and adjust the memory settings on the DRAM KV offload points." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX scenario-type: [agentic-coding] + +- config-keys: + - dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark + description: + - "Concurrency 1: 1P1D TP4/TP4 (8 GPUs) instead of TP8/TP8, with six DSpark draft tokens on both roles." + - "Concurrency 16: 1P1D with a TP4 prefill and a TP8 decode (12 GPUs) instead of TP8/TP8, with six DSpark draft tokens; the TP4 prefill adds the lmcache_offload host KV tier (374 GB per rank, dram-utilization 0.50)." + - "gpu-memory-utilization, image and the conc 64/128/256 points are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD + scenario-type: [agentic-coding] From 9bf567a8e650b7ce7446f4a0d973369ed91027a4 Mon Sep 17 00:00:00 2001 From: Yi Date: Mon, 26 Oct 2026 11:57:26 +0800 Subject: [PATCH 2/5] perf-changelog: link PR #4527 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index dd011a5213..7bb16db036 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11217,5 +11217,5 @@ - "Concurrency 1: 1P1D TP4/TP4 (8 GPUs) instead of TP8/TP8, with six DSpark draft tokens on both roles." - "Concurrency 16: 1P1D with a TP4 prefill and a TP8 decode (12 GPUs) instead of TP8/TP8, with six DSpark draft tokens; the TP4 prefill adds the lmcache_offload host KV tier (374 GB per rank, dram-utilization 0.50)." - "gpu-memory-utilization, image and the conc 64/128/256 points are unchanged." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4527 scenario-type: [agentic-coding] From 8a39ef3f697a308e7b0380ab2888042999bad4c1 Mon Sep 17 00:00:00 2001 From: Yi Date: Mon, 26 Oct 2026 13:38:13 +0800 Subject: [PATCH 3/5] dsv4 ATOM PD AgentX: EP8 Mega + LMCache MP for conc 128/256/512 - Move conc 128 and 256 to dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark on nightly_202610141644 and add conc 512. - Prefill: DP attention + EP8 + Mega MoE + EPLB, no TBO, KV offload to an LMCache MP server on the prefill node (1000 GB lazy L1). - Decode: DP attention without offload; dp_sticky routing on both roles. Co-Authored-By: Claude Opus 5.5 --- .../agentx/disagg-lmcache-dspark.yaml | 137 +++++++++++------- inferencex-e2e/configs/amd-master.yaml | 80 +++++++--- inferencex-e2e/perf-changelog.yaml | 4 +- 3 files changed, 150 insertions(+), 71 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml index 05c64cff58..06bb65b779 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml @@ -83,7 +83,7 @@ base: # One variant per point; each point runs in its own allocation. Admission is # 2x CONC on both roles. Decode captures every batch up to min(64, 2x CONC) on -# TP, or CONC/4 per rank under DP attention. DP attention adds TBO on prefill. +# TP, or CONC/4 per rank under DP attention. c64 adds TBO on prefill. # Concurrency 1 runs 1P1D TP4/TP4 and concurrency 16 runs 1P1D TP4/TP8, both # with six DSpark draft tokens; concurrency 16 adds the LMCache host KV tier on # the TP4 prefill. @@ -175,79 +175,118 @@ override_dpa8_c64: cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' enable-dp-attention: true -override_dpa8_c128: - frontend: +# Concurrency 128, 256 and 512 run on their own master-config key: prefill is +# DP attention with EP8, the Mega MoE backend and EPLB, without TBO, and +# offloads KV to one LMCache MP server on the prefill node (lazy 1000 GB L1 +# shared by the eight prefill ranks); decode is DP attention without offload. +# Both roles route dp_sticky. +override_ep8_c128_mega_mp: + services: &mega_mp_services + - name: lmcache + type: lmcache-server + placement: + node: prefill + args: + - --chunk-size + - '256' + - --null-block-id + - '-1' + - --separate-object-groups + - --supported-transfer-mode + - lmcache_driven + - --l1-size-gb + - '1000' + - --l1-use-lazy + - --l1-read-ttl-seconds + - '900' + - --eviction-policy + - LRU + - --eviction-trigger-watermark + - '0.98' + frontend: &mega_mp_frontend args: dp-aware: true - prefill-policy: cache_aware - decode-policy: cache_aware - cache-threshold: 0.8 - balance-abs-threshold: 20 - balance-rel-threshold: 2.0 - eviction-interval: 300 + prefill-policy: dp_sticky + decode-policy: dp_sticky atom-pd-rank-mapping-policy: none roles: prefill: - env: + env: &mega_mp_prefill_env ATOM_ENABLE_PREFILL_DELAYER: '0' GPU_MAX_HW_QUEUES: '5' - args: + PYTHONHASHSEED: '0' + OFFLOAD_COPY_WORKERS: '1' + OFFLOAD_MIN_LOAD_TOKENS: '8192' + OFFLOAD_MIN_SAVE_TOKENS: '8192' + OFFLOAD_MAX_PENDING_SAVES: '8' + LMCACHE_CHUNK_SIZE: '256' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '512' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '1023' + ATOM_EPLB_MAX_REBALANCES: '4' + args: &mega_mp_prefill_args max-num-seqs: 256 enable-dp-attention: true - enable-tbo: true + enable-expert-parallel: true + all2all-backend: high-throughput + moe-backend: mega + enable-eplb: true + eplb-config: '{"load_window_size":100,"rebalance_interval":200}' + # lmcache-server listens on 8750 on the prefill node. + extra-kv-connectors: + - kv_connector: lmcache_mp + kv_role: offload + kv_connector_extra_config: + lmcache.mp.host: tcp://127.0.0.1 + lmcache.mp.port: 8750 + lmcache.mp.tp_rank_collapse: true decode: - env: + env: &mega_mp_decode_env ATOM_ENABLE_PREFILL_DELAYER: '0' args: max-num-seqs: 256 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' enable-dp-attention: true + benchmark: + env: + # dp_sticky keys sessions on the correlation id. + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' -override_dpa8_c256_lmcache: - frontend: - args: - dp-aware: true - prefill-policy: dp_sticky - decode-policy: dp_sticky - atom-pd-rank-mapping-policy: none +override_ep8_c256_mega_mp: + services: *mega_mp_services + frontend: *mega_mp_frontend roles: prefill: - env: - ATOM_ENABLE_PREFILL_DELAYER: '0' - GPU_MAX_HW_QUEUES: '5' - PYTHONHASHSEED: '0' - OFFLOAD_COPY_WORKERS: '1' - OFFLOAD_MIN_LOAD_TOKENS: '8192' - OFFLOAD_SLOT_STAGING_SLOTS: '4' + env: *mega_mp_prefill_env args: + <<: *mega_mp_prefill_args max-num-seqs: 512 - enable-dp-attention: true - enable-tbo: true - extra-kv-connectors: - - kv_connector: lmcache_offload - kv_role: offload - offload_layout: hybrid - max_pending_saves: 8 - slot_sidecar_staging_slots: 4 - lmcache.local_cpu: true - # total-cpu-dram-gb 1499 (dram-utilization 0.5) over the eight prefill ranks. - lmcache.max_local_cpu_size: 187 - lmcache.local_disk: null - lmcache.max_local_disk_size: 0 - lmcache.remote_url: null - lmcache.chunk_size: 256 - lmcache.cache_policy: LRU - lmcache.lookup_server_worker_ids: [] - lmcache.store_location: LocalCPUBackend - lmcache.retrieve_locations: [LocalCPUBackend] decode: - env: - ATOM_ENABLE_PREFILL_DELAYER: '0' + env: *mega_mp_decode_env args: max-num-seqs: 512 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]' enable-dp-attention: true benchmark: env: - # dp_sticky keys sessions on the correlation id. + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_ep8_c512_mega_mp: + services: *mega_mp_services + frontend: *mega_mp_frontend + roles: + prefill: + env: *mega_mp_prefill_env + args: + <<: *mega_mp_prefill_args + max-num-seqs: 1024 + decode: + env: *mega_mp_decode_env + args: + max-num-seqs: 1024 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128]' + enable-dp-attention: true + benchmark: + env: AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 86ef6511f2..223c402c44 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1326,7 +1326,8 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: agentic-coding: # srt-slurm recipe benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml # (ATOM recipes/DeepSeek-V4-Agentic-PD-Max.md). Concurrency 1 runs TP4/TP4; - # 64/128 add DP attention with prefill TBO and the cache-aware router. + # 64 adds DP attention with prefill TBO and the cache-aware router. + # 128-512 run as dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark. - dram-utilization: 0.80 search-space: - spec-decoding: "draft_model" @@ -1357,24 +1358,9 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: tp: 8 ep: 1 dp-attn: true - - spec-decoding: "draft_model" - conc-list: [ 128 ] - kv-offloading: none - srt-recipe: disagg-lmcache-dspark.yaml:override_dpa8_c128 - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - # ATOM's in-process LMCache CPU offload on prefill (lmcache_offload); decode - # is plain Mooncake. Concurrency 16 runs a TP4 prefill and a TP8 decode; - # concurrency 256 adds DP attention. dram-utilization 0.50 as measured in - # run 35825955863. + # ATOM's in-process LMCache CPU offload on the TP4 prefill (lmcache_offload) + # of concurrency 16; decode is plain Mooncake on TP8. dram-utilization 0.50 + # as measured in run 35825955863. - dram-utilization: 0.50 search-space: - spec-decoding: "draft_model" @@ -1392,16 +1378,68 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: tp: 8 ep: 1 dp-attn: false + +# Concurrency 128, 256 and 512 of the DeepSeek-V4-Pro-0813 MI355X ATOM 1P1D AgentX +# sweep, sharing disagg-lmcache-dspark.yaml: prefill DP attention with EP8, the +# Mega MoE backend and EPLB, offloading KV to an LMCache MP server on the prefill +# node (1000 GB lazy L1); decode DP attention without offload; dp_sticky routing. +# dram-utilization 0.34 records that 1000 GB. +dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark: + image: rocm/atom-dev:nightly_202610141644 + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: atom-disagg + router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: dsv4/atom/mi355x-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 0.34 + search-space: + - spec-decoding: "draft_model" + conc-list: [ 128 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-lmcache-dspark.yaml:override_ep8_c128_mega_mp + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true - spec-decoding: "draft_model" conc-list: [ 256 ] kv-offloading: dram - kv-offload-backend: { name: lmcache } - srt-recipe: disagg-lmcache-dspark.yaml:override_dpa8_c256_lmcache + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-lmcache-dspark.yaml:override_ep8_c256_mega_mp prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + decode: num-worker: 1 tp: 8 ep: 1 dp-attn: true + - spec-decoding: "draft_model" + conc-list: [ 512 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-lmcache-dspark.yaml:override_ep8_c512_mega_mp + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true decode: num-worker: 1 tp: 8 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 7bb16db036..e062054f7c 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11213,9 +11213,11 @@ - config-keys: - dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark + - dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark description: - "Concurrency 1: 1P1D TP4/TP4 (8 GPUs) instead of TP8/TP8, with six DSpark draft tokens on both roles." - "Concurrency 16: 1P1D with a TP4 prefill and a TP8 decode (12 GPUs) instead of TP8/TP8, with six DSpark draft tokens; the TP4 prefill adds the lmcache_offload host KV tier (374 GB per rank, dram-utilization 0.50)." - - "gpu-memory-utilization, image and the conc 64/128/256 points are unchanged." + - "Concurrency 128 and 256 move to the new key dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark on rocm/atom-dev:nightly_202610141644, which also adds concurrency 512. Prefill runs DP attention with EP8, the Mega MoE backend (high-throughput all2all) and EPLB (window 100, interval 200, at most 4 rebalances), without TBO, and offloads KV to an LMCache MP server on the prefill node (lmcache_mp connector, tp_rank_collapse, 1000 GB lazy L1, 900 s read TTL, LRU). Decode runs DP attention without offload. AToMesh routes dp_sticky on both roles." + - "gpu-memory-utilization (0.9) and the conc 64 point are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4527 scenario-type: [agentic-coding] From f9b53bed9e9737997a4da7931fab1f33a4e7835f Mon Sep 17 00:00:00 2001 From: Yi Date: Wed, 28 Oct 2026 00:21:24 +0800 Subject: [PATCH 4/5] dsv4 ATOM PD AgentX conc 128-512: bounded state replay, per-rank max-num-seqs, meta image - Prefill keeps only prompt-end state checkpoints (interval -1, demand off) and rebuilds other hits by bounded state replay. - max-num-seqs sized per DP rank on both roles. - rocm/atom-dev:nightly_202610271520_meta for the mega-mp key. Co-Authored-By: Claude Opus 5.5 --- .../agentx/disagg-lmcache-dspark.yaml | 22 +++++++++++++------ inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 3 ++- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml index 06bb65b779..7915fa2e34 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml @@ -179,7 +179,9 @@ override_dpa8_c64: # DP attention with EP8, the Mega MoE backend and EPLB, without TBO, and # offloads KV to one LMCache MP server on the prefill node (lazy 1000 GB L1 # shared by the eight prefill ranks); decode is DP attention without offload. -# Both roles route dp_sticky. +# Both roles route dp_sticky. Prefill keeps only prompt-end state checkpoints +# and rebuilds any other hit by bounded state replay. max-num-seqs is per DP +# rank. override_ep8_c128_mega_mp: services: &mega_mp_services - name: lmcache @@ -225,8 +227,11 @@ override_ep8_c128_mega_mp: ATOM_MEGA_DECODE_MTPR: '512' AITER_MEGA_FIXED_SLOT_MAX_MTPR: '1023' ATOM_EPLB_MAX_REBALANCES: '4' + ATOM_DSV4_STATE_REPLAY: '1' + ATOM_STATE_CHECKPOINT_DEMAND: '0' args: &mega_mp_prefill_args - max-num-seqs: 256 + max-num-seqs: 128 + state-checkpoint-interval-tokens: -1 enable-dp-attention: true enable-expert-parallel: true all2all-backend: high-throughput @@ -245,7 +250,8 @@ override_ep8_c128_mega_mp: env: &mega_mp_decode_env ATOM_ENABLE_PREFILL_DELAYER: '0' args: - max-num-seqs: 256 + max-num-seqs: 64 + state-checkpoint-interval-tokens: -1 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' enable-dp-attention: true benchmark: @@ -261,11 +267,12 @@ override_ep8_c256_mega_mp: env: *mega_mp_prefill_env args: <<: *mega_mp_prefill_args - max-num-seqs: 512 + max-num-seqs: 192 decode: env: *mega_mp_decode_env args: - max-num-seqs: 512 + max-num-seqs: 128 + state-checkpoint-interval-tokens: -1 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]' enable-dp-attention: true benchmark: @@ -280,11 +287,12 @@ override_ep8_c512_mega_mp: env: *mega_mp_prefill_env args: <<: *mega_mp_prefill_args - max-num-seqs: 1024 + max-num-seqs: 256 decode: env: *mega_mp_decode_env args: - max-num-seqs: 1024 + max-num-seqs: 192 + state-checkpoint-interval-tokens: -1 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128]' enable-dp-attention: true benchmark: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 08944d2a9c..c4a64fb261 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1413,7 +1413,7 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: # node (1000 GB lazy L1); decode DP attention without offload; dp_sticky routing. # dram-utilization 0.34 records that 1000 GB. dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark: - image: rocm/atom-dev:nightly_202610141644 + image: rocm/atom-dev:nightly_202610271520_meta model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 6e693c1698..358dbf021c 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11420,7 +11420,8 @@ description: - "Concurrency 1: 1P1D TP4/TP4 (8 GPUs) instead of TP8/TP8, with six DSpark draft tokens on both roles." - "Concurrency 16: 1P1D with a TP4 prefill and a TP8 decode (12 GPUs) instead of TP8/TP8, with six DSpark draft tokens; the TP4 prefill adds the lmcache_offload host KV tier (374 GB per rank, dram-utilization 0.50)." - - "Concurrency 128 and 256 move to the new key dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark on rocm/atom-dev:nightly_202610141644, which also adds concurrency 512. Prefill runs DP attention with EP8, the Mega MoE backend (high-throughput all2all) and EPLB (window 100, interval 200, at most 4 rebalances), without TBO, and offloads KV to an LMCache MP server on the prefill node (lmcache_mp connector, tp_rank_collapse, 1000 GB lazy L1, 900 s read TTL, LRU). Decode runs DP attention without offload. AToMesh routes dp_sticky on both roles." + - "Concurrency 128 and 256 move to the new key dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark on rocm/atom-dev:nightly_202610271520_meta, which also adds concurrency 512. Prefill runs DP attention with EP8, the Mega MoE backend (high-throughput all2all) and EPLB (window 100, interval 200, at most 4 rebalances), without TBO, and offloads KV to an LMCache MP server on the prefill node (lmcache_mp connector, tp_rank_collapse, 1000 GB lazy L1, 900 s read TTL, LRU). Decode runs DP attention without offload. AToMesh routes dp_sticky on both roles." + - "On those points prefill keeps only prompt-end state checkpoints (state-checkpoint-interval-tokens -1, ATOM_STATE_CHECKPOINT_DEMAND=0) and rebuilds other hits by bounded state replay (ATOM_DSV4_STATE_REPLAY=1); max-num-seqs is sized per DP rank (prefill 128/192/256, decode 64/128/192 at concurrency 128/256/512)." - "gpu-memory-utilization (0.9) and the conc 64 point are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4527 scenario-type: [agentic-coding] From ec86d67b0ff8c959af7be45d4bdf86bc18b14dec Mon Sep 17 00:00:00 2001 From: Yi Date: Wed, 28 Oct 2026 00:27:15 +0800 Subject: [PATCH 5/5] dsv4 ATOM PD AgentX: LMCache MP server with four workers Co-Authored-By: Claude Opus 5.5 --- .../dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml | 2 ++ inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml index 7915fa2e34..059d5af1b9 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml @@ -205,6 +205,8 @@ override_ep8_c128_mega_mp: - LRU - --eviction-trigger-watermark - '0.98' + - --max-workers + - '4' frontend: &mega_mp_frontend args: dp-aware: true diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 358dbf021c..cc056aac24 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11421,7 +11421,7 @@ - "Concurrency 1: 1P1D TP4/TP4 (8 GPUs) instead of TP8/TP8, with six DSpark draft tokens on both roles." - "Concurrency 16: 1P1D with a TP4 prefill and a TP8 decode (12 GPUs) instead of TP8/TP8, with six DSpark draft tokens; the TP4 prefill adds the lmcache_offload host KV tier (374 GB per rank, dram-utilization 0.50)." - "Concurrency 128 and 256 move to the new key dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark on rocm/atom-dev:nightly_202610271520_meta, which also adds concurrency 512. Prefill runs DP attention with EP8, the Mega MoE backend (high-throughput all2all) and EPLB (window 100, interval 200, at most 4 rebalances), without TBO, and offloads KV to an LMCache MP server on the prefill node (lmcache_mp connector, tp_rank_collapse, 1000 GB lazy L1, 900 s read TTL, LRU). Decode runs DP attention without offload. AToMesh routes dp_sticky on both roles." - - "On those points prefill keeps only prompt-end state checkpoints (state-checkpoint-interval-tokens -1, ATOM_STATE_CHECKPOINT_DEMAND=0) and rebuilds other hits by bounded state replay (ATOM_DSV4_STATE_REPLAY=1); max-num-seqs is sized per DP rank (prefill 128/192/256, decode 64/128/192 at concurrency 128/256/512)." + - "On those points prefill keeps only prompt-end state checkpoints (state-checkpoint-interval-tokens -1, ATOM_STATE_CHECKPOINT_DEMAND=0) and rebuilds other hits by bounded state replay (ATOM_DSV4_STATE_REPLAY=1); max-num-seqs is sized per DP rank (prefill 128/192/256, decode 64/128/192 at concurrency 128/256/512); the LMCache MP server runs four workers." - "gpu-memory-utilization (0.9) and the conc 64 point are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4527 scenario-type: [agentic-coding]