dsv4 ATOM PD AgentX on MI355X: TP4 low-concurrency points, EP8 Mega + LMCache MP at 128-512 / 低并发改 TP4，高并发改 EP8 Mega + LMCache MP

inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml

@@ -83,35 +83,67 @@ base:
 
 # One variant per point; each point runs in its own allocation. Admission is
 # 2x CONC on both roles. Decode captures every batch up to min(64, 2x CONC) on
-# TP, or CONC/4 per rank under DP attention. DP attention adds TBO on prefill.
+# TP, or CONC/4 per rank under DP attention. c64 adds TBO on prefill.
+# Concurrency 1 runs 1P1D TP4/TP4 and concurrency 16 runs 1P1D TP4/TP8, both
+# with six DSpark draft tokens; concurrency 16 adds the LMCache host KV tier on
+# the TP4 prefill.
 
-override_tp8_c1:
+override_tp4_c1:
   frontend:
     args:
       prefill-policy: round_robin
       decode-policy: round_robin
       atom-pd-rank-mapping-policy: none
   roles:
     prefill:
+      gpus: 4
       args:
+        num-speculative-tokens: 6
         max-num-seqs: 2
     decode:
+      gpus: 4
       args:
+        num-speculative-tokens: 6
         max-num-seqs: 2
         cudagraph-capture-sizes: '[1,2]'
 
-override_tp8_c16:
+override_tp4tp8_c16_lmcache:
   frontend:
     args:
       prefill-policy: round_robin
       decode-policy: round_robin
       atom-pd-rank-mapping-policy: none
   roles:
     prefill:
+      gpus: 4
+      env:
+        PYTHONHASHSEED: '0'
+        OFFLOAD_COPY_WORKERS: '1'
+        OFFLOAD_MIN_LOAD_TOKENS: '8192'
+        OFFLOAD_SLOT_STAGING_SLOTS: '4'
       args:
+        num-speculative-tokens: 6
         max-num-seqs: 32
+        extra-kv-connectors:
+          - kv_connector: lmcache_offload
+            kv_role: offload
+            offload_layout: hybrid
+            max_pending_saves: 8
+            slot_sidecar_staging_slots: 4
+            lmcache.local_cpu: true
+            # total-cpu-dram-gb 1499 (dram-utilization 0.5) over the four prefill ranks.
+            lmcache.max_local_cpu_size: 374
+            lmcache.local_disk: null
+            lmcache.max_local_disk_size: 0
+            lmcache.remote_url: null
+            lmcache.chunk_size: 256
+            lmcache.cache_policy: LRU
+            lmcache.lookup_server_worker_ids: []
+            lmcache.store_location: LocalCPUBackend
+            lmcache.retrieve_locations: [LocalCPUBackend]
     decode:
       args:
+        num-speculative-tokens: 6
         max-num-seqs: 32
         cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]'
 
@@ -143,79 +175,128 @@ override_dpa8_c64:
         cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]'
         enable-dp-attention: true
 
-override_dpa8_c128:
-  frontend:
-    args:
-      dp-aware: true
-      prefill-policy: cache_aware
-      decode-policy: cache_aware
-      cache-threshold: 0.8
-      balance-abs-threshold: 20
-      balance-rel-threshold: 2.0
-      eviction-interval: 300
-      atom-pd-rank-mapping-policy: none
-  roles:
-    prefill:
-      env:
-        ATOM_ENABLE_PREFILL_DELAYER: '0'
-        GPU_MAX_HW_QUEUES: '5'
+# Concurrency 128, 256 and 512 run on their own master-config key: prefill is
+# DP attention with EP8, the Mega MoE backend and EPLB, without TBO, and
+# offloads KV to one LMCache MP server on the prefill node (lazy 1000 GB L1
+# shared by the eight prefill ranks); decode is DP attention without offload.
+# Both roles route dp_sticky. Prefill keeps only prompt-end state checkpoints
+# and rebuilds any other hit by bounded state replay. max-num-seqs is per DP
+# rank.
+override_ep8_c128_mega_mp:
+  services: &mega_mp_services
+    - name: lmcache
+      type: lmcache-server
+      placement:
+        node: prefill
       args:
-        max-num-seqs: 256
-        enable-dp-attention: true
-        enable-tbo: true
-    decode:
-      env:
-        ATOM_ENABLE_PREFILL_DELAYER: '0'
-      args:
-        max-num-seqs: 256
-        cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]'
-        enable-dp-attention: true
-
-override_dpa8_c256_lmcache:
-  frontend:
+        - --chunk-size
+        - '256'
+        - --null-block-id
+        - '-1'
+        - --separate-object-groups
+        - --supported-transfer-mode
+        - lmcache_driven
+        - --l1-size-gb
+        - '1000'
+        - --l1-use-lazy
+        - --l1-read-ttl-seconds
+        - '900'
+        - --eviction-policy
+        - LRU
+        - --eviction-trigger-watermark
+        - '0.98'
+        - --max-workers
+        - '4'
+  frontend: &mega_mp_frontend
     args:
       dp-aware: true
       prefill-policy: dp_sticky
       decode-policy: dp_sticky
       atom-pd-rank-mapping-policy: none
   roles:
     prefill:
-      env:
+      env: &mega_mp_prefill_env
         ATOM_ENABLE_PREFILL_DELAYER: '0'
         GPU_MAX_HW_QUEUES: '5'
         PYTHONHASHSEED: '0'
         OFFLOAD_COPY_WORKERS: '1'
         OFFLOAD_MIN_LOAD_TOKENS: '8192'
-        OFFLOAD_SLOT_STAGING_SLOTS: '4'
-      args:
-        max-num-seqs: 512
+        OFFLOAD_MIN_SAVE_TOKENS: '8192'
+        OFFLOAD_MAX_PENDING_SAVES: '8'
+        LMCACHE_CHUNK_SIZE: '256'
+        MORI_SHMEM_HEAP_SIZE: 16G
+        ATOM_MEGA_MASK_PAD_ROWS: '1'
+        ATOM_MEGA_DECODE_MTPR: '512'
+        AITER_MEGA_FIXED_SLOT_MAX_MTPR: '1023'
+        ATOM_EPLB_MAX_REBALANCES: '4'
+        ATOM_DSV4_STATE_REPLAY: '1'
+        ATOM_STATE_CHECKPOINT_DEMAND: '0'
+      args: &mega_mp_prefill_args
+        max-num-seqs: 128
+        state-checkpoint-interval-tokens: -1
         enable-dp-attention: true
-        enable-tbo: true
+        enable-expert-parallel: true
+        all2all-backend: high-throughput
+        moe-backend: mega
+        enable-eplb: true
+        eplb-config: '{"load_window_size":100,"rebalance_interval":200}'
+        # lmcache-server listens on 8750 on the prefill node.
         extra-kv-connectors:
-          - kv_connector: lmcache_offload
+          - kv_connector: lmcache_mp
             kv_role: offload
-            offload_layout: hybrid
-            max_pending_saves: 8
-            slot_sidecar_staging_slots: 4
-            lmcache.local_cpu: true
-            # total-cpu-dram-gb 1499 (dram-utilization 0.5) over the eight prefill ranks.
-            lmcache.max_local_cpu_size: 187
-            lmcache.local_disk: null
-            lmcache.max_local_disk_size: 0
-            lmcache.remote_url: null
-            lmcache.chunk_size: 256
-            lmcache.cache_policy: LRU
-            lmcache.lookup_server_worker_ids: []
-            lmcache.store_location: LocalCPUBackend
-            lmcache.retrieve_locations: [LocalCPUBackend]
+            kv_connector_extra_config:
+              lmcache.mp.host: tcp://127.0.0.1
+              lmcache.mp.port: 8750
+              lmcache.mp.tp_rank_collapse: true
     decode:
-      env:
+      env: &mega_mp_decode_env
         ATOM_ENABLE_PREFILL_DELAYER: '0'
       args:
-        max-num-seqs: 512
-        cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]'
+        max-num-seqs: 64
+        state-checkpoint-interval-tokens: -1
+        cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]'
         enable-dp-attention: true
   benchmark:
     env:
       # dp_sticky keys sessions on the correlation id.
       AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'
+
+override_ep8_c256_mega_mp:
+  services: *mega_mp_services
+  frontend: *mega_mp_frontend
+  roles:
+    prefill:
+      env: *mega_mp_prefill_env
+      args:
+        <<: *mega_mp_prefill_args
+        max-num-seqs: 192
+    decode:
+      env: *mega_mp_decode_env
+      args:
+        max-num-seqs: 128
+        state-checkpoint-interval-tokens: -1
+        cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]'
+        enable-dp-attention: true
+  benchmark:
+    env:
+      AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'
+
+override_ep8_c512_mega_mp:
+  services: *mega_mp_services
+  frontend: *mega_mp_frontend
+  roles:
+    prefill:
+      env: *mega_mp_prefill_env
+      args:
+        <<: *mega_mp_prefill_args
+        max-num-seqs: 256
+    decode:
+      env: *mega_mp_decode_env
+      args:
+        max-num-seqs: 192
+        state-checkpoint-interval-tokens: -1
+        cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128]'
+        enable-dp-attention: true
+  benchmark:
+    env:
+      AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'

inferencex-e2e/configs/amd-master.yaml

@@ -1353,80 +1353,120 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark:
   scenarios:
     agentic-coding:
     # srt-slurm recipe benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml
-    # (ATOM recipes/DeepSeek-V4-Agentic-PD-Max.md). Concurrency 1/16 run plain
-    # TP8; 64/128 add DP attention with prefill TBO and the cache-aware router.
+    # (ATOM recipes/DeepSeek-V4-Agentic-PD-Max.md). Concurrency 1 runs TP4/TP4;
+    # 64 adds DP attention with prefill TBO and the cache-aware router.
+    # 128-512 run as dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark.
     - dram-utilization: 0.80
       search-space:
       - spec-decoding: "draft_model"
         conc-list: [ 1 ]
         kv-offloading: none
-        srt-recipe: disagg-lmcache-dspark.yaml:override_tp8_c1
+        srt-recipe: disagg-lmcache-dspark.yaml:override_tp4_c1
         prefill:
           num-worker: 1
-          tp: 8
+          tp: 4
           ep: 1
           dp-attn: false
         decode:
           num-worker: 1
-          tp: 8
+          tp: 4
           ep: 1
           dp-attn: false
       - spec-decoding: "draft_model"
-        conc-list: [ 16 ]
+        conc-list: [ 64 ]
         kv-offloading: none
-        srt-recipe: disagg-lmcache-dspark.yaml:override_tp8_c16
+        srt-recipe: disagg-lmcache-dspark.yaml:override_dpa8_c64
         prefill:
           num-worker: 1
           tp: 8
           ep: 1
+          dp-attn: true
+        decode:
+          num-worker: 1
+          tp: 8
+          ep: 1
+          dp-attn: true
+    # ATOM's in-process LMCache CPU offload on the TP4 prefill (lmcache_offload)
+    # of concurrency 16; decode is plain Mooncake on TP8. dram-utilization 0.50
+    # as measured in run 35825955863.
+    - dram-utilization: 0.50
+      search-space:
+      - spec-decoding: "draft_model"
+        conc-list: [ 16 ]
+        kv-offloading: dram
+        kv-offload-backend: { name: lmcache }
+        srt-recipe: disagg-lmcache-dspark.yaml:override_tp4tp8_c16_lmcache
+        prefill:
+          num-worker: 1
+          tp: 4
+          ep: 1
           dp-attn: false
         decode:
           num-worker: 1
           tp: 8
           ep: 1
           dp-attn: false
+
+# Concurrency 128, 256 and 512 of the DeepSeek-V4-Pro-0813 MI355X ATOM 1P1D AgentX
+# sweep, sharing disagg-lmcache-dspark.yaml: prefill DP attention with EP8, the
+# Mega MoE backend and EPLB, offloading KV to an LMCache MP server on the prefill
+# node (1000 GB lazy L1); decode DP attention without offload; dp_sticky routing.
+# dram-utilization 0.34 records that 1000 GB.
+dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark:
+  image: rocm/atom-dev:nightly_202610271520_meta
+  model: deepseek-ai/DeepSeek-V4-Pro-0813
+  model-prefix: dsv4
+  runner: cluster:mi355x-amds
+  precision: fp4
+  framework: atom-disagg
+  router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" }
+  kv-p2p-transfer: mooncake
+  multinode: true
+  disagg: true
+  srt-recipe-dir: dsv4/atom/mi355x-fp4/agentx
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.34
+      search-space:
       - spec-decoding: "draft_model"
-        conc-list: [ 64 ]
-        kv-offloading: none
-        srt-recipe: disagg-lmcache-dspark.yaml:override_dpa8_c64
+        conc-list: [ 128 ]
+        kv-offloading: dram
+        kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }
+        srt-recipe: disagg-lmcache-dspark.yaml:override_ep8_c128_mega_mp
         prefill:
           num-worker: 1
           tp: 8
-          ep: 1
+          ep: 8
           dp-attn: true
         decode:
           num-worker: 1
           tp: 8
           ep: 1
           dp-attn: true
       - spec-decoding: "draft_model"
-        conc-list: [ 128 ]
-        kv-offloading: none
-        srt-recipe: disagg-lmcache-dspark.yaml:override_dpa8_c128
+        conc-list: [ 256 ]
+        kv-offloading: dram
+        kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }
+        srt-recipe: disagg-lmcache-dspark.yaml:override_ep8_c256_mega_mp
         prefill:
           num-worker: 1
           tp: 8
-          ep: 1
+          ep: 8
           dp-attn: true
         decode:
           num-worker: 1
           tp: 8
           ep: 1
           dp-attn: true
-    # DP attention plus ATOM's in-process LMCache CPU offload on prefill
-    # (lmcache_offload); decode is plain Mooncake. dram-utilization 0.50 as
-    # measured in run 35825955863.
-    - dram-utilization: 0.50
-      search-space:
       - spec-decoding: "draft_model"
-        conc-list: [ 256 ]
+        conc-list: [ 512 ]
         kv-offloading: dram
-        kv-offload-backend: { name: lmcache }
-        srt-recipe: disagg-lmcache-dspark.yaml:override_dpa8_c256_lmcache
+        kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }
+        srt-recipe: disagg-lmcache-dspark.yaml:override_ep8_c512_mega_mp
         prefill:
           num-worker: 1
           tp: 8
-          ep: 1
+          ep: 8
           dp-attn: true
         decode:
           num-worker: 1