diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml index aa985aec5b..059d5af1b9 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml @@ -83,9 +83,12 @@ base: # One variant per point; each point runs in its own allocation. Admission is # 2x CONC on both roles. Decode captures every batch up to min(64, 2x CONC) on -# TP, or CONC/4 per rank under DP attention. DP attention adds TBO on prefill. +# TP, or CONC/4 per rank under DP attention. c64 adds TBO on prefill. +# Concurrency 1 runs 1P1D TP4/TP4 and concurrency 16 runs 1P1D TP4/TP8, both +# with six DSpark draft tokens; concurrency 16 adds the LMCache host KV tier on +# the TP4 prefill. -override_tp8_c1: +override_tp4_c1: frontend: args: prefill-policy: round_robin @@ -93,14 +96,18 @@ override_tp8_c1: atom-pd-rank-mapping-policy: none roles: prefill: + gpus: 4 args: + num-speculative-tokens: 6 max-num-seqs: 2 decode: + gpus: 4 args: + num-speculative-tokens: 6 max-num-seqs: 2 cudagraph-capture-sizes: '[1,2]' -override_tp8_c16: +override_tp4tp8_c16_lmcache: frontend: args: prefill-policy: round_robin @@ -108,10 +115,35 @@ override_tp8_c16: atom-pd-rank-mapping-policy: none roles: prefill: + gpus: 4 + env: + PYTHONHASHSEED: '0' + OFFLOAD_COPY_WORKERS: '1' + OFFLOAD_MIN_LOAD_TOKENS: '8192' + OFFLOAD_SLOT_STAGING_SLOTS: '4' args: + num-speculative-tokens: 6 max-num-seqs: 32 + extra-kv-connectors: + - kv_connector: lmcache_offload + kv_role: offload + offload_layout: hybrid + max_pending_saves: 8 + slot_sidecar_staging_slots: 4 + lmcache.local_cpu: true + # total-cpu-dram-gb 1499 (dram-utilization 0.5) over the four prefill ranks. + lmcache.max_local_cpu_size: 374 + lmcache.local_disk: null + lmcache.max_local_disk_size: 0 + lmcache.remote_url: null + lmcache.chunk_size: 256 + lmcache.cache_policy: LRU + lmcache.lookup_server_worker_ids: [] + lmcache.store_location: LocalCPUBackend + lmcache.retrieve_locations: [LocalCPUBackend] decode: args: + num-speculative-tokens: 6 max-num-seqs: 32 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' @@ -143,36 +175,39 @@ override_dpa8_c64: cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' enable-dp-attention: true -override_dpa8_c128: - frontend: - args: - dp-aware: true - prefill-policy: cache_aware - decode-policy: cache_aware - cache-threshold: 0.8 - balance-abs-threshold: 20 - balance-rel-threshold: 2.0 - eviction-interval: 300 - atom-pd-rank-mapping-policy: none - roles: - prefill: - env: - ATOM_ENABLE_PREFILL_DELAYER: '0' - GPU_MAX_HW_QUEUES: '5' +# Concurrency 128, 256 and 512 run on their own master-config key: prefill is +# DP attention with EP8, the Mega MoE backend and EPLB, without TBO, and +# offloads KV to one LMCache MP server on the prefill node (lazy 1000 GB L1 +# shared by the eight prefill ranks); decode is DP attention without offload. +# Both roles route dp_sticky. Prefill keeps only prompt-end state checkpoints +# and rebuilds any other hit by bounded state replay. max-num-seqs is per DP +# rank. +override_ep8_c128_mega_mp: + services: &mega_mp_services + - name: lmcache + type: lmcache-server + placement: + node: prefill args: - max-num-seqs: 256 - enable-dp-attention: true - enable-tbo: true - decode: - env: - ATOM_ENABLE_PREFILL_DELAYER: '0' - args: - max-num-seqs: 256 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' - enable-dp-attention: true - -override_dpa8_c256_lmcache: - frontend: + - --chunk-size + - '256' + - --null-block-id + - '-1' + - --separate-object-groups + - --supported-transfer-mode + - lmcache_driven + - --l1-size-gb + - '1000' + - --l1-use-lazy + - --l1-read-ttl-seconds + - '900' + - --eviction-policy + - LRU + - --eviction-trigger-watermark + - '0.98' + - --max-workers + - '4' + frontend: &mega_mp_frontend args: dp-aware: true prefill-policy: dp_sticky @@ -180,42 +215,88 @@ override_dpa8_c256_lmcache: atom-pd-rank-mapping-policy: none roles: prefill: - env: + env: &mega_mp_prefill_env ATOM_ENABLE_PREFILL_DELAYER: '0' GPU_MAX_HW_QUEUES: '5' PYTHONHASHSEED: '0' OFFLOAD_COPY_WORKERS: '1' OFFLOAD_MIN_LOAD_TOKENS: '8192' - OFFLOAD_SLOT_STAGING_SLOTS: '4' - args: - max-num-seqs: 512 + OFFLOAD_MIN_SAVE_TOKENS: '8192' + OFFLOAD_MAX_PENDING_SAVES: '8' + LMCACHE_CHUNK_SIZE: '256' + MORI_SHMEM_HEAP_SIZE: 16G + ATOM_MEGA_MASK_PAD_ROWS: '1' + ATOM_MEGA_DECODE_MTPR: '512' + AITER_MEGA_FIXED_SLOT_MAX_MTPR: '1023' + ATOM_EPLB_MAX_REBALANCES: '4' + ATOM_DSV4_STATE_REPLAY: '1' + ATOM_STATE_CHECKPOINT_DEMAND: '0' + args: &mega_mp_prefill_args + max-num-seqs: 128 + state-checkpoint-interval-tokens: -1 enable-dp-attention: true - enable-tbo: true + enable-expert-parallel: true + all2all-backend: high-throughput + moe-backend: mega + enable-eplb: true + eplb-config: '{"load_window_size":100,"rebalance_interval":200}' + # lmcache-server listens on 8750 on the prefill node. extra-kv-connectors: - - kv_connector: lmcache_offload + - kv_connector: lmcache_mp kv_role: offload - offload_layout: hybrid - max_pending_saves: 8 - slot_sidecar_staging_slots: 4 - lmcache.local_cpu: true - # total-cpu-dram-gb 1499 (dram-utilization 0.5) over the eight prefill ranks. - lmcache.max_local_cpu_size: 187 - lmcache.local_disk: null - lmcache.max_local_disk_size: 0 - lmcache.remote_url: null - lmcache.chunk_size: 256 - lmcache.cache_policy: LRU - lmcache.lookup_server_worker_ids: [] - lmcache.store_location: LocalCPUBackend - lmcache.retrieve_locations: [LocalCPUBackend] + kv_connector_extra_config: + lmcache.mp.host: tcp://127.0.0.1 + lmcache.mp.port: 8750 + lmcache.mp.tp_rank_collapse: true decode: - env: + env: &mega_mp_decode_env ATOM_ENABLE_PREFILL_DELAYER: '0' args: - max-num-seqs: 512 - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]' + max-num-seqs: 64 + state-checkpoint-interval-tokens: -1 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' enable-dp-attention: true benchmark: env: # dp_sticky keys sessions on the correlation id. AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_ep8_c256_mega_mp: + services: *mega_mp_services + frontend: *mega_mp_frontend + roles: + prefill: + env: *mega_mp_prefill_env + args: + <<: *mega_mp_prefill_args + max-num-seqs: 192 + decode: + env: *mega_mp_decode_env + args: + max-num-seqs: 128 + state-checkpoint-interval-tokens: -1 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]' + enable-dp-attention: true + benchmark: + env: + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' + +override_ep8_c512_mega_mp: + services: *mega_mp_services + frontend: *mega_mp_frontend + roles: + prefill: + env: *mega_mp_prefill_env + args: + <<: *mega_mp_prefill_args + max-num-seqs: 256 + decode: + env: *mega_mp_decode_env + args: + max-num-seqs: 192 + state-checkpoint-interval-tokens: -1 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128]' + enable-dp-attention: true + benchmark: + env: + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index ed18bec2d6..396d4d82ca 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1353,46 +1353,90 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: scenarios: agentic-coding: # srt-slurm recipe benchmarks/multi_node/srt-slurm-recipes/dsv4/atom/mi355x-fp4/agentx/disagg-lmcache-dspark.yaml - # (ATOM recipes/DeepSeek-V4-Agentic-PD-Max.md). Concurrency 1/16 run plain - # TP8; 64/128 add DP attention with prefill TBO and the cache-aware router. + # (ATOM recipes/DeepSeek-V4-Agentic-PD-Max.md). Concurrency 1 runs TP4/TP4; + # 64 adds DP attention with prefill TBO and the cache-aware router. + # 128-512 run as dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark. - dram-utilization: 0.80 search-space: - spec-decoding: "draft_model" conc-list: [ 1 ] kv-offloading: none - srt-recipe: disagg-lmcache-dspark.yaml:override_tp8_c1 + srt-recipe: disagg-lmcache-dspark.yaml:override_tp4_c1 prefill: num-worker: 1 - tp: 8 + tp: 4 ep: 1 dp-attn: false decode: num-worker: 1 - tp: 8 + tp: 4 ep: 1 dp-attn: false - spec-decoding: "draft_model" - conc-list: [ 16 ] + conc-list: [ 64 ] kv-offloading: none - srt-recipe: disagg-lmcache-dspark.yaml:override_tp8_c16 + srt-recipe: disagg-lmcache-dspark.yaml:override_dpa8_c64 prefill: num-worker: 1 tp: 8 ep: 1 + dp-attn: true + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + # ATOM's in-process LMCache CPU offload on the TP4 prefill (lmcache_offload) + # of concurrency 16; decode is plain Mooncake on TP8. dram-utilization 0.50 + # as measured in run 35825955863. + - dram-utilization: 0.50 + search-space: + - spec-decoding: "draft_model" + conc-list: [ 16 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache } + srt-recipe: disagg-lmcache-dspark.yaml:override_tp4tp8_c16_lmcache + prefill: + num-worker: 1 + tp: 4 + ep: 1 dp-attn: false decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false + +# Concurrency 128, 256 and 512 of the DeepSeek-V4-Pro-0813 MI355X ATOM 1P1D AgentX +# sweep, sharing disagg-lmcache-dspark.yaml: prefill DP attention with EP8, the +# Mega MoE backend and EPLB, offloading KV to an LMCache MP server on the prefill +# node (1000 GB lazy L1); decode DP attention without offload; dp_sticky routing. +# dram-utilization 0.34 records that 1000 GB. +dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark: + image: rocm/atom-dev:nightly_202610271520_meta + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: atom-disagg + router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: dsv4/atom/mi355x-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 0.34 + search-space: - spec-decoding: "draft_model" - conc-list: [ 64 ] - kv-offloading: none - srt-recipe: disagg-lmcache-dspark.yaml:override_dpa8_c64 + conc-list: [ 128 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-lmcache-dspark.yaml:override_ep8_c128_mega_mp prefill: num-worker: 1 tp: 8 - ep: 1 + ep: 8 dp-attn: true decode: num-worker: 1 @@ -1400,33 +1444,29 @@ dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark: ep: 1 dp-attn: true - spec-decoding: "draft_model" - conc-list: [ 128 ] - kv-offloading: none - srt-recipe: disagg-lmcache-dspark.yaml:override_dpa8_c128 + conc-list: [ 256 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-lmcache-dspark.yaml:override_ep8_c256_mega_mp prefill: num-worker: 1 tp: 8 - ep: 1 + ep: 8 dp-attn: true decode: num-worker: 1 tp: 8 ep: 1 dp-attn: true - # DP attention plus ATOM's in-process LMCache CPU offload on prefill - # (lmcache_offload); decode is plain Mooncake. dram-utilization 0.50 as - # measured in run 35825955863. - - dram-utilization: 0.50 - search-space: - spec-decoding: "draft_model" - conc-list: [ 256 ] + conc-list: [ 512 ] kv-offloading: dram - kv-offload-backend: { name: lmcache } - srt-recipe: disagg-lmcache-dspark.yaml:override_dpa8_c256_lmcache + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-lmcache-dspark.yaml:override_ep8_c512_mega_mp prefill: num-worker: 1 tp: 8 - ep: 1 + ep: 8 dp-attn: true decode: num-worker: 1 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 920eaaaf82..ce39eea0f2 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11499,3 +11499,15 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4509 scenario-type: [agentic-coding] append-only: true + +- config-keys: + - dsv4-fp4-mi355x-atom-disagg-agentic-lmcache-dspark + - dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark + description: + - "Concurrency 1: 1P1D TP4/TP4 (8 GPUs) instead of TP8/TP8, with six DSpark draft tokens on both roles." + - "Concurrency 16: 1P1D with a TP4 prefill and a TP8 decode (12 GPUs) instead of TP8/TP8, with six DSpark draft tokens; the TP4 prefill adds the lmcache_offload host KV tier (374 GB per rank, dram-utilization 0.50)." + - "Concurrency 128 and 256 move to the new key dsv4-fp4-mi355x-atom-disagg-agentic-mega-mp-dspark on rocm/atom-dev:nightly_202610271520_meta, which also adds concurrency 512. Prefill runs DP attention with EP8, the Mega MoE backend (high-throughput all2all) and EPLB (window 100, interval 200, at most 4 rebalances), without TBO, and offloads KV to an LMCache MP server on the prefill node (lmcache_mp connector, tp_rank_collapse, 1000 GB lazy L1, 900 s read TTL, LRU). Decode runs DP attention without offload. AToMesh routes dp_sticky on both roles." + - "On those points prefill keeps only prompt-end state checkpoints (state-checkpoint-interval-tokens -1, ATOM_STATE_CHECKPOINT_DEMAND=0) and rebuilds other hits by bounded state replay (ATOM_DSV4_STATE_REPLAY=1); max-num-seqs is sized per DP rank (prefill 128/192/256, decode 64/128/192 at concurrency 128/256/512); the LMCache MP server runs four workers." + - "gpu-memory-utilization (0.9) and the conc 64 point are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4527 + scenario-type: [agentic-coding]