Refresh Kimi-K3 PD on ROCm nightly / 更新 Kimi-K3 PD nightly 配置

inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4/agentx/disagg-variants.yaml

@@ -65,14 +65,14 @@ base:
         enable-prefix-caching: true
         prefix-match-unit: 128
         kv-cache-dtype: fp8
-        mamba-ssm-cache-dtype: float32
+        mamba-ssm-cache-dtype: bfloat16
         block-size: 128
         attention-backend: ROCM_AITER_MLA
         attention-config: '{"mla_prefill_backend":"ROCM_AITER_FA","use_prefill_query_quantization":true}'
         # The native srt driver supplies the measured AL for the selected DSpark
         # draft length in throughput runs and keeps real block rejection for eval.
         speculative-config: '{"model":"/models/Inferact-Kimi-K3-DSpark","method":"dspark","num_speculative_tokens":4,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
-        kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"MoRIIOConnector","kv_role":"kv_producer","kv_load_failure_policy":"fail","kv_connector_extra_config":{"backend":"rdma"}},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1799000000000,"lazy_offload":false}}]}}'
+        kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"MoRIIOConnector","kv_role":"kv_producer","kv_load_failure_policy":"fail","kv_connector_extra_config":{"backend":"rdma"}},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":2399000000000,"lazy_offload":false}}]}}'
       env: &common_env
         NCCL_IB_HCA: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7
         NCCL_SOCKET_IFNAME: eno0
@@ -118,22 +118,29 @@ base:
     placement:
       node: head
 
-override_1p3d_c12:
+override_1p1d_c8:
   roles:
     prefill:
       args:
-        max-num-seqs: 24
+        decode-context-parallel-size: 8
+        mamba-ssm-cache-dtype: float32
+        max-num-seqs: 1
         max-num-batched-tokens: 8192
         compilation-config: '{"mode":3,"cudagraph_mode":"NONE","max_cudagraph_capture_size":0,"custom_ops":["+fused_rms_norm_gated","+quant_fp8","+grouped_topk","+sparse_attn_indexer","none"]}'
-      env:
-        HSA_NO_SCRATCH_RECLAIM: '0'
+        speculative-config: &spec_k7 '{"model":"/models/Inferact-Kimi-K3-DSpark","method":"dspark","num_speculative_tokens":7,"attention_backend":"ROCM_AITER_MLA","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
+        kv-transfer-config: '{"kv_connector":"MoRIIOConnector","kv_role":"kv_producer","kv_load_failure_policy":"fail","kv_connector_extra_config":{"backend":"rdma"}}'
     decode:
-      nodes: 3
-      workers: 3
       args:
-        max-num-seqs: 8
+        decode-context-parallel-size: 8
+        mamba-ssm-cache-dtype: float32
+        max-num-seqs: 1
         max-num-batched-tokens: 512
-        compilation-config: '{"mode":3,"cudagraph_mode":"FULL_DECODE_ONLY","max_cudagraph_capture_size":40,"cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40],"custom_ops":["+fused_rms_norm_gated","+quant_fp8","+grouped_topk","+sparse_attn_indexer","none"]}'
+        additional-config: '{"decode_backend":"latency_specialized"}'
+        speculative-config: *spec_k7
+        # K7 verifies eight tokens for the single active request.
+        compilation-config: '{"mode":3,"cudagraph_mode":"FULL_DECODE_ONLY","max_cudagraph_capture_size":8,"cudagraph_capture_sizes":[1,8],"custom_ops":["+fused_rms_norm_gated","+quant_fp8","+grouped_topk","+sparse_attn_indexer","none"]}'
+      env:
+        VLLM_USE_V2_MODEL_RUNNER: '1'
 
 override_1p2d_c24: &tp8_c24
   roles: &tp8_c24_roles
@@ -167,8 +174,8 @@ override_1p3d_c24:
 
 override_1p1d_c48: &tp8_c48
   roles: &tp8_c48_roles
-    prefill: &tp8_c48_prefill
-      args: &tp8_c48_prefill_args
+    prefill:
+      args:
         decode-context-parallel-size: 8
         max-num-seqs: 96
         max-num-batched-tokens: 8192
@@ -183,22 +190,6 @@ override_1p1d_c48: &tp8_c48
         max-num-batched-tokens: 512
         compilation-config: '{"mode":3,"cudagraph_mode":"FULL_DECODE_ONLY","max_cudagraph_capture_size":240,"cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,130,140,150,160,170,180,190,200,210,220,230,240],"custom_ops":["+fused_rms_norm_gated","+quant_fp8","+grouped_topk","+sparse_attn_indexer","none"]}'
 
-override_1p1d_c56:
-  <<: *tp8_c48
-  roles:
-    <<: *tp8_c48_roles
-    prefill:
-      <<: *tp8_c48_prefill
-      args:
-        <<: *tp8_c48_prefill_args
-        max-num-seqs: 112
-    decode:
-      <<: *tp8_c48_decode
-      args:
-        <<: *tp8_c48_decode_args
-        max-num-seqs: 112
-        compilation-config: '{"mode":3,"cudagraph_mode":"FULL_DECODE_ONLY","max_cudagraph_capture_size":280,"cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,130,140,150,160,170,180,190,200,210,220,230,240,250,260,270,280],"custom_ops":["+fused_rms_norm_gated","+quant_fp8","+grouped_topk","+sparse_attn_indexer","none"]}'
-
 override_1p2d_c48:
   <<: *tp8_c48
   roles:

inferencex-e2e/configs/amd-master.yaml

@@ -1715,7 +1715,7 @@ musespark-fp4-mi355x-sglang-agentic-dflash:
 
 # Native Kimi-K3 PD with TP8 workers and MoRIIO READ.
 kimik3-fp4-mi355x-vllm-disagg-agentic:
-  image: vllm/vllm-openai-rocm:nightly-81198e97ba7eee2a22540caaa756b7fdddcb4d93@sha256:a401e4f46872dcde77e07ae3d7fa2d3a71b7f2773714316116532e2b39deb179
+  image: vllm/vllm-openai-rocm:nightly-c50aca14416ef29f6ed0ed256302f30646726dcd@sha256:ad061ff860a530c324b0ebc5a33c18cd9da170eee2f55704bd52621beedc0b31
   model: moonshotai/Kimi-K3
   model-prefix: kimik3
   runner: cluster:mi355x-amds
@@ -1728,22 +1728,8 @@ kimik3-fp4-mi355x-vllm-disagg-agentic:
   srt-recipe-dir: kimik3/vllm/mi355x-fp4/agentx
   scenarios:
     agentic-coding:
-    - dram-utilization: 0.6
+    - dram-utilization: 0.8
       search-space:
-      - spec-decoding: mtp
-        conc-list: [12]
-        kv-offloading: dram
-        kv-offload-backend: { name: vllm-simple }
-        srt-recipe: disagg-variants.yaml:override_1p3d_c12
-        prefill:
-          num-worker: 1
-          tp: 8
-          dcp-size: 1
-          ep: 1
-          dp-attn: false
-          additional-settings:
-          - TOTAL_CPU_DRAM_GB=1799
-        decode: { num-worker: 3, tp: 8, dcp-size: 1, ep: 1, dp-attn: false }
       - spec-decoding: mtp
         conc-list: [24]
         kv-offloading: dram
@@ -1756,7 +1742,7 @@ kimik3-fp4-mi355x-vllm-disagg-agentic:
           ep: 1
           dp-attn: false
           additional-settings:
-          - TOTAL_CPU_DRAM_GB=1799
+          - TOTAL_CPU_DRAM_GB=2399
         decode: { num-worker: 3, tp: 8, dcp-size: 8, ep: 1, dp-attn: false }
       - spec-decoding: mtp
         conc-list: [24]
@@ -1770,7 +1756,7 @@ kimik3-fp4-mi355x-vllm-disagg-agentic:
           ep: 1
           dp-attn: false
           additional-settings:
-          - TOTAL_CPU_DRAM_GB=1799
+          - TOTAL_CPU_DRAM_GB=2399
         decode: { num-worker: 2, tp: 8, dcp-size: 8, ep: 1, dp-attn: false }
       - spec-decoding: mtp
         conc-list: [48]
@@ -1785,21 +1771,7 @@ kimik3-fp4-mi355x-vllm-disagg-agentic:
           dp-attn: false
           additional-settings:
           # Recipe enables P-only scratch reclaim; D and GMU0.90 are unchanged.
-          - TOTAL_CPU_DRAM_GB=1799
-        decode: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false }
-      - spec-decoding: mtp
-        conc-list: [56]
-        kv-offloading: dram
-        kv-offload-backend: { name: vllm-simple }
-        srt-recipe: disagg-variants.yaml:override_1p1d_c56
-        prefill:
-          num-worker: 1
-          tp: 8
-          dcp-size: 8
-          ep: 1
-          dp-attn: false
-          additional-settings:
-          - TOTAL_CPU_DRAM_GB=1799
+          - TOTAL_CPU_DRAM_GB=2399
         decode: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false }
       - spec-decoding: mtp
         conc-list: [48]
@@ -1814,5 +1786,12 @@ kimik3-fp4-mi355x-vllm-disagg-agentic:
           dp-attn: false
           additional-settings:
           # Inherits the same P-only scratch-reclaim setting as 1P1D c48.
-          - TOTAL_CPU_DRAM_GB=1799
+          - TOTAL_CPU_DRAM_GB=2399
         decode: { num-worker: 2, tp: 8, dcp-size: 8, ep: 1, dp-attn: false }
+    - search-space:
+      - spec-decoding: mtp
+        conc-list: [8]
+        kv-offloading: none
+        srt-recipe: disagg-variants.yaml:override_1p1d_c8
+        prefill: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false }
+        decode: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false }

inferencex-e2e/perf-changelog.yaml

@@ -11414,6 +11414,14 @@
   - Enable upstream MiniMax-M3 monokernel at TP4 with Model Runner V2, AITER MoE, FP8 KV/indexer and block size 128; keep TP2 on ordinary decode. Preserve EAGLE3-GQA with three draft tokens and use automatic golden acceptance for throughput with real verification for evals.
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4418
 
+- config-keys:
+    - kimik3-fp4-mi355x-vllm-disagg-agentic
+  description:
+    - "Use ROCm nightly-c50aca14416ef29f6ed0ed256302f30646726dcd pinned by digest and BF16 SSM states for the throughput points."
+    - "Use 1P1D c8 with K7 specialized decode, max-num-seqs=1 on P/D, FP32 SSM states, and GPU-resident KV; retire c56 and 1P3D c12."
+    - "Set prefill-only SimpleCPUOffload points to DRAM utilization 0.8 (2399 GB per P worker)."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4391
+
 - config-keys:
     - glm5.2-fp4-mi355x-atom-agentic-mtp
   scenario-type: