DeepSeek-V4-Pro-0813 B200 FP4 vLLM AgentX/ DeepSeek-V4-Pro-0813 B200 FP4 vLLM AgentX

inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic-lowconc.yaml

@@ -0,0 +1,73 @@
+base:
+  schema: 2
+  name: dsv4-fp4-b200-vllm-agentic-lowconc
+  resources:
+    gpu_type: b200
+    gpus_per_node: 8
+  frontend:
+    type: vllm
+    enable_multiple_frontends: false
+  observability:
+    enabled: false
+    tachometer:
+      enabled: false
+  engine:
+    type: vllm
+    connector: null
+  # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready.
+  health_check:
+    interval_seconds: 10
+    max_attempts: 720
+  roles:
+    agg:
+      nodes: 1
+      workers: 1
+      args:
+        served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
+        trust-remote-code: true
+        kv-cache-dtype: fp8
+        block-size: 256
+        max-model-len: 1048576
+        gpu-memory-utilization: 0.90
+        numa-bind: true
+        enable-cumem-allocator: true
+        no-enable-flashinfer-autotune: true
+        tokenizer-mode: deepseek_v4
+        tool-call-parser: deepseek_v4
+        enable-auto-tool-choice: true
+        reasoning-parser: deepseek_v4
+        attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"indexer_kv_dtype":"mxfp4"}'
+        # Throughput runs switch to synthetic rejection at the golden acceptance length.
+        speculative-config: '{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"probabilistic"}'
+        no-disable-hybrid-kv-cache-manager: true
+        disable-uvicorn-access-log: true
+      env:
+        VLLM_ENGINE_READY_TIMEOUT_S: '7200'
+        VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '32768'
+        VLLM_USE_V2_MODEL_RUNNER: '1'
+        VLLM_USE_RUST_FRONTEND: '1'
+        VLLM_DSV4_MEGA_FP8_COMBINE: '1'
+        VLLM_RPC_TIMEOUT: '600000'
+        TORCH_CUDA_ARCH_LIST: '10.0'
+        PYTHONNOUSERSITE: '1'
+        VLLM_FLOAT32_MATMUL_PRECISION: high
+  benchmark:
+    env:
+      AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:'
+
+override_tp8_c4:
+  roles:
+    agg:
+      gpus: 8
+      args:
+        tensor-parallel-size: 8
+        data-parallel-size: 1
+        max-num-batched-tokens: 16384
+        kv-cache-memory-bytes: 42949672960
+        sparse-indexer-topk-backend: deep_select
+        max-num-seqs: 4
+        compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,100,200,300,400,500,640,800,1024,1280,1600,2048,2560,3200,4096,5120,6400,8192,10240,12800,16384],"max_cudagraph_capture_size":16384}'
+  benchmark:
+    env:
+      CONC: '4'
+      KV_OFFLOADING: none

inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic-tp2dp4-c48.yaml

@@ -0,0 +1,75 @@
+name: dsv4-fp4-b200-vllm-agentic_tp2dp4_c48
+schema: 2
+resources:
+  gpu_type: b200
+  gpus_per_node: 8
+frontend:
+  type: vllm-router
+  enable_multiple_frontends: false
+  args:
+    disable-retries: true
+    policy: consistent_hash
+    prometheus-host: 127.0.0.1
+    prometheus-port: 18000
+    request-timeout-secs: 14400
+observability:
+  enabled: false
+  tachometer:
+    enabled: false
+engine:
+  type: vllm
+  connector: null
+health_check:
+  interval_seconds: 10
+  max_attempts: 720
+roles:
+  agg:
+    nodes: 1
+    workers: 1
+    args:
+      attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"indexer_kv_dtype":"mxfp4"}'
+      block-size: 256
+      compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168],"mode":0}'
+      data-parallel-size: 4
+      disable-uvicorn-access-log: true
+      enable-auto-tool-choice: true
+      enable-cumem-allocator: true
+      enable-ep-weight-filter: true
+      enable-expert-parallel: true
+      gpu-memory-utilization: 0.9
+      kv-cache-dtype: fp8
+      kv-cache-memory-bytes: 37580963840
+      kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":241500000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'
+      long-prefill-token-threshold: 512
+      max-model-len: 1048576
+      max-num-batched-tokens: 8360
+      max-num-seqs: 24
+      moe-backend: deep_gemm_mega_moe
+      no-disable-hybrid-kv-cache-manager: true
+      no-enable-flashinfer-autotune: true
+      numa-bind: true
+      prefill-schedule-interval: 16
+      reasoning-parser: deepseek_v4
+      served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
+      speculative-config: '{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.77}'
+      tensor-parallel-size: 2
+      tokenizer-mode: deepseek_v4
+      tool-call-parser: deepseek_v4
+      trust-remote-code: true
+    env:
+      PYTHONHASHSEED: '42'
+      PYTHONNOUSERSITE: '1'
+      PYTORCH_ALLOC_CONF: expandable_segments:True
+      TORCH_CUDA_ARCH_LIST: '10.0'
+      VLLM_ENGINE_READY_TIMEOUT_S: '7200'
+      VLLM_FLOAT32_MATMUL_PRECISION: high
+      VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '32768'
+      VLLM_RPC_TIMEOUT: '600000'
+      VLLM_USE_RUST_FRONTEND: '1'
+      VLLM_USE_V2_MODEL_RUNNER: '1'
+    gpus: 8
+benchmark:
+  env:
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:'
+    AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'
+setup_script: vllm-router.sh

inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic-tp2dp4-c64.yaml

@@ -0,0 +1,75 @@
+name: dsv4-fp4-b200-vllm-agentic_tp2dp4_c64
+schema: 2
+resources:
+  gpu_type: b200
+  gpus_per_node: 8
+frontend:
+  type: vllm-router
+  enable_multiple_frontends: false
+  args:
+    disable-retries: true
+    policy: consistent_hash
+    prometheus-host: 127.0.0.1
+    prometheus-port: 18000
+    request-timeout-secs: 14400
+observability:
+  enabled: false
+  tachometer:
+    enabled: false
+engine:
+  type: vllm
+  connector: null
+health_check:
+  interval_seconds: 10
+  max_attempts: 720
+roles:
+  agg:
+    nodes: 1
+    workers: 1
+    args:
+      attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"indexer_kv_dtype":"mxfp4"}'
+      block-size: 256
+      compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224],"mode":0}'
+      data-parallel-size: 4
+      disable-uvicorn-access-log: true
+      enable-auto-tool-choice: true
+      enable-cumem-allocator: true
+      enable-ep-weight-filter: true
+      enable-expert-parallel: true
+      gpu-memory-utilization: 0.9
+      kv-cache-dtype: fp8
+      kv-cache-memory-bytes: 37580963840
+      kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":241500000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'
+      long-prefill-token-threshold: 512
+      max-model-len: 1048576
+      max-num-batched-tokens: 8416
+      max-num-seqs: 32
+      moe-backend: deep_gemm_mega_moe
+      no-disable-hybrid-kv-cache-manager: true
+      no-enable-flashinfer-autotune: true
+      numa-bind: true
+      prefill-schedule-interval: 16
+      reasoning-parser: deepseek_v4
+      served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813
+      speculative-config: '{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.77}'
+      tensor-parallel-size: 2
+      tokenizer-mode: deepseek_v4
+      tool-call-parser: deepseek_v4
+      trust-remote-code: true
+    env:
+      PYTHONHASHSEED: '42'
+      PYTHONNOUSERSITE: '1'
+      PYTORCH_ALLOC_CONF: expandable_segments:True
+      TORCH_CUDA_ARCH_LIST: '10.0'
+      VLLM_ENGINE_READY_TIMEOUT_S: '7200'
+      VLLM_FLOAT32_MATMUL_PRECISION: high
+      VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '32768'
+      VLLM_RPC_TIMEOUT: '600000'
+      VLLM_USE_RUST_FRONTEND: '1'
+      VLLM_USE_V2_MODEL_RUNNER: '1'
+    gpus: 8
+benchmark:
+  env:
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:'
+    AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1'
+setup_script: vllm-router.sh

inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml

@@ -62,7 +62,7 @@ base:
 # consistent-hash vLLM Router (turns of one conversation share a rank), admits
 # 2x CONC across the ranks with MegaMoE experts, reserves verification slots
 # beyond an 8192-token prefill budget, and splits TOTAL_CPU_DRAM_GB GB of eager
-# offload across the ranks; c160 and c192 shrink the per-rank KV cache.
+# offload across the ranks; c96 to c192 shrink the per-rank KV cache.
 override_tp8_c1:
   roles:
     agg:
@@ -116,12 +116,17 @@ override_tp8_c10:
     agg:
       gpus: 8
       args:
+        attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"indexer_kv_dtype":"mxfp4","indexer_tp_shard_min_rows":4096}'
+        prefix-cache-retention-interval: 32768
+        prefill-schedule-interval: 64
         tensor-parallel-size: 8
         data-parallel-size: 1
         max-num-batched-tokens: 16384
         kv-cache-memory-bytes: 53687091200
         max-num-seqs: 10
         compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,100,200,300,400,500]}'
+      env:
+        VLLM_DSV4_MEGA_FP8_COMBINE: null
   benchmark:
     env:
       CONC: '10'
@@ -173,20 +178,21 @@ override_dep8_c32:
     agg:
       gpus: 8
       args:
-        tensor-parallel-size: 1
-        data-parallel-size: 8
+        tensor-parallel-size: 4
+        data-parallel-size: 2
         enable-expert-parallel: true
         enable-ep-weight-filter: true
-        moe-backend: deep_gemm_amxf4_mega_moe
+        moe-backend: deep_gemm_mega_moe
         prefill-schedule-interval: 16
         long-prefill-token-threshold: 512
-        kv-cache-memory-bytes: 37580963840
-        max-num-batched-tokens: 8248
-        max-num-seqs: 8
-        compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56],"mode":0}'
+        kv-cache-memory-bytes: 51539607552
+        max-num-batched-tokens: 8416
+        max-num-seqs: 32
+        compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224],"mode":0}'
         kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":241500000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}'
       env:
         PYTORCH_ALLOC_CONF: 'expandable_segments:True'
+        VLLM_DSV4_MEGA_FP8_COMBINE: null
         PYTHONHASHSEED: '42'
   benchmark:
     env:
@@ -213,7 +219,7 @@ override_dep8_c64:
         data-parallel-size: 8
         enable-expert-parallel: true
         enable-ep-weight-filter: true
-        moe-backend: deep_gemm_amxf4_mega_moe
+        moe-backend: deep_gemm_mega_moe
         prefill-schedule-interval: 16
         long-prefill-token-threshold: 512
         kv-cache-memory-bytes: 37580963840
@@ -249,10 +255,10 @@ override_dep8_c96:
         data-parallel-size: 8
         enable-expert-parallel: true
         enable-ep-weight-filter: true
-        moe-backend: deep_gemm_amxf4_mega_moe
+        moe-backend: deep_gemm_mega_moe
         prefill-schedule-interval: 16
         long-prefill-token-threshold: 512
-        kv-cache-memory-bytes: 37580963840
+        kv-cache-memory-bytes: 33285996544
         max-num-batched-tokens: 8360
         max-num-seqs: 24
         compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168],"mode":0}'
@@ -285,10 +291,10 @@ override_dep8_c128:
         data-parallel-size: 8
         enable-expert-parallel: true
         enable-ep-weight-filter: true
-        moe-backend: deep_gemm_amxf4_mega_moe
+        moe-backend: deep_gemm_mega_moe
         prefill-schedule-interval: 16
         long-prefill-token-threshold: 512
-        kv-cache-memory-bytes: 37580963840
+        kv-cache-memory-bytes: 33285996544
         max-num-batched-tokens: 8416
         max-num-seqs: 32
         compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224],"mode":0}'
@@ -321,10 +327,10 @@ override_dep8_c160:
         data-parallel-size: 8
         enable-expert-parallel: true
         enable-ep-weight-filter: true
-        moe-backend: deep_gemm_amxf4_mega_moe
+        moe-backend: deep_gemm_mega_moe
         prefill-schedule-interval: 16
         long-prefill-token-threshold: 512
-        kv-cache-memory-bytes: 36507222016
+        kv-cache-memory-bytes: 32212254720
         max-num-batched-tokens: 8472
         max-num-seqs: 40
         compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224,231,238,245,252,259,266,273,280],"mode":0}'
@@ -357,10 +363,10 @@ override_dep8_c192:
         data-parallel-size: 8
         enable-expert-parallel: true
         enable-ep-weight-filter: true
-        moe-backend: deep_gemm_amxf4_mega_moe
+        moe-backend: deep_gemm_mega_moe
         prefill-schedule-interval: 16
         long-prefill-token-threshold: 512
-        kv-cache-memory-bytes: 36507222016
+        kv-cache-memory-bytes: 32212254720
         max-num-batched-tokens: 8528
         max-num-seqs: 48
         compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224,231,238,245,252,259,266,273,280,287,294,301,308,315,322,329,336],"mode":0}'

inferencex-e2e/configs/nvidia-master.yaml

@@ -889,7 +889,7 @@ dsv4-fp4-b200-sglang-agentic-hicache-mtp:
       - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [64, 96, 128, 160], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml }
 
 dsv4-fp4-b200-vllm-agentic-mtp:
-  image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-20ebb28@sha256:dfdee3dbba3d8d30554d90f723a3a18dac562f8442f2c2e8f81804acd1ec673f
+  image: vllm/vllm-openai:nightly-8e2beaa3bff595577a069d49d96bde9074294ec2@sha256:3207cb99c0f59b92f7d47a74da8c6eb5467565d811a4f8d9df51df5188f59c70
   model: deepseek-ai/DeepSeek-V4-Pro-0813
   model-prefix: dsv4
   runner: cluster:b200-nscale
@@ -902,10 +902,28 @@ dsv4-fp4-b200-vllm-agentic-mtp:
     - dram-utilization: 0.90
       search-space:
       # TP8 resident KV + DSpark6
-      - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 6, 10, 14, 16], srt-recipe: agentic.yaml }
+      - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 6, 14], srt-recipe: agentic.yaml }
       # DEP8 SimpleCPU + DSpark6
       - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [32, 64, 96, 128, 160, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml }
 
+dsv4-fp4-b200-vllm-agentic-mtp-lowconc:
+  image: vllm/vllm-openai@sha256:d428631bb917a33d22308265d1b8be91522f862bf9fd133bee62dde3313664ae
+  model: deepseek-ai/DeepSeek-V4-Pro-0813
+  model-prefix: dsv4
+  runner: cluster:b200-nscale
+  precision: fp4
+  framework: vllm
+  multinode: false
+  srt-recipe-dir: dsv4/vllm/b200-fp4-mtp
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.90
+      search-space:
+      - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [4], srt-recipe: agentic-lowconc.yaml }
+      - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [10], srt-recipe: agentic.yaml }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [64], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic-tp2dp4-c64.yaml }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [48], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic-tp2dp4-c48.yaml }
+
 # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1
 # does not have a B300-specific recipe, so this config reuses the existing DSR1 FP4
 # B200 SGLang recipe as-is until B300-specific tuning is available.

inferencex-e2e/perf-changelog.yaml

@@ -11459,3 +11459,13 @@
   description:
   - 'Update GB300 Kimi-K3 AgentX to image vllm/vllm-openai:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d and Dynamo a0de541ea8c861d5ac01d15953ca529bea68637b, refresh the recipe settings, drop 1P2D c56, and add 3P1D TP8 prefill + DEP16 decode c336 with online MXFP8 target quantization (--quantization-config targets: in_proj_qkvgfab, fused_qkv_a_g_proj and routed-expert up/down projections on both roles, plus o_proj and shared experts on decode; the DSpark draft is not quantized).'
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4516
+
+- config-keys:
+    - dsv4-fp4-b200-vllm-agentic-mtp
+    - dsv4-fp4-b200-vllm-agentic-mtp-lowconc
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Update the vLLM image to vllm/vllm-openai:nightly-8e2beaa3bff595577a069d49d96bde9074294ec2 (digest-pinned), and retune DeepSeek-V4-Pro-0813 FP4 B200 AgentX DEP8 c32 with MegaMoE and SimpleCPU offload."
+    - "Add the dsv4-fp4-b200-vllm-agentic-mtp-lowconc key (vllm/vllm-openai digest-pinned) with TP8 c1, c4, c10 and c16 and TP2 x DP4 c48 and c64."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4554