diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic-lowconc.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic-lowconc.yaml new file mode 100644 index 0000000000..d17065c477 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic-lowconc.yaml @@ -0,0 +1,73 @@ +base: + schema: 2 + name: dsv4-fp4-b200-vllm-agentic-lowconc + resources: + gpu_type: b200 + gpus_per_node: 8 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + # The engine may take the full VLLM_ENGINE_READY_TIMEOUT_S to become ready. + health_check: + interval_seconds: 10 + max_attempts: 720 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + kv-cache-dtype: fp8 + block-size: 256 + max-model-len: 1048576 + gpu-memory-utilization: 0.90 + numa-bind: true + enable-cumem-allocator: true + no-enable-flashinfer-autotune: true + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"indexer_kv_dtype":"mxfp4"}' + # Throughput runs switch to synthetic rejection at the golden acceptance length. + speculative-config: '{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"probabilistic"}' + no-disable-hybrid-kv-cache-manager: true + disable-uvicorn-access-log: true + env: + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '32768' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_DSV4_MEGA_FP8_COMBINE: '1' + VLLM_RPC_TIMEOUT: '600000' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + VLLM_FLOAT32_MATMUL_PRECISION: high + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +override_tp8_c4: + roles: + agg: + gpus: 8 + args: + tensor-parallel-size: 8 + data-parallel-size: 1 + max-num-batched-tokens: 16384 + kv-cache-memory-bytes: 42949672960 + sparse-indexer-topk-backend: deep_select + max-num-seqs: 4 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[7,14,21,28,100,200,300,400,500,640,800,1024,1280,1600,2048,2560,3200,4096,5120,6400,8192,10240,12800,16384],"max_cudagraph_capture_size":16384}' + benchmark: + env: + CONC: '4' + KV_OFFLOADING: none diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic-tp2dp4-c48.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic-tp2dp4-c48.yaml new file mode 100644 index 0000000000..b9797b3037 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic-tp2dp4-c48.yaml @@ -0,0 +1,75 @@ +name: dsv4-fp4-b200-vllm-agentic_tp2dp4_c48 +schema: 2 +resources: + gpu_type: b200 + gpus_per_node: 8 +frontend: + type: vllm-router + enable_multiple_frontends: false + args: + disable-retries: true + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 +observability: + enabled: false + tachometer: + enabled: false +engine: + type: vllm + connector: null +health_check: + interval_seconds: 10 + max_attempts: 720 +roles: + agg: + nodes: 1 + workers: 1 + args: + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"indexer_kv_dtype":"mxfp4"}' + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168],"mode":0}' + data-parallel-size: 4 + disable-uvicorn-access-log: true + enable-auto-tool-choice: true + enable-cumem-allocator: true + enable-ep-weight-filter: true + enable-expert-parallel: true + gpu-memory-utilization: 0.9 + kv-cache-dtype: fp8 + kv-cache-memory-bytes: 37580963840 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":241500000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + long-prefill-token-threshold: 512 + max-model-len: 1048576 + max-num-batched-tokens: 8360 + max-num-seqs: 24 + moe-backend: deep_gemm_mega_moe + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + numa-bind: true + prefill-schedule-interval: 16 + reasoning-parser: deepseek_v4 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + speculative-config: '{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.77}' + tensor-parallel-size: 2 + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + trust-remote-code: true + env: + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTORCH_ALLOC_CONF: expandable_segments:True + TORCH_CUDA_ARCH_LIST: '10.0' + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '32768' + VLLM_RPC_TIMEOUT: '600000' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + gpus: 8 +benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' +setup_script: vllm-router.sh diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic-tp2dp4-c64.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic-tp2dp4-c64.yaml new file mode 100644 index 0000000000..6c6612e669 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic-tp2dp4-c64.yaml @@ -0,0 +1,75 @@ +name: dsv4-fp4-b200-vllm-agentic_tp2dp4_c64 +schema: 2 +resources: + gpu_type: b200 + gpus_per_node: 8 +frontend: + type: vllm-router + enable_multiple_frontends: false + args: + disable-retries: true + policy: consistent_hash + prometheus-host: 127.0.0.1 + prometheus-port: 18000 + request-timeout-secs: 14400 +observability: + enabled: false + tachometer: + enabled: false +engine: + type: vllm + connector: null +health_check: + interval_seconds: 10 + max_attempts: 720 +roles: + agg: + nodes: 1 + workers: 1 + args: + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"indexer_kv_dtype":"mxfp4"}' + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224],"mode":0}' + data-parallel-size: 4 + disable-uvicorn-access-log: true + enable-auto-tool-choice: true + enable-cumem-allocator: true + enable-ep-weight-filter: true + enable-expert-parallel: true + gpu-memory-utilization: 0.9 + kv-cache-dtype: fp8 + kv-cache-memory-bytes: 37580963840 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":241500000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' + long-prefill-token-threshold: 512 + max-model-len: 1048576 + max-num-batched-tokens: 8416 + max-num-seqs: 32 + moe-backend: deep_gemm_mega_moe + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + numa-bind: true + prefill-schedule-interval: 16 + reasoning-parser: deepseek_v4 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + speculative-config: '{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.77}' + tensor-parallel-size: 2 + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + trust-remote-code: true + env: + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTORCH_ALLOC_CONF: expandable_segments:True + TORCH_CUDA_ARCH_LIST: '10.0' + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '32768' + VLLM_RPC_TIMEOUT: '600000' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + gpus: 8 +benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: '1' +setup_script: vllm-router.sh diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml index 6c62fc75c2..960b13e235 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv4/vllm/b200-fp4-mtp/agentic.yaml @@ -62,7 +62,7 @@ base: # consistent-hash vLLM Router (turns of one conversation share a rank), admits # 2x CONC across the ranks with MegaMoE experts, reserves verification slots # beyond an 8192-token prefill budget, and splits TOTAL_CPU_DRAM_GB GB of eager -# offload across the ranks; c160 and c192 shrink the per-rank KV cache. +# offload across the ranks; c96 to c192 shrink the per-rank KV cache. override_tp8_c1: roles: agg: @@ -173,20 +173,21 @@ override_dep8_c32: agg: gpus: 8 args: - tensor-parallel-size: 1 - data-parallel-size: 8 + tensor-parallel-size: 4 + data-parallel-size: 2 enable-expert-parallel: true enable-ep-weight-filter: true - moe-backend: deep_gemm_amxf4_mega_moe + moe-backend: deep_gemm_mega_moe prefill-schedule-interval: 16 long-prefill-token-threshold: 512 - kv-cache-memory-bytes: 37580963840 - max-num-batched-tokens: 8248 - max-num-seqs: 8 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56],"mode":0}' + kv-cache-memory-bytes: 51539607552 + max-num-batched-tokens: 8416 + max-num-seqs: 32 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224],"mode":0}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":241500000000,"enable_cross_layers_blocks":"true","lazy_offload":false}}' env: PYTORCH_ALLOC_CONF: 'expandable_segments:True' + VLLM_DSV4_MEGA_FP8_COMBINE: null PYTHONHASHSEED: '42' benchmark: env: @@ -213,7 +214,7 @@ override_dep8_c64: data-parallel-size: 8 enable-expert-parallel: true enable-ep-weight-filter: true - moe-backend: deep_gemm_amxf4_mega_moe + moe-backend: deep_gemm_mega_moe prefill-schedule-interval: 16 long-prefill-token-threshold: 512 kv-cache-memory-bytes: 37580963840 @@ -249,10 +250,10 @@ override_dep8_c96: data-parallel-size: 8 enable-expert-parallel: true enable-ep-weight-filter: true - moe-backend: deep_gemm_amxf4_mega_moe + moe-backend: deep_gemm_mega_moe prefill-schedule-interval: 16 long-prefill-token-threshold: 512 - kv-cache-memory-bytes: 37580963840 + kv-cache-memory-bytes: 33285996544 max-num-batched-tokens: 8360 max-num-seqs: 24 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168],"mode":0}' @@ -285,10 +286,10 @@ override_dep8_c128: data-parallel-size: 8 enable-expert-parallel: true enable-ep-weight-filter: true - moe-backend: deep_gemm_amxf4_mega_moe + moe-backend: deep_gemm_mega_moe prefill-schedule-interval: 16 long-prefill-token-threshold: 512 - kv-cache-memory-bytes: 37580963840 + kv-cache-memory-bytes: 33285996544 max-num-batched-tokens: 8416 max-num-seqs: 32 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224],"mode":0}' @@ -321,10 +322,10 @@ override_dep8_c160: data-parallel-size: 8 enable-expert-parallel: true enable-ep-weight-filter: true - moe-backend: deep_gemm_amxf4_mega_moe + moe-backend: deep_gemm_mega_moe prefill-schedule-interval: 16 long-prefill-token-threshold: 512 - kv-cache-memory-bytes: 36507222016 + kv-cache-memory-bytes: 32212254720 max-num-batched-tokens: 8472 max-num-seqs: 40 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224,231,238,245,252,259,266,273,280],"mode":0}' @@ -357,10 +358,10 @@ override_dep8_c192: data-parallel-size: 8 enable-expert-parallel: true enable-ep-weight-filter: true - moe-backend: deep_gemm_amxf4_mega_moe + moe-backend: deep_gemm_mega_moe prefill-schedule-interval: 16 long-prefill-token-threshold: 512 - kv-cache-memory-bytes: 36507222016 + kv-cache-memory-bytes: 32212254720 max-num-batched-tokens: 8528 max-num-seqs: 48 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133,140,147,154,161,168,175,182,189,196,203,210,217,224,231,238,245,252,259,266,273,280,287,294,301,308,315,322,329,336],"mode":0}' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 9e29891fd0..9b35361ab4 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -889,7 +889,7 @@ dsv4-fp4-b200-sglang-agentic-hicache-mtp: - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [64, 96, 128, 160], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } dsv4-fp4-b200-vllm-agentic-mtp: - image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-20ebb28@sha256:dfdee3dbba3d8d30554d90f723a3a18dac562f8442f2c2e8f81804acd1ec673f + image: vllm/vllm-openai:nightly-8e2beaa3bff595577a069d49d96bde9074294ec2@sha256:3207cb99c0f59b92f7d47a74da8c6eb5467565d811a4f8d9df51df5188f59c70 model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:b200-nscale @@ -902,10 +902,27 @@ dsv4-fp4-b200-vllm-agentic-mtp: - dram-utilization: 0.90 search-space: # TP8 resident KV + DSpark6 - - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 6, 10, 14, 16], srt-recipe: agentic.yaml } + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 6, 14], srt-recipe: agentic.yaml } # DEP8 SimpleCPU + DSpark6 - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [32, 64, 96, 128, 160, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } +dsv4-fp4-b200-vllm-agentic-mtp-lowconc: + image: vllm/vllm-openai@sha256:d428631bb917a33d22308265d1b8be91522f862bf9fd133bee62dde3313664ae + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:b200-nscale + precision: fp4 + framework: vllm + multinode: false + srt-recipe-dir: dsv4/vllm/b200-fp4-mtp + scenarios: + agentic-coding: + - dram-utilization: 0.90 + search-space: + - { tp: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [4], srt-recipe: agentic-lowconc.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [64], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic-tp2dp4-c64.yaml } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [48], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic-tp2dp4-c48.yaml } + # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 # does not have a B300-specific recipe, so this config reuses the existing DSR1 FP4 # B200 SGLang recipe as-is until B300-specific tuning is available. diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index adf7231b28..f29d4ac640 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11524,3 +11524,13 @@ - "Rerun the whole curve so every point uses the FP8 draft KV cache." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4598 scenario-type: [agentic-coding] + +- config-keys: + - dsv4-fp4-b200-vllm-agentic-mtp + - dsv4-fp4-b200-vllm-agentic-mtp-lowconc + scenario-type: + - agentic-coding + description: + - "Update the DeepSeek-V4-Pro-0813 FP4 B200 AgentX vLLM image to vllm/vllm-openai:nightly-8e2beaa3bff595577a069d49d96bde9074294ec2 (digest-pinned). Keep TP8 c1, c6 and c14, move TP8 c4 to the lowconc key and remove TP8 c10 and c16. Retune c32 to TP4 x DP2 with deep_gemm_mega_moe and SimpleCPU offload, and switch DEP8 c64 to c192 from deep_gemm_amxf4_mega_moe to deep_gemm_mega_moe with a smaller per-rank KV cache at c96 to c192." + - "Add the dsv4-fp4-b200-vllm-agentic-mtp-lowconc key (vllm/vllm-openai digest-pinned) with TP8 c4 and TP2 x DP4 c48 and c64." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4554