From 8461aab797429806d63716d8981e78515cf08431 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sat, 24 Oct 2026 08:28:17 -0400 Subject: [PATCH 01/10] Use latest configs for MiniMax M3.1 AgentX submission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 使用最新配置提交 MiniMax M3.1 AgentX。 --- .../vllm/gb300-fp4-mtp/agentic.yaml | 1 + .../vllm/gb300-fp4-mtp/agg-variants.yaml | 101 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 13 ++- inferencex-e2e/perf-changelog.yaml | 7 ++ 4 files changed, 117 insertions(+), 5 deletions(-) create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agentic.yaml index 94332f36cf..8696a837cd 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agentic.yaml @@ -43,6 +43,7 @@ base: env: VLLM_FLOAT32_MATMUL_PRECISION: high PYTHONUNBUFFERED: '1' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' VLLM_ENGINE_READY_TIMEOUT_S: '7200' # Rendering the chat template for long AgentX conversations exceeds the # 30 s default under concurrent load; vLLM then aborts the request. diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml new file mode 100644 index 0000000000..4b615db5dc --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml @@ -0,0 +1,101 @@ +schema: 2 +base: + name: minimaxm31flash-fp4-gb300-vllm-agentic-dspark + resources: + gpu_type: gb300 + gpus_per_node: 4 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + health_check: + interval_seconds: 5 + max_attempts: 1440 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + language-model-only: true + trust-remote-code: true + reasoning-parser: minimax_m3 + tool-call-parser: minimax_m3 + enable-auto-tool-choice: true + block-size: 128 + kv-cache-dtype: nvfp4 + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + enable-prefix-caching: true + gpu-memory-utilization: 0.92 + max-model-len: 1048576 + max-num-batched-tokens: 16384 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + disable-uvicorn-access-log: true + stream-interval: 20 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + PYTHONUNBUFFERED: '1' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: '600' + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +override_tp2_c48: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-num-seqs: 72 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' + benchmark: + env: + CONC: '48' + +override_tp4_c64: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 96 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":868000000000,"lazy_offload":true}}' + benchmark: + env: + CONC: '64' + +override_tp2_c64: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + max-num-seqs: 96 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' + benchmark: + env: + CONC: '64' + + +override_tp4_c96: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 144 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":868000000000,"lazy_offload":true}}' + benchmark: + env: + CONC: '96' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 5ef2f719c1..6a9b580525 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8885,10 +8885,6 @@ dsv41flash-fp4-gb300-vllm-agentic-dspark: # Router; FlashInfer attention up to conc 64, MegaAttention above. - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 32, 64, 128, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } -# MiniMax-M3.1-Flash AgentX on GB300: the B200 recipe and search space plus a -# DEP4 c96 point, with the vLLM blog's FlashInfer PrimiTS DSpark drafter and the -# day-zero arm64 CUDA 13.4 build of the same image -# (manifest sha256:86f582dae25ea92802353727164ae4eb03b37b18baf9119b805ffaec2a7ef07c). minimaxm31flash-fp4-gb300-vllm-agentic-dspark: image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 model: MiniMaxAI/MiniMax-M3.1-Flash @@ -8902,7 +8898,14 @@ minimaxm31flash-fp4-gb300-vllm-agentic-dspark: agentic-coding: - search-space: - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } - - { tp: 4, ep: 4, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 96, 128], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } + - dram-utilization: 1.0 + search-space: + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [48], srt-recipe: 'agg-variants.yaml:override_tp2_c48' } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [64], srt-recipe: 'agg-variants.yaml:override_tp2_c64' } + - dram-utilization: 0.90 + search-space: + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [64], srt-recipe: 'agg-variants.yaml:override_tp4_c64' } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [96], srt-recipe: 'agg-variants.yaml:override_tp4_c96' } # H200 AgentX arm for DeepSeek-V4.1-Flash. Upstream marks h200 verified and says # the GB200 NVL4 TP4 layout becomes TP8 on 8-GPU nodes, so this is TP8. diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index a50995ce2b..2e4215bab9 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11100,3 +11100,10 @@ - "Search space: Balanced TP4 at conc 1-32; TP8 + EP8 with attention DP2 at conc 64-128." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4466 scenario-type: [agentic-coding] + +- config-keys: + - minimaxm31flash-fp4-gb300-vllm-agentic-dspark + description: + - "Use latest configs for MiniMax M3.1 AgentX submission" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4468 + scenario-type: [agentic-coding] From 452fe6cb5ad72e8aec0032713a175c24548b21f3 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sun, 25 Oct 2026 09:56:15 -0400 Subject: [PATCH 02/10] Use latest configs for MiniMax M3.1 AgentX submission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 使用最新配置提交 MiniMax M3.1 AgentX。 Co-Authored-By: Claude Opus 5.5 --- .../vllm/gb300-fp4-mtp/agentic.yaml | 7 +++- .../vllm/gb300-fp4-mtp/agg-variants.yaml | 42 +++++++------------ inferencex-e2e/configs/nvidia-master.yaml | 9 ++-- 3 files changed, 22 insertions(+), 36 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agentic.yaml index 8696a837cd..27bd3ea70e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agentic.yaml @@ -59,8 +59,11 @@ zip_override_tp4: args: tensor-parallel-size: 4 max-num-seqs: [2, 8, 16, 32, 64] - # Golden AL 4.04. FlashInfer PrimiTS drafter, per the vLLM blog's GB300 runs. - speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-num-batched-tokens: 32768 + # Prefill CUDA graphs up to 8192 tokens; the default capture stops at 512. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' + # Golden AL 4.04. + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: ['1', '4', '8', '16', '32'] diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml index 4b615db5dc..4c01811965 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml @@ -36,10 +36,9 @@ base: gpu-memory-utilization: 0.92 max-model-len: 1048576 max-num-batched-tokens: 16384 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' - speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,264,272,280,288,296,304,312,320,328,336,344,352,360,368,376,384]}' + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' disable-uvicorn-access-log: true - stream-interval: 20 env: VLLM_FLOAT32_MATMUL_PRECISION: high PYTHONUNBUFFERED: '1' @@ -51,51 +50,38 @@ base: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' -override_tp2_c48: +override_tp2_c24: roles: agg: gpus: 2 args: tensor-parallel-size: 2 - max-num-seqs: 72 + max-num-seqs: 36 kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' benchmark: env: - CONC: '48' + CONC: '24' -override_tp4_c64: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - max-num-seqs: 96 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":868000000000,"lazy_offload":true}}' - benchmark: - env: - CONC: '64' - -override_tp2_c64: +override_tp2_c32: roles: agg: gpus: 2 args: tensor-parallel-size: 2 - max-num-seqs: 96 + max-num-seqs: 48 kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' benchmark: env: - CONC: '64' - + CONC: '32' -override_tp4_c96: +override_tp2_c40: roles: agg: - gpus: 4 + gpus: 2 args: - tensor-parallel-size: 4 - max-num-seqs: 144 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":868000000000,"lazy_offload":true}}' + tensor-parallel-size: 2 + max-num-seqs: 60 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' benchmark: env: - CONC: '96' + CONC: '40' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 34e736372e..4fead740c3 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8921,12 +8921,9 @@ minimaxm31flash-fp4-gb300-vllm-agentic-dspark: - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } - dram-utilization: 1.0 search-space: - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [48], srt-recipe: 'agg-variants.yaml:override_tp2_c48' } - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [64], srt-recipe: 'agg-variants.yaml:override_tp2_c64' } - - dram-utilization: 0.90 - search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [64], srt-recipe: 'agg-variants.yaml:override_tp4_c64' } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [96], srt-recipe: 'agg-variants.yaml:override_tp4_c96' } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24], srt-recipe: 'agg-variants.yaml:override_tp2_c24' } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32], srt-recipe: 'agg-variants.yaml:override_tp2_c32' } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [40], srt-recipe: 'agg-variants.yaml:override_tp2_c40' } # MiniMax-M3.1-Flash AgentX on GB300 with SGLang: day-zero image from the SGLang # cookbook (https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1), From c733cfaf685ad364c7ec089f41e430f0669d7242 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sun, 25 Oct 2026 13:45:00 -0400 Subject: [PATCH 03/10] Use latest configs for MiniMax M3.1 AgentX submission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 使用最新配置提交 MiniMax M3.1 AgentX。 Co-Authored-By: Claude Opus 5.5 --- .../vllm/gb300-fp4-mtp/agg-variants.yaml | 55 +++++++++++++++---- inferencex-e2e/configs/nvidia-master.yaml | 4 +- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml index 4c01811965..f0b37ae98c 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml @@ -50,38 +50,71 @@ base: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' -override_tp2_c24: +override_tp2_c32: roles: agg: gpus: 2 args: tensor-parallel-size: 2 - max-num-seqs: 36 + max-num-seqs: 48 kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' benchmark: env: - CONC: '24' + CONC: '32' -override_tp2_c32: +override_tp2_c40: roles: agg: gpus: 2 args: tensor-parallel-size: 2 - max-num-seqs: 48 + max-num-seqs: 60 kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' benchmark: env: - CONC: '32' + CONC: '40' -override_tp2_c40: +override_tp4_c40: roles: agg: - gpus: 2 + gpus: 4 args: - tensor-parallel-size: 2 - max-num-seqs: 60 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' + tensor-parallel-size: 4 + max-num-seqs: 64 + gpu-memory-utilization: 0.86 + max-num-batched-tokens: 32768 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' benchmark: env: CONC: '40' + +override_tp4_c48: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 64 + gpu-memory-utilization: 0.86 + max-num-batched-tokens: 32768 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' + benchmark: + env: + CONC: '48' + +override_tp4_c56: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 96 + gpu-memory-utilization: 0.86 + max-num-batched-tokens: 32768 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' + benchmark: + env: + CONC: '56' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 4fead740c3..4ce8c7cc5e 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8921,7 +8921,9 @@ minimaxm31flash-fp4-gb300-vllm-agentic-dspark: - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } - dram-utilization: 1.0 search-space: - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24], srt-recipe: 'agg-variants.yaml:override_tp2_c24' } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [40], srt-recipe: 'agg-variants.yaml:override_tp4_c40' } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [48], srt-recipe: 'agg-variants.yaml:override_tp4_c48' } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [56], srt-recipe: 'agg-variants.yaml:override_tp4_c56' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32], srt-recipe: 'agg-variants.yaml:override_tp2_c32' } - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [40], srt-recipe: 'agg-variants.yaml:override_tp2_c40' } From 4d0d9088aa132fefab9be71a2782300829f08361 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Mon, 26 Oct 2026 12:37:35 -0400 Subject: [PATCH 04/10] Capture prefill graphs for GB300 TP2 DRAM points --- .../minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml index f0b37ae98c..6dd22ed0be 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml @@ -57,6 +57,9 @@ override_tp2_c32: args: tensor-parallel-size: 2 max-num-seqs: 48 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 32768 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' benchmark: env: @@ -69,6 +72,9 @@ override_tp2_c40: args: tensor-parallel-size: 2 max-num-seqs: 60 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 32768 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' benchmark: env: From cb6537b075c4418e21024a0d1dc926f40c2d80d0 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Mon, 26 Oct 2026 18:55:01 -0400 Subject: [PATCH 05/10] Add Dynamo disagg points for GB300 --- .../gb300-fp4/agentx/disagg-mooncake.yaml | 242 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 33 +++ inferencex-e2e/perf-changelog.yaml | 1 + 3 files changed, 276 insertions(+) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml new file mode 100644 index 0000000000..c40f3c978f --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml @@ -0,0 +1,242 @@ +schema: 2 +base: + identity: + frameworks: + dynamo: 1.6.0.dev20261023 + dynamo: + install: true + source: + wheel: 1.6.0.dev20261023 + resources: + gpu_type: gb300 + gpus_per_node: 4 + health_check: + max_attempts: 1440 + interval_seconds: 5 + environment: + PYTHONHASHSEED: '0' + NCCL_NET_GDR_C2C: '1' + NVSHMEM_REMOTE_TRANSPORT: none + NVSHMEM_ENABLE_NIC_PE_MAPPING: '1' + NVSHMEM_HCA_PE_MAPPING: mlx5_0:1:1,mlx5_1:1:1,mlx5_2:1:1,mlx5_3:1:1 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + preamble: bash /configs/vllm-mooncake.sh + args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + options: + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 200GB + local_buffer_size: 4GB + protocol: rdma + device_name: '@fabric.rdma-devices' + enable_offload: false + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: kv + router-session-affinity-ttl-secs: 14400 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + DYN_TCP_CONNECT_TIMEOUT: '120' + placement: + node: head + engine: + type: vllm + connector: null + dp_launch_mode: per_node + roles: + prefill: + kv_events: true + nodes: 3 + workers: 6 + gpus: 2 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + DYN_TCP_CONNECT_TIMEOUT: '120' + UCX_CUDA_IPC_ENABLE_MNNVL: 'n' + UCX_MEMTYPE_CACHE: 'n' + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_REG_WHOLE: 'n' + UCX_RNDV_PIPELINE_ERROR_HANDLING: 'y' + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + NIXL_UCX_SGL_ENABLE: '1' + NCCL_IB_HCA: '@fabric.rdma-devices' + VLLM_USE_V2_MODEL_RUNNER: '1' + NIXL_TELEMETRY_ENABLE: 'y' + NIXL_TELEMETRY_PROMETHEUS_PORT: '0' + UCX_NET_DEVICES: '@fabric.rdma-device-ports' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: '1024' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: '1024' + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: '2048' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '8' + MC_SLICE_SIZE: '1048576' + MC_MAX_MR_SIZE: '4294967296' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + language-model-only: true + trust-remote-code: true + enable-expert-parallel: true + kv-cache-dtype: nvfp4 + block-size: 128 + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + max-model-len: 1048576 + kernel-config: '{"moe_backend":"deep_gemm_mega_moe"}' + numa-bind: true + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + tensor-parallel-size: 1 + gpu-memory-utilization: 0.85 + max-num-seqs: 64 + max-num-batched-tokens: 32768 + long-prefill-token-threshold: 16384 + prefill-schedule-interval: 1 + enable-prefix-caching: true + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + max-cudagraph-capture-size: 2048 + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"kv_lease_duration":120}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}]}}' + data-parallel-size: 2 + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + DYN_TCP_CONNECT_TIMEOUT: '120' + UCX_CUDA_IPC_ENABLE_MNNVL: 'n' + UCX_MEMTYPE_CACHE: 'n' + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_REG_WHOLE: 'n' + UCX_RNDV_PIPELINE_ERROR_HANDLING: 'y' + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + NIXL_UCX_SGL_ENABLE: '1' + NCCL_IB_HCA: '@fabric.rdma-devices' + VLLM_USE_V2_MODEL_RUNNER: '1' + NIXL_TELEMETRY_ENABLE: 'y' + NIXL_TELEMETRY_PROMETHEUS_PORT: '0' + UCX_NET_DEVICES: '@fabric.rdma-device-ports' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: '1024' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: '1024' + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: '2048' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + MC_MAX_MR_SIZE: '4294967296' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + VLLM_MAX_ASYNC_KV_ADMITS_PER_STEP: '4' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + language-model-only: true + trust-remote-code: true + enable-expert-parallel: true + kv-cache-dtype: nvfp4 + block-size: 128 + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + max-model-len: 1048576 + kernel-config: '{"moe_backend":"deep_gemm_mega_moe"}' + numa-bind: true + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + tensor-parallel-size: 1 + gpu-memory-utilization: 0.95 + max-num-seqs: 48 + max-num-batched-tokens: 8192 + watermark: 0.05 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + max-cudagraph-capture-size: 384 + numa-bind-nodes: + - 0 + - 0 + - 1 + - 1 + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"kv_lease_duration":120}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false,"save_decode_cache":true}}]}}' + data-parallel-size: 8 + enable-prefix-caching: true + data-parallel-rpc-port: 13345 + sbatch_directives: + cpus-per-task: '144' + mem: '0' + srun_options: + container-remap-root: '' + benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: '1.0' + placement: + node: dedicated + name: minimaxm31flash-gb300-disagg-mooncake + setup_script: vllm-mooncake.sh +override_2p2d8: + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + preamble: bash /configs/vllm-mooncake.sh + args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + options: + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 180GB + local_buffer_size: 4GB + protocol: rdma + device_name: '@fabric.rdma-devices' + enable_offload: false + roles: + prefill: + workers: 4 + nodes: 2 + name: minimaxm31flash-gb300-disagg-mooncake-2p2d8 +override_3p2d8: + name: minimaxm31flash-gb300-disagg-mooncake-3p2d8 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index bbdf4a104a..fc9c790790 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -10448,3 +10448,36 @@ glm5.3-fp4-b300-sglang-agentic-mtp: - dram-utilization: 1.0 search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [22, 24, 25, 28], srt-recipe: agentic.yaml } + +minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg: + image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: + name: dynamo-router + version: 1.6.0.dev20261023 + kv-p2p-transfer: nixl + multinode: true + disagg: true + srt-recipe-dir: minimaxm31flash/vllm/gb300-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 1.0 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } + conc-list: [320] + srt-recipe: disagg-mooncake.yaml:override_2p2d8 + prefill: { num-worker: 4, tp: 2, ep: 2, dp-attn: true } + decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } + conc-list: [360, 400] + srt-recipe: disagg-mooncake.yaml:override_3p2d8 + prefill: { num-worker: 6, tp: 2, ep: 2, dp-attn: true } + decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 725eefb24b..69796f6638 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11269,6 +11269,7 @@ - config-keys: - minimaxm31flash-fp4-gb300-vllm-agentic-dspark + - minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg description: - "Use latest configs for MiniMax M3.1 AgentX submission" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4468 From 08e5a20c9e03710924dd32d788eb7147160debac Mon Sep 17 00:00:00 2001 From: Xin Li Date: Mon, 26 Oct 2026 22:10:21 -0400 Subject: [PATCH 06/10] Leave host memory headroom for GB300 disagg prefill --- .../minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml index c40f3c978f..d9fb85e9bd 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml @@ -40,7 +40,7 @@ base: options: store_config: metadata_server: P2PHANDSHAKE - global_segment_size: 200GB + global_segment_size: 180GB local_buffer_size: 4GB protocol: rdma device_name: '@fabric.rdma-devices' From 333a8ee1fb0404d420dfedc408c515227bdf89f6 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 27 Oct 2026 06:52:14 -0400 Subject: [PATCH 07/10] Serve GB300 MiniMax M3.1 aggregated AgentX points through Dynamo Co-Authored-By: Claude Opus 5.5 --- .../vllm/gb300-fp4/agentx}/agg-variants.yaml | 191 ++++++++++++------ .../vllm/gb300-fp4-mtp/agentic.yaml | 8 +- inferencex-e2e/configs/nvidia-master.yaml | 96 ++++++++- inferencex-e2e/perf-changelog.yaml | 2 +- 4 files changed, 217 insertions(+), 80 deletions(-) rename inferencex-e2e/benchmarks/{single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp => multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx}/agg-variants.yaml (53%) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml similarity index 53% rename from inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml rename to inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml index 6dd22ed0be..7850d44d3d 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml @@ -1,126 +1,185 @@ schema: 2 base: - name: minimaxm31flash-fp4-gb300-vllm-agentic-dspark + name: minimaxm31flash-gb300-dynamo-agg + identity: + frameworks: + dynamo: 1.6.0.dev20261023 + dynamo: + install: true + source: + wheel: 1.6.0.dev20261023 resources: gpu_type: gb300 gpus_per_node: 4 + # Cold loads of the 250 GB checkpoint from the hub cache run past half an hour. + health_check: + max_attempts: 1440 + interval_seconds: 5 + environment: + PYTHONHASHSEED: '0' + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 frontend: - type: vllm + type: dynamo enable_multiple_frontends: false - observability: - enabled: false - tachometer: - enabled: false + args: + router-mode: kv + router-session-affinity-ttl-secs: 14400 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + DYN_TCP_CONNECT_TIMEOUT: '120' engine: type: vllm connector: null - health_check: - interval_seconds: 5 - max_attempts: 1440 roles: agg: nodes: 1 workers: 1 + gpus: 4 + kv_events: true + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + VLLM_FLOAT32_MATMUL_PRECISION: high + PYTHONUNBUFFERED: '1' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: '600' + DYN_TCP_CONNECT_TIMEOUT: '120' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: '1024' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: '1024' + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: '2048' args: served-model-name: MiniMaxAI/MiniMax-M3.1-Flash language-model-only: true trust-remote-code: true - reasoning-parser: minimax_m3 - tool-call-parser: minimax_m3 - enable-auto-tool-choice: true + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + # MSA sparse/index cache needs 128-token blocks. block-size: 128 + # The checkpoint is QAT-trained for an NVFP4 KV cache. kv-cache-dtype: nvfp4 attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' disable-hybrid-kv-cache-manager: true enable-prefix-caching: true - gpu-memory-utilization: 0.92 + tensor-parallel-size: 4 + gpu-memory-utilization: 0.9 max-model-len: 1048576 - max-num-batched-tokens: 16384 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,264,272,280,288,296,304,312,320,328,336,344,352,360,368,376,384]}' + max-num-batched-tokens: 32768 + # Prefill CUDA graphs up to 8192 tokens; the default capture stops at 512. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' + # Golden AL 4.04. speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - disable-uvicorn-access-log: true - env: - VLLM_FLOAT32_MATMUL_PRECISION: high - PYTHONUNBUFFERED: '1' - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' - VLLM_ENGINE_READY_TIMEOUT_S: '7200' - VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: '600' - VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + sbatch_directives: + cpus-per-task: '144' + mem: '0' + srun_options: + container-remap-root: '' benchmark: env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' -override_tp2_c32: +override_tp4_c1: + name: minimaxm31flash-gb300-dynamo-agg-tp4-c1 roles: agg: - gpus: 2 args: - tensor-parallel-size: 2 - max-num-seqs: 48 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 32768 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' - benchmark: - env: - CONC: '32' + max-num-seqs: 2 -override_tp2_c40: +override_tp4_c4: + name: minimaxm31flash-gb300-dynamo-agg-tp4-c4 roles: agg: - gpus: 2 args: - tensor-parallel-size: 2 - max-num-seqs: 60 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 32768 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' - benchmark: - env: - CONC: '40' + max-num-seqs: 8 + +override_tp4_c8: + name: minimaxm31flash-gb300-dynamo-agg-tp4-c8 + roles: + agg: + args: + max-num-seqs: 16 + +override_tp4_c16: + name: minimaxm31flash-gb300-dynamo-agg-tp4-c16 + roles: + agg: + args: + max-num-seqs: 32 + +override_tp4_c32: + name: minimaxm31flash-gb300-dynamo-agg-tp4-c32 + roles: + agg: + args: + max-num-seqs: 64 override_tp4_c40: + name: minimaxm31flash-gb300-dynamo-agg-tp4-dram-c40 roles: agg: - gpus: 4 + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' args: - tensor-parallel-size: 4 max-num-seqs: 64 gpu-memory-utilization: 0.86 - max-num-batched-tokens: 32768 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' - benchmark: - env: - CONC: '40' override_tp4_c48: + name: minimaxm31flash-gb300-dynamo-agg-tp4-dram-c48 roles: agg: - gpus: 4 + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' args: - tensor-parallel-size: 4 max-num-seqs: 64 gpu-memory-utilization: 0.86 - max-num-batched-tokens: 32768 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' - benchmark: - env: - CONC: '48' override_tp4_c56: + name: minimaxm31flash-gb300-dynamo-agg-tp4-dram-c56 roles: agg: - gpus: 4 + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' args: - tensor-parallel-size: 4 max-num-seqs: 96 gpu-memory-utilization: 0.86 - max-num-batched-tokens: 32768 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' - benchmark: - env: - CONC: '56' + +override_tp2_c32: + name: minimaxm31flash-gb300-dynamo-agg-tp2-dram-c32 + roles: + agg: + gpus: 2 + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + args: + tensor-parallel-size: 2 + max-num-seqs: 48 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' + +override_tp2_c40: + name: minimaxm31flash-gb300-dynamo-agg-tp2-dram-c40 + roles: + agg: + gpus: 2 + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + args: + tensor-parallel-size: 2 + max-num-seqs: 60 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agentic.yaml index 27bd3ea70e..94332f36cf 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4-mtp/agentic.yaml @@ -43,7 +43,6 @@ base: env: VLLM_FLOAT32_MATMUL_PRECISION: high PYTHONUNBUFFERED: '1' - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' VLLM_ENGINE_READY_TIMEOUT_S: '7200' # Rendering the chat template for long AgentX conversations exceeds the # 30 s default under concurrent load; vLLM then aborts the request. @@ -59,11 +58,8 @@ zip_override_tp4: args: tensor-parallel-size: 4 max-num-seqs: [2, 8, 16, 32, 64] - max-num-batched-tokens: 32768 - # Prefill CUDA graphs up to 8192 tokens; the default capture stops at 512. - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' - # Golden AL 4.04. - speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + # Golden AL 4.04. FlashInfer PrimiTS drafter, per the vLLM blog's GB300 runs. + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: CONC: ['1', '4', '8', '16', '32'] diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 2704838985..ba5d1f1dc5 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8904,6 +8904,10 @@ dsv41flash-fp4-gb300-vllm-agentic-dspark: # Router; FlashInfer attention up to conc 64, MegaAttention above. - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 32, 64, 128, 192], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } +# MiniMax-M3.1-Flash AgentX on GB300: the B200 recipe and search space plus a +# DEP4 c96 point, with the vLLM blog's FlashInfer PrimiTS DSpark drafter and the +# day-zero arm64 CUDA 13.4 build of the same image +# (manifest sha256:86f582dae25ea92802353727164ae4eb03b37b18baf9119b805ffaec2a7ef07c). minimaxm31flash-fp4-gb300-vllm-agentic-dspark: image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 model: MiniMaxAI/MiniMax-M3.1-Flash @@ -8917,13 +8921,7 @@ minimaxm31flash-fp4-gb300-vllm-agentic-dspark: agentic-coding: - search-space: - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } - - dram-utilization: 1.0 - search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [40], srt-recipe: 'agg-variants.yaml:override_tp4_c40' } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [48], srt-recipe: 'agg-variants.yaml:override_tp4_c48' } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [56], srt-recipe: 'agg-variants.yaml:override_tp4_c56' } - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32], srt-recipe: 'agg-variants.yaml:override_tp2_c32' } - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [40], srt-recipe: 'agg-variants.yaml:override_tp2_c40' } + - { tp: 4, ep: 4, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 96, 128], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } # MiniMax-M3.1-Flash AgentX on GB300 with SGLang: day-zero image from the SGLang # cookbook (https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1), @@ -10446,6 +10444,90 @@ glm5.3-fp4-b300-sglang-agentic-mtp: search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [22, 24, 25, 28], srt-recipe: agentic.yaml } +minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg: + image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: + name: dynamo-router + version: 1.6.0.dev20261023 + multinode: true + disagg: false + srt-recipe-dir: minimaxm31flash/vllm/gb300-fp4/agentx + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: [1] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c1 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [4] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c4 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [8] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c8 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [16] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c16 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [32] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c32 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - dram-utilization: 1.0 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [40] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c40 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [48] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c48 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [56] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c56 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [32] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp2_c32 + worker: { num-worker: 1, tp: 2, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [40] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp2_c40 + worker: { num-worker: 1, tp: 2, ep: 1, dp-attn: false } + minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg: image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 model: MiniMaxAI/MiniMax-M3.1-Flash diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 82b390a7a5..1eb27ae898 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11406,7 +11406,7 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4480 - config-keys: - - minimaxm31flash-fp4-gb300-vllm-agentic-dspark + - minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg - minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg description: - "Use latest configs for MiniMax M3.1 AgentX submission" From d12c648d67247c94d7cadb593c758a12f41bc40a Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 27 Oct 2026 09:06:35 -0400 Subject: [PATCH 08/10] Leave more host memory on GB300 disagg prefill trays --- .../vllm/gb300-fp4/agentx/disagg-mooncake.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml index d9fb85e9bd..48a2d1dec0 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml @@ -40,7 +40,7 @@ base: options: store_config: metadata_server: P2PHANDSHAKE - global_segment_size: 180GB + global_segment_size: 160GB local_buffer_size: 4GB protocol: rdma device_name: '@fabric.rdma-devices' @@ -228,7 +228,7 @@ override_2p2d8: options: store_config: metadata_server: P2PHANDSHAKE - global_segment_size: 180GB + global_segment_size: 160GB local_buffer_size: 4GB protocol: rdma device_name: '@fabric.rdma-devices' From e743bb1095b96c7d6b4c33159ce2cb3a5c982202 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 27 Oct 2026 14:31:31 -0400 Subject: [PATCH 09/10] Disable Tachometer on GB multi-node recipes --- .../minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml | 3 +++ .../minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml index 7850d44d3d..51c85e19e9 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml @@ -12,6 +12,9 @@ base: gpu_type: gb300 gpus_per_node: 4 # Cold loads of the 250 GB checkpoint from the hub cache run past half an hour. + observability: + tachometer: + enabled: false health_check: max_attempts: 1440 interval_seconds: 5 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml index 48a2d1dec0..8d76fa88df 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml @@ -10,6 +10,9 @@ base: resources: gpu_type: gb300 gpus_per_node: 4 + observability: + tachometer: + enabled: false health_check: max_attempts: 1440 interval_seconds: 5 From 59c31f1edc67c918b5450695566a98a6a2960578 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 27 Oct 2026 23:32:41 -0400 Subject: [PATCH 10/10] Use lower-latency GB300 disagg points --- .../vllm/gb300-fp4/agentx/agg-variants.yaml | 24 ----------- .../gb300-fp4/agentx/disagg-mooncake.yaml | 42 ++++--------------- inferencex-e2e/configs/nvidia-master.yaml | 26 +++--------- 3 files changed, 15 insertions(+), 77 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml index 51c85e19e9..4d5b07b59e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml @@ -162,27 +162,3 @@ override_tp4_c56: max-num-seqs: 96 gpu-memory-utilization: 0.86 kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' - -override_tp2_c32: - name: minimaxm31flash-gb300-dynamo-agg-tp2-dram-c32 - roles: - agg: - gpus: 2 - env: - VLLM_USE_SIMPLE_KV_OFFLOAD: '1' - args: - tensor-parallel-size: 2 - max-num-seqs: 48 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' - -override_tp2_c40: - name: minimaxm31flash-gb300-dynamo-agg-tp2-dram-c40 - roles: - agg: - gpus: 2 - env: - VLLM_USE_SIMPLE_KV_OFFLOAD: '1' - args: - tensor-parallel-size: 2 - max-num-seqs: 60 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":482000000000,"lazy_offload":true}}' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml index 8d76fa88df..6bf4f89e09 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml @@ -118,7 +118,7 @@ base: gpu-memory-utilization: 0.85 max-num-seqs: 64 max-num-batched-tokens: 32768 - long-prefill-token-threshold: 16384 + long-prefill-token-threshold: 4096 prefill-schedule-interval: 1 enable-prefix-caching: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' @@ -209,37 +209,13 @@ base: node: dedicated name: minimaxm31flash-gb300-disagg-mooncake setup_script: vllm-mooncake.sh -override_2p2d8: - services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - preamble: bash /configs/vllm-mooncake.sh - args: - - --default_kv_lease_ttl=60000 - - --eviction_high_watermark_ratio=0.95 - - --eviction_ratio=0.10 - options: - store_config: - metadata_server: P2PHANDSHAKE - global_segment_size: 160GB - local_buffer_size: 4GB - protocol: rdma - device_name: '@fabric.rdma-devices' - enable_offload: false - roles: - prefill: - workers: 4 - nodes: 2 - name: minimaxm31flash-gb300-disagg-mooncake-2p2d8 override_3p2d8: name: minimaxm31flash-gb300-disagg-mooncake-3p2d8 +override_3p4d16: + name: minimaxm31flash-gb300-disagg-mooncake-3p4d16 + roles: + decode: + nodes: 4 + gpus: 16 + args: + data-parallel-size: 16 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index ba5d1f1dc5..0224099c8f 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -10513,20 +10513,6 @@ minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg: num-nodes: 1 srt-recipe: agg-variants.yaml:override_tp4_c56 worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: vllm-simple } - conc-list: [32] - num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp2_c32 - worker: { num-worker: 1, tp: 2, ep: 1, dp-attn: false } - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: vllm-simple } - conc-list: [40] - num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp2_c40 - worker: { num-worker: 1, tp: 2, ep: 1, dp-attn: false } minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg: image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 @@ -10549,14 +10535,14 @@ minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } - conc-list: [320] - srt-recipe: disagg-mooncake.yaml:override_2p2d8 - prefill: { num-worker: 4, tp: 2, ep: 2, dp-attn: true } + conc-list: [280, 320, 360] + srt-recipe: disagg-mooncake.yaml:override_3p2d8 + prefill: { num-worker: 6, tp: 2, ep: 2, dp-attn: true } decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } - conc-list: [360, 400] - srt-recipe: disagg-mooncake.yaml:override_3p2d8 + conc-list: [300] + srt-recipe: disagg-mooncake.yaml:override_3p4d16 prefill: { num-worker: 6, tp: 2, ep: 2, dp-attn: true } - decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: true }