From 50407d2219c313878221b87bf72a82e839ce5a49 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sat, 24 Oct 2026 09:06:46 -0400 Subject: [PATCH 01/15] Use latest configs for MiniMax M3.1 AgentX submission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 使用最新配置提交 MiniMax M3.1 AgentX。 --- .../vllm/gb200-fp4-mtp/agentic.yaml | 7 +- .../vllm/gb200-fp4-mtp/agg-variants.yaml | 116 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 9 +- inferencex-e2e/perf-changelog.yaml | 7 ++ 4 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml index c9924c776f..90e8e5d0d5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml @@ -54,12 +54,15 @@ zip_override_tp4: gpus: 4 args: tensor-parallel-size: 4 - max-num-seqs: [2, 8, 16, 32, 64] + max-num-seqs: [2, 8, 16, 32] + max-num-batched-tokens: 32768 + # Prefill graphs to 8192 tokens; longer capture lists cost KV memory. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' # Golden AL 4.04. speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: - CONC: ['1', '4', '8', '16', '32'] + CONC: ['1', '4', '8', '16'] zip_override_dep4: frontend: diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml new file mode 100644 index 0000000000..2e93124b2c --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml @@ -0,0 +1,116 @@ +base: + schema: 2 + name: minimaxm31flash-fp4-gb200-vllm-agentic-dspark + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: vllm + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: + type: vllm + connector: null + health_check: + interval_seconds: 5 + max_attempts: 1440 + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + language-model-only: true + trust-remote-code: true + reasoning-parser: minimax_m3 + tool-call-parser: minimax_m3 + enable-auto-tool-choice: true + # MSA sparse/index cache needs 128-token blocks. + block-size: 128 + # The checkpoint is QAT-trained for an NVFP4 KV cache. + kv-cache-dtype: nvfp4 + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + enable-prefix-caching: true + max-model-len: 1048576 + max-num-batched-tokens: 8192 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + disable-uvicorn-access-log: true + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + PYTHONUNBUFFERED: '1' + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +override_tp4_c32_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 64 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + gpu-memory-utilization: 0.86 + max-num-batched-tokens: 32768 + kv-cache-metrics: true + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' + benchmark: + env: + CONC: '32' + +override_tp4_c40_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 64 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + gpu-memory-utilization: 0.86 + max-num-batched-tokens: 32768 + kv-cache-metrics: true + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' + benchmark: + env: + CONC: '40' + +override_tp4_c48_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 64 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + gpu-memory-utilization: 0.86 + max-num-batched-tokens: 32768 + kv-cache-metrics: true + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' + benchmark: + env: + CONC: '48' + +override_tp4_c56_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 96 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + gpu-memory-utilization: 0.86 + max-num-batched-tokens: 32768 + kv-cache-metrics: true + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' + benchmark: + env: + CONC: '56' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 7c0a948b31..6a95a32833 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5655,9 +5655,14 @@ minimaxm31flash-fp4-gb200-vllm-agentic-dspark: srt-recipe-dir: minimaxm31flash/vllm/gb200-fp4-mtp scenarios: agentic-coding: + - dram-utilization: 1.0 + search-space: + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32], srt-recipe: agg-variants.yaml:override_tp4_c32_dram } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [40], srt-recipe: agg-variants.yaml:override_tp4_c40_dram } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [48], srt-recipe: agg-variants.yaml:override_tp4_c48_dram } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [56], srt-recipe: agg-variants.yaml:override_tp4_c56_dram } - search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } - - { tp: 4, ep: 4, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 128], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16], srt-recipe: agentic.yaml } minimaxm3-fp4-b200-trtllm-agentic-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 model: nvidia/MiniMax-M3-NVFP4 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index e77afffe55..f9f213bb36 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11148,3 +11148,10 @@ - "Update the vLLM image from v0.32.0 to nightly-rocm72-c50aca14, pinned by digest." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4461 scenario-type: [agentic-coding] + +- config-keys: + - minimaxm31flash-fp4-gb200-vllm-agentic-dspark + description: + - "Use latest configs for MiniMax M3.1 AgentX submission" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4464 + scenario-type: [agentic-coding] From 9589ef54bab0b4208e6163ad8f359f2b0f0756db Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sun, 25 Oct 2026 03:27:06 -0400 Subject: [PATCH 02/15] Raise HTTP keep-alive for GB200 vLLM server --- .../minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml index 90e8e5d0d5..8166436a50 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml @@ -44,6 +44,7 @@ base: VLLM_FLOAT32_MATMUL_PRECISION: high PYTHONUNBUFFERED: '1' VLLM_ENGINE_READY_TIMEOUT_S: '7200' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' benchmark: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' From 0439b5330918b594af5db17d8877f90589088ab9 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sun, 25 Oct 2026 05:10:31 -0400 Subject: [PATCH 03/15] Leave more HBM headroom at GB200 conc 56 --- .../minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml index 2e93124b2c..2ad8b8f8af 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml @@ -106,7 +106,7 @@ override_tp4_c56_dram: tensor-parallel-size: 4 max-num-seqs: 96 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.86 + gpu-memory-utilization: 0.82 max-num-batched-tokens: 32768 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' From d1f1c7eccf5f41a648f784a500718162cb256a28 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sun, 25 Oct 2026 09:54:08 -0400 Subject: [PATCH 04/15] Keep-alive and HBM headroom for GB200 DRAM points --- .../minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml index 2ad8b8f8af..fc13704dea 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml @@ -43,6 +43,7 @@ base: VLLM_FLOAT32_MATMUL_PRECISION: high PYTHONUNBUFFERED: '1' VLLM_ENGINE_READY_TIMEOUT_S: '7200' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' benchmark: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' @@ -55,7 +56,7 @@ override_tp4_c32_dram: tensor-parallel-size: 4 max-num-seqs: 64 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.86 + gpu-memory-utilization: 0.82 max-num-batched-tokens: 32768 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' @@ -72,7 +73,7 @@ override_tp4_c40_dram: tensor-parallel-size: 4 max-num-seqs: 64 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.86 + gpu-memory-utilization: 0.82 max-num-batched-tokens: 32768 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' @@ -89,7 +90,7 @@ override_tp4_c48_dram: tensor-parallel-size: 4 max-num-seqs: 64 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.86 + gpu-memory-utilization: 0.82 max-num-batched-tokens: 32768 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' From 658cca42b6b827fc907024e0da9e5786ca16a8c3 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Mon, 26 Oct 2026 21:05:23 -0400 Subject: [PATCH 05/15] Add Dynamo disagg points for GB200 --- .../gb200-fp4/agentx/disagg-mooncake.yaml | 217 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 33 +++ inferencex-e2e/perf-changelog.yaml | 1 + 3 files changed, 251 insertions(+) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml new file mode 100644 index 0000000000..e19dffcacb --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml @@ -0,0 +1,217 @@ +schema: 2 +base: + identity: + frameworks: + dynamo: 1.6.0.dev20261023 + dynamo: + install: true + source: + wheel: 1.6.0.dev20261023 + resources: + gpu_type: gb200 + gpus_per_node: 4 + health_check: + max_attempts: 1440 + interval_seconds: 5 + environment: + PYTHONHASHSEED: '0' + NCCL_NET_GDR_C2C: '1' + NVSHMEM_REMOTE_TRANSPORT: none + NVSHMEM_ENABLE_NIC_PE_MAPPING: '1' + NVSHMEM_HCA_PE_MAPPING: mlx5_0:1:1,mlx5_1:1:1,mlx5_2:1:1,mlx5_3:1:1 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + preamble: bash /configs/vllm-mooncake.sh + args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + options: + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 170GB + local_buffer_size: 4GB + protocol: rdma + device_name: '@fabric.rdma-devices' + enable_offload: false + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: kv + router-session-affinity-ttl-secs: 14400 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + DYN_TCP_CONNECT_TIMEOUT: '120' + placement: + node: head + engine: + type: vllm + connector: null + dp_launch_mode: per_node + roles: + prefill: + kv_events: true + nodes: 3 + workers: 3 + gpus: 4 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + DYN_TCP_CONNECT_TIMEOUT: '120' + UCX_CUDA_IPC_ENABLE_MNNVL: 'n' + UCX_MEMTYPE_CACHE: 'n' + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_REG_WHOLE: 'n' + UCX_RNDV_PIPELINE_ERROR_HANDLING: 'y' + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + NIXL_UCX_SGL_ENABLE: '1' + NCCL_IB_HCA: '@fabric.rdma-devices' + VLLM_USE_V2_MODEL_RUNNER: '1' + UCX_NET_DEVICES: '@fabric.rdma-device-ports' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: '1024' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: '1024' + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: '2048' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '8' + MC_SLICE_SIZE: '1048576' + MC_MAX_MR_SIZE: '4294967296' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + language-model-only: true + trust-remote-code: true + enable-expert-parallel: true + kv-cache-dtype: nvfp4 + block-size: 128 + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + max-model-len: 1048576 + kernel-config: '{"moe_backend":"deep_gemm_mega_moe"}' + numa-bind: true + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + tensor-parallel-size: 1 + gpu-memory-utilization: 0.88 + max-num-seqs: 64 + max-num-batched-tokens: 16384 + long-prefill-token-threshold: 16384 + prefill-schedule-interval: 1 + enable-prefix-caching: true + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + max-cudagraph-capture-size: 2048 + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"kv_lease_duration":120}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}]}}' + data-parallel-size: 4 + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + DYN_TCP_CONNECT_TIMEOUT: '120' + UCX_CUDA_IPC_ENABLE_MNNVL: 'n' + UCX_MEMTYPE_CACHE: 'n' + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_REG_WHOLE: 'n' + UCX_RNDV_PIPELINE_ERROR_HANDLING: 'y' + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + NIXL_UCX_SGL_ENABLE: '1' + NCCL_IB_HCA: '@fabric.rdma-devices' + VLLM_USE_V2_MODEL_RUNNER: '1' + UCX_NET_DEVICES: '@fabric.rdma-device-ports' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: '1024' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: '1024' + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: '2048' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + MC_MAX_MR_SIZE: '4294967296' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + VLLM_MAX_ASYNC_KV_ADMITS_PER_STEP: '4' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + language-model-only: true + trust-remote-code: true + enable-expert-parallel: true + kv-cache-dtype: nvfp4 + block-size: 128 + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + max-model-len: 1048576 + kernel-config: '{"moe_backend":"deep_gemm_mega_moe"}' + numa-bind: true + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + tensor-parallel-size: 1 + gpu-memory-utilization: 0.92 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + watermark: 0.05 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + max-cudagraph-capture-size: 512 + numa-bind-nodes: + - 0 + - 0 + - 1 + - 1 + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"kv_lease_duration":120}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false,"save_decode_cache":true}}]}}' + data-parallel-size: 8 + enable-prefix-caching: true + data-parallel-rpc-port: 13345 + sbatch_directives: + cpus-per-task: '72' + mem: '0' + srun_options: + container-remap-root: '' + benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: '1.0' + placement: + node: dedicated + name: minimaxm31flash-gb200-disagg-mooncake + setup_script: vllm-mooncake.sh +override_3p1d_c280: + name: minimaxm31flash-gb200-disagg-mooncake-3p1d-c280 +override_2p1d_c224: + name: minimaxm31flash-gb200-disagg-mooncake-2p1d-c224 + roles: + prefill: + nodes: 2 + workers: 2 + decode: + args: + max-num-seqs: 48 + max-cudagraph-capture-size: 384 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index fe26b335db..e614a7fb6c 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -10329,3 +10329,36 @@ glm5.3-fp4-b300-sglang-agentic-mtp: - dram-utilization: 1.0 search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [22, 24, 25, 28], srt-recipe: agentic.yaml } + +minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-disagg: + image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: + name: dynamo-router + version: 1.6.0.dev20261023 + kv-p2p-transfer: nixl + multinode: true + disagg: true + srt-recipe-dir: minimaxm31flash/vllm/gb200-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 1.0 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } + conc-list: [224] + srt-recipe: disagg-mooncake.yaml:override_2p1d_c224 + prefill: { num-worker: 2, tp: 4, ep: 4, dp-attn: true } + decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } + conc-list: [280] + srt-recipe: disagg-mooncake.yaml:override_3p1d_c280 + prefill: { num-worker: 3, tp: 4, ep: 4, dp-attn: true } + decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 1047adf055..de43d85ef2 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11194,6 +11194,7 @@ - config-keys: - minimaxm31flash-fp4-gb200-vllm-agentic-dspark + - minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-disagg description: - "Use latest configs for MiniMax M3.1 AgentX submission" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4464 From 2c6578c2866e81006df2175df68077a8ce5bce2d Mon Sep 17 00:00:00 2001 From: Xin Li Date: Mon, 26 Oct 2026 23:33:33 -0400 Subject: [PATCH 06/15] Leave HBM headroom for GB200 autotune and raise chat template timeout --- .../minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml | 1 + .../minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml index 8166436a50..38a5e6f88a 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml @@ -45,6 +45,7 @@ base: PYTHONUNBUFFERED: '1' VLLM_ENGINE_READY_TIMEOUT_S: '7200' VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' + VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: '600' benchmark: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml index fc13704dea..560c0eae38 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml @@ -44,6 +44,7 @@ base: PYTHONUNBUFFERED: '1' VLLM_ENGINE_READY_TIMEOUT_S: '7200' VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' + VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: '600' benchmark: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' @@ -56,7 +57,7 @@ override_tp4_c32_dram: tensor-parallel-size: 4 max-num-seqs: 64 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.82 + gpu-memory-utilization: 0.80 max-num-batched-tokens: 32768 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' @@ -73,7 +74,7 @@ override_tp4_c40_dram: tensor-parallel-size: 4 max-num-seqs: 64 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.82 + gpu-memory-utilization: 0.80 max-num-batched-tokens: 32768 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' @@ -90,7 +91,7 @@ override_tp4_c48_dram: tensor-parallel-size: 4 max-num-seqs: 64 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.82 + gpu-memory-utilization: 0.80 max-num-batched-tokens: 32768 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' @@ -107,7 +108,7 @@ override_tp4_c56_dram: tensor-parallel-size: 4 max-num-seqs: 96 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.82 + gpu-memory-utilization: 0.80 max-num-batched-tokens: 32768 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' From b50846257f23281a413db6800b5993bc42cd3de4 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 27 Oct 2026 00:15:24 -0400 Subject: [PATCH 07/15] Leave more HBM headroom for GB200 DRAM points --- .../minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml index 560c0eae38..2bf22b1a5d 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml @@ -57,7 +57,7 @@ override_tp4_c32_dram: tensor-parallel-size: 4 max-num-seqs: 64 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.80 + gpu-memory-utilization: 0.76 max-num-batched-tokens: 32768 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' @@ -74,7 +74,7 @@ override_tp4_c40_dram: tensor-parallel-size: 4 max-num-seqs: 64 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.80 + gpu-memory-utilization: 0.76 max-num-batched-tokens: 32768 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' @@ -91,7 +91,7 @@ override_tp4_c48_dram: tensor-parallel-size: 4 max-num-seqs: 64 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.80 + gpu-memory-utilization: 0.76 max-num-batched-tokens: 32768 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' @@ -108,7 +108,7 @@ override_tp4_c56_dram: tensor-parallel-size: 4 max-num-seqs: 96 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.80 + gpu-memory-utilization: 0.76 max-num-batched-tokens: 32768 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' From bf45980d4006c8b911cb065bc318baed0687c2a9 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 27 Oct 2026 02:36:38 -0400 Subject: [PATCH 08/15] Smaller prefill batch for GB200 DRAM warmup and host memory headroom for disagg --- .../vllm/gb200-fp4/agentx/disagg-mooncake.yaml | 2 +- .../vllm/gb200-fp4-mtp/agg-variants.yaml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml index e19dffcacb..21c91baecf 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml @@ -40,7 +40,7 @@ base: options: store_config: metadata_server: P2PHANDSHAKE - global_segment_size: 170GB + global_segment_size: 150GB local_buffer_size: 4GB protocol: rdma device_name: '@fabric.rdma-devices' diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml index 2bf22b1a5d..20ff507628 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml @@ -57,8 +57,8 @@ override_tp4_c32_dram: tensor-parallel-size: 4 max-num-seqs: 64 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.76 - max-num-batched-tokens: 32768 + gpu-memory-utilization: 0.78 + max-num-batched-tokens: 16384 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' @@ -74,8 +74,8 @@ override_tp4_c40_dram: tensor-parallel-size: 4 max-num-seqs: 64 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.76 - max-num-batched-tokens: 32768 + gpu-memory-utilization: 0.78 + max-num-batched-tokens: 16384 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' @@ -91,8 +91,8 @@ override_tp4_c48_dram: tensor-parallel-size: 4 max-num-seqs: 64 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.76 - max-num-batched-tokens: 32768 + gpu-memory-utilization: 0.78 + max-num-batched-tokens: 16384 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' @@ -108,8 +108,8 @@ override_tp4_c56_dram: tensor-parallel-size: 4 max-num-seqs: 96 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.76 - max-num-batched-tokens: 32768 + gpu-memory-utilization: 0.78 + max-num-batched-tokens: 16384 kv-cache-metrics: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' From a8f468f61e247068aacce4a5cfe593e039e15d60 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 27 Oct 2026 06:55:56 -0400 Subject: [PATCH 09/15] Use latest configs for MiniMax M3.1 AgentX submission --- .../vllm/gb200-fp4/agentx/agg-variants.yaml | 153 ++++++++++++++++++ .../gb200-fp4/agentx/disagg-mooncake.yaml | 4 +- .../vllm/gb200-fp4-mtp/agentic.yaml | 9 +- .../vllm/gb200-fp4-mtp/agg-variants.yaml | 118 -------------- inferencex-e2e/configs/nvidia-master.yaml | 80 ++++++++- inferencex-e2e/perf-changelog.yaml | 2 +- 6 files changed, 232 insertions(+), 134 deletions(-) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml delete mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..9d37f4474e --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,153 @@ +schema: 2 +base: + identity: + frameworks: + dynamo: 1.6.0.dev20261023 + dynamo: + install: true + source: + wheel: 1.6.0.dev20261023 + resources: + gpu_type: gb200 + gpus_per_node: 4 + health_check: + max_attempts: 1440 + interval_seconds: 5 + environment: + PYTHONHASHSEED: '0' + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: kv + router-session-affinity-ttl-secs: 14400 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + DYN_TCP_CONNECT_TIMEOUT: '120' + engine: + type: vllm + connector: null + roles: + agg: + kv_events: true + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + DYN_TCP_CONNECT_TIMEOUT: '120' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: '1024' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: '1024' + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: '2048' + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + language-model-only: true + trust-remote-code: true + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + # MSA sparse/index cache needs 128-token blocks. + block-size: 128 + # The checkpoint is QAT-trained for an NVFP4 KV cache. + kv-cache-dtype: nvfp4 + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + enable-prefix-caching: true + max-model-len: 1048576 + tensor-parallel-size: 4 + max-num-batched-tokens: 32768 + # Prefill graphs to 8192 tokens; longer capture lists cost KV memory. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' + # Golden AL 4.04. + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + sbatch_directives: + cpus-per-task: '72' + mem: '0' + srun_options: + container-remap-root: '' + benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + name: minimaxm31flash-gb200-agg +override_tp4_c1: + name: minimaxm31flash-gb200-agg-tp4-c1 + roles: + agg: + args: + max-num-seqs: 2 +override_tp4_c4: + name: minimaxm31flash-gb200-agg-tp4-c4 + roles: + agg: + args: + max-num-seqs: 8 +override_tp4_c8: + name: minimaxm31flash-gb200-agg-tp4-c8 + roles: + agg: + args: + max-num-seqs: 16 +override_tp4_c16: + name: minimaxm31flash-gb200-agg-tp4-c16 + roles: + agg: + args: + max-num-seqs: 32 +override_tp4_c32: + name: minimaxm31flash-gb200-agg-tp4-dram-c32 + roles: + agg: + args: + max-num-seqs: 64 + gpu-memory-utilization: 0.80 + kv-cache-metrics: true + load-format: fastsafetensors + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":720000000000,"lazy_offload":true}}' +override_tp4_c40: + name: minimaxm31flash-gb200-agg-tp4-dram-c40 + roles: + agg: + args: + max-num-seqs: 64 + gpu-memory-utilization: 0.80 + kv-cache-metrics: true + load-format: fastsafetensors + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":720000000000,"lazy_offload":true}}' +override_tp4_c48: + name: minimaxm31flash-gb200-agg-tp4-dram-c48 + roles: + agg: + args: + max-num-seqs: 64 + gpu-memory-utilization: 0.80 + kv-cache-metrics: true + load-format: fastsafetensors + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":720000000000,"lazy_offload":true}}' +override_tp4_c56: + name: minimaxm31flash-gb200-agg-tp4-dram-c56 + roles: + agg: + args: + max-num-seqs: 96 + gpu-memory-utilization: 0.80 + kv-cache-metrics: true + load-format: fastsafetensors + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":720000000000,"lazy_offload":true}}' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml index 21c91baecf..af87726f5a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml @@ -105,6 +105,7 @@ base: attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' disable-hybrid-kv-cache-manager: true max-model-len: 1048576 + load-format: fastsafetensors kernel-config: '{"moe_backend":"deep_gemm_mega_moe"}' numa-bind: true dyn-tool-call-parser: minimax_m3 @@ -167,6 +168,7 @@ base: attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' disable-hybrid-kv-cache-manager: true max-model-len: 1048576 + load-format: fastsafetensors kernel-config: '{"moe_backend":"deep_gemm_mega_moe"}' numa-bind: true dyn-tool-call-parser: minimax_m3 @@ -184,7 +186,7 @@ base: - 0 - 1 - 1 - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"kv_lease_duration":120}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false,"save_decode_cache":true}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"kv_lease_duration":120}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false,"save_decode_cache":false,"enable_lookup":false}}]}}' data-parallel-size: 8 enable-prefix-caching: true data-parallel-rpc-port: 13345 diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml index 38a5e6f88a..c9924c776f 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agentic.yaml @@ -44,8 +44,6 @@ base: VLLM_FLOAT32_MATMUL_PRECISION: high PYTHONUNBUFFERED: '1' VLLM_ENGINE_READY_TIMEOUT_S: '7200' - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' - VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: '600' benchmark: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' @@ -56,15 +54,12 @@ zip_override_tp4: gpus: 4 args: tensor-parallel-size: 4 - max-num-seqs: [2, 8, 16, 32] - max-num-batched-tokens: 32768 - # Prefill graphs to 8192 tokens; longer capture lists cost KV memory. - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' + max-num-seqs: [2, 8, 16, 32, 64] # Golden AL 4.04. speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' benchmark: env: - CONC: ['1', '4', '8', '16'] + CONC: ['1', '4', '8', '16', '32'] zip_override_dep4: frontend: diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml deleted file mode 100644 index 20ff507628..0000000000 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4-mtp/agg-variants.yaml +++ /dev/null @@ -1,118 +0,0 @@ -base: - schema: 2 - name: minimaxm31flash-fp4-gb200-vllm-agentic-dspark - resources: - gpu_type: gb200 - gpus_per_node: 4 - frontend: - type: vllm - enable_multiple_frontends: false - observability: - enabled: false - tachometer: - enabled: false - engine: - type: vllm - connector: null - health_check: - interval_seconds: 5 - max_attempts: 1440 - roles: - agg: - nodes: 1 - workers: 1 - args: - served-model-name: MiniMaxAI/MiniMax-M3.1-Flash - language-model-only: true - trust-remote-code: true - reasoning-parser: minimax_m3 - tool-call-parser: minimax_m3 - enable-auto-tool-choice: true - # MSA sparse/index cache needs 128-token blocks. - block-size: 128 - # The checkpoint is QAT-trained for an NVFP4 KV cache. - kv-cache-dtype: nvfp4 - attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' - disable-hybrid-kv-cache-manager: true - enable-prefix-caching: true - max-model-len: 1048576 - max-num-batched-tokens: 8192 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' - disable-uvicorn-access-log: true - env: - VLLM_FLOAT32_MATMUL_PRECISION: high - PYTHONUNBUFFERED: '1' - VLLM_ENGINE_READY_TIMEOUT_S: '7200' - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' - VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: '600' - benchmark: - env: - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' - -override_tp4_c32_dram: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - max-num-seqs: 64 - speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.78 - max-num-batched-tokens: 16384 - kv-cache-metrics: true - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' - benchmark: - env: - CONC: '32' - -override_tp4_c40_dram: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - max-num-seqs: 64 - speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.78 - max-num-batched-tokens: 16384 - kv-cache-metrics: true - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' - benchmark: - env: - CONC: '40' - -override_tp4_c48_dram: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - max-num-seqs: 64 - speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.78 - max-num-batched-tokens: 16384 - kv-cache-metrics: true - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' - benchmark: - env: - CONC: '48' - -override_tp4_c56_dram: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - max-num-seqs: 96 - speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - gpu-memory-utilization: 0.78 - max-num-batched-tokens: 16384 - kv-cache-metrics: true - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' - benchmark: - env: - CONC: '56' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 6399b7f5e7..6c9f01e6a7 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5652,14 +5652,9 @@ minimaxm31flash-fp4-gb200-vllm-agentic-dspark: srt-recipe-dir: minimaxm31flash/vllm/gb200-fp4-mtp scenarios: agentic-coding: - - dram-utilization: 1.0 - search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32], srt-recipe: agg-variants.yaml:override_tp4_c32_dram } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [40], srt-recipe: agg-variants.yaml:override_tp4_c40_dram } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [48], srt-recipe: agg-variants.yaml:override_tp4_c48_dram } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [56], srt-recipe: agg-variants.yaml:override_tp4_c56_dram } - search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16], srt-recipe: agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 16, 32], srt-recipe: agentic.yaml } + - { tp: 4, ep: 4, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 128], router: { name: vllm-router, version: "0.1.14" }, srt-recipe: agentic.yaml } minimaxm3-fp4-b200-trtllm-agentic-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 model: nvidia/MiniMax-M3-NVFP4 @@ -10449,6 +10444,77 @@ glm5.3-fp4-b300-sglang-agentic-mtp: search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [22, 24, 25, 28], srt-recipe: agentic.yaml } +minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-agg: + image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: + name: dynamo-router + version: 1.6.0.dev20261023 + multinode: true + disagg: false + srt-recipe-dir: minimaxm31flash/vllm/gb200-fp4/agentx + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: [1] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c1 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [4] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c4 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [8] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c8 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [16] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c16 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - dram-utilization: 1.0 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [32] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c32 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [40] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c40 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [48] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c48 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [56] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c56 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-disagg: image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 model: MiniMaxAI/MiniMax-M3.1-Flash diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index cd51b78cec..90307fa8ec 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11415,7 +11415,7 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4418 - config-keys: - - minimaxm31flash-fp4-gb200-vllm-agentic-dspark + - minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-agg - minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-disagg description: - "Use latest configs for MiniMax M3.1 AgentX submission" From 850e19bf62eda2d91b75c290084ec28edfe39ba4 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 27 Oct 2026 09:06:39 -0400 Subject: [PATCH 10/15] Use latest configs for MiniMax M3.1 AgentX submission --- .../vllm/gb200-fp4/agentx/agg-variants.yaml | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml index 9d37f4474e..5593d301d3 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml @@ -117,37 +117,41 @@ override_tp4_c32: agg: args: max-num-seqs: 64 - gpu-memory-utilization: 0.80 + gpu-memory-utilization: 0.78 + max-num-batched-tokens: 16384 kv-cache-metrics: true load-format: fastsafetensors - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":720000000000,"lazy_offload":true}}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":600000000000,"lazy_offload":true}}' override_tp4_c40: name: minimaxm31flash-gb200-agg-tp4-dram-c40 roles: agg: args: max-num-seqs: 64 - gpu-memory-utilization: 0.80 + gpu-memory-utilization: 0.78 + max-num-batched-tokens: 16384 kv-cache-metrics: true load-format: fastsafetensors - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":720000000000,"lazy_offload":true}}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":600000000000,"lazy_offload":true}}' override_tp4_c48: name: minimaxm31flash-gb200-agg-tp4-dram-c48 roles: agg: args: max-num-seqs: 64 - gpu-memory-utilization: 0.80 + gpu-memory-utilization: 0.78 + max-num-batched-tokens: 16384 kv-cache-metrics: true load-format: fastsafetensors - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":720000000000,"lazy_offload":true}}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":600000000000,"lazy_offload":true}}' override_tp4_c56: name: minimaxm31flash-gb200-agg-tp4-dram-c56 roles: agg: args: max-num-seqs: 96 - gpu-memory-utilization: 0.80 + gpu-memory-utilization: 0.78 + max-num-batched-tokens: 16384 kv-cache-metrics: true load-format: fastsafetensors - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":720000000000,"lazy_offload":true}}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":600000000000,"lazy_offload":true}}' From 85915e36e7ad7e5216a0a4864dadd21609364614 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 27 Oct 2026 09:28:08 -0400 Subject: [PATCH 11/15] Use latest configs for MiniMax M3.1 AgentX submission --- .../vllm/gb200-fp4/agentx/agg-variants.yaml | 34 +++---------------- inferencex-e2e/configs/nvidia-master.yaml | 14 -------- 2 files changed, 4 insertions(+), 44 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml index 5593d301d3..c09c1065b9 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml @@ -117,41 +117,15 @@ override_tp4_c32: agg: args: max-num-seqs: 64 - gpu-memory-utilization: 0.78 - max-num-batched-tokens: 16384 + gpu-memory-utilization: 0.82 kv-cache-metrics: true - load-format: fastsafetensors - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":600000000000,"lazy_offload":true}}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":360000000000,"lazy_offload":true}}' override_tp4_c40: name: minimaxm31flash-gb200-agg-tp4-dram-c40 roles: agg: args: max-num-seqs: 64 - gpu-memory-utilization: 0.78 - max-num-batched-tokens: 16384 + gpu-memory-utilization: 0.82 kv-cache-metrics: true - load-format: fastsafetensors - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":600000000000,"lazy_offload":true}}' -override_tp4_c48: - name: minimaxm31flash-gb200-agg-tp4-dram-c48 - roles: - agg: - args: - max-num-seqs: 64 - gpu-memory-utilization: 0.78 - max-num-batched-tokens: 16384 - kv-cache-metrics: true - load-format: fastsafetensors - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":600000000000,"lazy_offload":true}}' -override_tp4_c56: - name: minimaxm31flash-gb200-agg-tp4-dram-c56 - roles: - agg: - args: - max-num-seqs: 96 - gpu-memory-utilization: 0.78 - max-num-batched-tokens: 16384 - kv-cache-metrics: true - load-format: fastsafetensors - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":600000000000,"lazy_offload":true}}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":360000000000,"lazy_offload":true}}' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 6c9f01e6a7..ce64a96519 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -10500,20 +10500,6 @@ minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-agg: num-nodes: 1 srt-recipe: agg-variants.yaml:override_tp4_c40 worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: vllm-simple } - conc-list: [48] - num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp4_c48 - worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: vllm-simple } - conc-list: [56] - num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp4_c56 - worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-disagg: image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 From ad4d588063d9a19b92df2bd6b23754d89c169eae Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 27 Oct 2026 10:01:08 -0400 Subject: [PATCH 12/15] Leave HBM headroom on GB200 non-DRAM points --- .../minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml | 4 ++++ .../vllm/gb200-fp4/agentx/disagg-mooncake.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml index c09c1065b9..6599796c5b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml @@ -93,24 +93,28 @@ override_tp4_c1: agg: args: max-num-seqs: 2 + gpu-memory-utilization: 0.88 override_tp4_c4: name: minimaxm31flash-gb200-agg-tp4-c4 roles: agg: args: max-num-seqs: 8 + gpu-memory-utilization: 0.88 override_tp4_c8: name: minimaxm31flash-gb200-agg-tp4-c8 roles: agg: args: max-num-seqs: 16 + gpu-memory-utilization: 0.88 override_tp4_c16: name: minimaxm31flash-gb200-agg-tp4-c16 roles: agg: args: max-num-seqs: 32 + gpu-memory-utilization: 0.88 override_tp4_c32: name: minimaxm31flash-gb200-agg-tp4-dram-c32 roles: diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml index af87726f5a..ea2d1dc016 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml @@ -175,7 +175,7 @@ base: dyn-reasoning-parser: minimax_m3 speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' tensor-parallel-size: 1 - gpu-memory-utilization: 0.92 + gpu-memory-utilization: 0.88 max-num-seqs: 64 max-num-batched-tokens: 8192 watermark: 0.05 From 996cd9e750afd6329b2bb6aeaf3a7e9781f57711 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 27 Oct 2026 23:30:14 -0400 Subject: [PATCH 13/15] Lower GB200 disagg TTFT with shorter prefill chunks and a DEP16 decode point Co-Authored-By: Claude Opus 5.5 --- .../gb200-fp4/agentx/disagg-mooncake.yaml | 21 ++++++++++--------- inferencex-e2e/configs/nvidia-master.yaml | 12 +++++------ 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml index ea2d1dc016..96fb040a07 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml @@ -1,5 +1,8 @@ schema: 2 base: + observability: + tachometer: + enabled: false identity: frameworks: dynamo: 1.6.0.dev20261023 @@ -115,7 +118,7 @@ base: gpu-memory-utilization: 0.88 max-num-seqs: 64 max-num-batched-tokens: 16384 - long-prefill-token-threshold: 16384 + long-prefill-token-threshold: 4096 prefill-schedule-interval: 1 enable-prefix-caching: true compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' @@ -205,15 +208,13 @@ base: node: dedicated name: minimaxm31flash-gb200-disagg-mooncake setup_script: vllm-mooncake.sh -override_3p1d_c280: - name: minimaxm31flash-gb200-disagg-mooncake-3p1d-c280 -override_2p1d_c224: - name: minimaxm31flash-gb200-disagg-mooncake-2p1d-c224 +override_3p1d_c256: + name: minimaxm31flash-gb200-disagg-mooncake-3p1d-c256 +override_3p1d_dep16_c280: + name: minimaxm31flash-gb200-disagg-mooncake-3p1d-dep16-c280 roles: - prefill: - nodes: 2 - workers: 2 decode: + nodes: 4 + gpus: 16 args: - max-num-seqs: 48 - max-cudagraph-capture-size: 384 + data-parallel-size: 16 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index ce64a96519..62c1c6a3cb 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -10522,14 +10522,14 @@ minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-disagg: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } - conc-list: [224] - srt-recipe: disagg-mooncake.yaml:override_2p1d_c224 - prefill: { num-worker: 2, tp: 4, ep: 4, dp-attn: true } - decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } + conc-list: [280] + srt-recipe: disagg-mooncake.yaml:override_3p1d_dep16_c280 + prefill: { num-worker: 3, tp: 4, ep: 4, dp-attn: true } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: true } - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } - conc-list: [280] - srt-recipe: disagg-mooncake.yaml:override_3p1d_c280 + conc-list: [256] + srt-recipe: disagg-mooncake.yaml:override_3p1d_c256 prefill: { num-worker: 3, tp: 4, ep: 4, dp-attn: true } decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } From 261fa855b4d29e5ba312b21cc78facfd8ccda855 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 27 Oct 2026 23:30:40 -0400 Subject: [PATCH 14/15] Disable Tachometer on GB200 agg recipe --- .../minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml index 6599796c5b..bb68187af0 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml @@ -10,6 +10,9 @@ base: resources: gpu_type: gb200 gpus_per_node: 4 + observability: + tachometer: + enabled: false health_check: max_attempts: 1440 interval_seconds: 5 From da7cb501ad724422e9b3fefce04e59476b4e0b77 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Wed, 28 Oct 2026 01:11:22 -0400 Subject: [PATCH 15/15] Add a 4P DEP16 GB200 disagg point at conc 352 Co-Authored-By: Claude Opus 5.5 --- .../vllm/gb200-fp4/agentx/disagg-mooncake.yaml | 11 +++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml index 96fb040a07..91fc5461aa 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml @@ -218,3 +218,14 @@ override_3p1d_dep16_c280: gpus: 16 args: data-parallel-size: 16 +override_4p1d_dep16_c352: + name: minimaxm31flash-gb200-disagg-mooncake-4p1d-dep16-c352 + roles: + prefill: + nodes: 4 + workers: 4 + decode: + nodes: 4 + gpus: 16 + args: + data-parallel-size: 16 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 62c1c6a3cb..cb914260c4 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -10526,6 +10526,13 @@ minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-disagg: srt-recipe: disagg-mooncake.yaml:override_3p1d_dep16_c280 prefill: { num-worker: 3, tp: 4, ep: 4, dp-attn: true } decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: true } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } + conc-list: [352] + srt-recipe: disagg-mooncake.yaml:override_4p1d_dep16_c352 + prefill: { num-worker: 4, tp: 4, ep: 4, dp-attn: true } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: true } - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: 0.3.13.post1 }