diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..bb68187af0 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/agg-variants.yaml @@ -0,0 +1,138 @@ +schema: 2 +base: + identity: + frameworks: + dynamo: 1.6.0.dev20261023 + dynamo: + install: true + source: + wheel: 1.6.0.dev20261023 + resources: + gpu_type: gb200 + gpus_per_node: 4 + observability: + tachometer: + enabled: false + health_check: + max_attempts: 1440 + interval_seconds: 5 + environment: + PYTHONHASHSEED: '0' + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: kv + router-session-affinity-ttl-secs: 14400 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + DYN_TCP_CONNECT_TIMEOUT: '120' + engine: + type: vllm + connector: null + roles: + agg: + kv_events: true + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + DYN_TCP_CONNECT_TIMEOUT: '120' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: '1024' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: '1024' + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: '2048' + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + language-model-only: true + trust-remote-code: true + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + # MSA sparse/index cache needs 128-token blocks. + block-size: 128 + # The checkpoint is QAT-trained for an NVFP4 KV cache. + kv-cache-dtype: nvfp4 + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + enable-prefix-caching: true + max-model-len: 1048576 + tensor-parallel-size: 4 + max-num-batched-tokens: 32768 + # Prefill graphs to 8192 tokens; longer capture lists cost KV memory. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' + # Golden AL 4.04. + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + sbatch_directives: + cpus-per-task: '72' + mem: '0' + srun_options: + container-remap-root: '' + benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + name: minimaxm31flash-gb200-agg +override_tp4_c1: + name: minimaxm31flash-gb200-agg-tp4-c1 + roles: + agg: + args: + max-num-seqs: 2 + gpu-memory-utilization: 0.88 +override_tp4_c4: + name: minimaxm31flash-gb200-agg-tp4-c4 + roles: + agg: + args: + max-num-seqs: 8 + gpu-memory-utilization: 0.88 +override_tp4_c8: + name: minimaxm31flash-gb200-agg-tp4-c8 + roles: + agg: + args: + max-num-seqs: 16 + gpu-memory-utilization: 0.88 +override_tp4_c16: + name: minimaxm31flash-gb200-agg-tp4-c16 + roles: + agg: + args: + max-num-seqs: 32 + gpu-memory-utilization: 0.88 +override_tp4_c32: + name: minimaxm31flash-gb200-agg-tp4-dram-c32 + roles: + agg: + args: + max-num-seqs: 64 + gpu-memory-utilization: 0.82 + kv-cache-metrics: true + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":360000000000,"lazy_offload":true}}' +override_tp4_c40: + name: minimaxm31flash-gb200-agg-tp4-dram-c40 + roles: + agg: + args: + max-num-seqs: 64 + gpu-memory-utilization: 0.82 + kv-cache-metrics: true + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":360000000000,"lazy_offload":true}}' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml new file mode 100644 index 0000000000..91fc5461aa --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb200-fp4/agentx/disagg-mooncake.yaml @@ -0,0 +1,231 @@ +schema: 2 +base: + observability: + tachometer: + enabled: false + identity: + frameworks: + dynamo: 1.6.0.dev20261023 + dynamo: + install: true + source: + wheel: 1.6.0.dev20261023 + resources: + gpu_type: gb200 + gpus_per_node: 4 + health_check: + max_attempts: 1440 + interval_seconds: 5 + environment: + PYTHONHASHSEED: '0' + NCCL_NET_GDR_C2C: '1' + NVSHMEM_REMOTE_TRANSPORT: none + NVSHMEM_ENABLE_NIC_PE_MAPPING: '1' + NVSHMEM_HCA_PE_MAPPING: mlx5_0:1:1,mlx5_1:1:1,mlx5_2:1:1,mlx5_3:1:1 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + preamble: bash /configs/vllm-mooncake.sh + args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + options: + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 150GB + local_buffer_size: 4GB + protocol: rdma + device_name: '@fabric.rdma-devices' + enable_offload: false + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: kv + router-session-affinity-ttl-secs: 14400 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + DYN_TCP_CONNECT_TIMEOUT: '120' + placement: + node: head + engine: + type: vllm + connector: null + dp_launch_mode: per_node + roles: + prefill: + kv_events: true + nodes: 3 + workers: 3 + gpus: 4 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + DYN_TCP_CONNECT_TIMEOUT: '120' + UCX_CUDA_IPC_ENABLE_MNNVL: 'n' + UCX_MEMTYPE_CACHE: 'n' + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_REG_WHOLE: 'n' + UCX_RNDV_PIPELINE_ERROR_HANDLING: 'y' + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + NIXL_UCX_SGL_ENABLE: '1' + NCCL_IB_HCA: '@fabric.rdma-devices' + VLLM_USE_V2_MODEL_RUNNER: '1' + UCX_NET_DEVICES: '@fabric.rdma-device-ports' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: '1024' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: '1024' + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: '2048' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '8' + MC_SLICE_SIZE: '1048576' + MC_MAX_MR_SIZE: '4294967296' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + language-model-only: true + trust-remote-code: true + enable-expert-parallel: true + kv-cache-dtype: nvfp4 + block-size: 128 + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + max-model-len: 1048576 + load-format: fastsafetensors + kernel-config: '{"moe_backend":"deep_gemm_mega_moe"}' + numa-bind: true + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + tensor-parallel-size: 1 + gpu-memory-utilization: 0.88 + max-num-seqs: 64 + max-num-batched-tokens: 16384 + long-prefill-token-threshold: 4096 + prefill-schedule-interval: 1 + enable-prefix-caching: true + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + max-cudagraph-capture-size: 2048 + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"kv_lease_duration":120}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}]}}' + data-parallel-size: 4 + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + DYN_TCP_CONNECT_TIMEOUT: '120' + UCX_CUDA_IPC_ENABLE_MNNVL: 'n' + UCX_MEMTYPE_CACHE: 'n' + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_REG_WHOLE: 'n' + UCX_RNDV_PIPELINE_ERROR_HANDLING: 'y' + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + NIXL_UCX_SGL_ENABLE: '1' + NCCL_IB_HCA: '@fabric.rdma-devices' + VLLM_USE_V2_MODEL_RUNNER: '1' + UCX_NET_DEVICES: '@fabric.rdma-device-ports' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: '1024' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: '1024' + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: '2048' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + MC_MAX_MR_SIZE: '4294967296' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + VLLM_MAX_ASYNC_KV_ADMITS_PER_STEP: '4' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + language-model-only: true + trust-remote-code: true + enable-expert-parallel: true + kv-cache-dtype: nvfp4 + block-size: 128 + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + max-model-len: 1048576 + load-format: fastsafetensors + kernel-config: '{"moe_backend":"deep_gemm_mega_moe"}' + numa-bind: true + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + tensor-parallel-size: 1 + gpu-memory-utilization: 0.88 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + watermark: 0.05 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + max-cudagraph-capture-size: 512 + numa-bind-nodes: + - 0 + - 0 + - 1 + - 1 + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"kv_lease_duration":120}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false,"save_decode_cache":false,"enable_lookup":false}}]}}' + data-parallel-size: 8 + enable-prefix-caching: true + data-parallel-rpc-port: 13345 + sbatch_directives: + cpus-per-task: '72' + mem: '0' + srun_options: + container-remap-root: '' + benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: '1.0' + placement: + node: dedicated + name: minimaxm31flash-gb200-disagg-mooncake + setup_script: vllm-mooncake.sh +override_3p1d_c256: + name: minimaxm31flash-gb200-disagg-mooncake-3p1d-c256 +override_3p1d_dep16_c280: + name: minimaxm31flash-gb200-disagg-mooncake-3p1d-dep16-c280 + roles: + decode: + nodes: 4 + gpus: 16 + args: + data-parallel-size: 16 +override_4p1d_dep16_c352: + name: minimaxm31flash-gb200-disagg-mooncake-4p1d-dep16-c352 + roles: + prefill: + nodes: 4 + workers: 4 + decode: + nodes: 4 + gpus: 16 + args: + data-parallel-size: 16 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 9b35361ab4..7b4482e29a 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -10664,3 +10664,100 @@ glm5.3-fp4-b300-sglang-agentic-mtp: - dram-utilization: 1.0 search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [22, 24, 25, 28], srt-recipe: agentic.yaml } + +minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-agg: + image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: + name: dynamo-router + version: 1.6.0.dev20261023 + multinode: true + disagg: false + srt-recipe-dir: minimaxm31flash/vllm/gb200-fp4/agentx + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: [1] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c1 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [4] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c4 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [8] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c8 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [16] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c16 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - dram-utilization: 1.0 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [32] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c32 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [40] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c40 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + +minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-disagg: + image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: + name: dynamo-router + version: 1.6.0.dev20261023 + kv-p2p-transfer: nixl + multinode: true + disagg: true + srt-recipe-dir: minimaxm31flash/vllm/gb200-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 1.0 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } + conc-list: [280] + srt-recipe: disagg-mooncake.yaml:override_3p1d_dep16_c280 + prefill: { num-worker: 3, tp: 4, ep: 4, dp-attn: true } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: true } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } + conc-list: [352] + srt-recipe: disagg-mooncake.yaml:override_4p1d_dep16_c352 + prefill: { num-worker: 4, tp: 4, ep: 4, dp-attn: true } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: true } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } + conc-list: [256] + srt-recipe: disagg-mooncake.yaml:override_3p1d_c256 + prefill: { num-worker: 3, tp: 4, ep: 4, dp-attn: true } + decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index f29d4ac640..476c6af6d4 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11534,3 +11534,14 @@ - "Update the DeepSeek-V4-Pro-0813 FP4 B200 AgentX vLLM image to vllm/vllm-openai:nightly-8e2beaa3bff595577a069d49d96bde9074294ec2 (digest-pinned). Keep TP8 c1, c6 and c14, move TP8 c4 to the lowconc key and remove TP8 c10 and c16. Retune c32 to TP4 x DP2 with deep_gemm_mega_moe and SimpleCPU offload, and switch DEP8 c64 to c192 from deep_gemm_amxf4_mega_moe to deep_gemm_mega_moe with a smaller per-rank KV cache at c96 to c192." - "Add the dsv4-fp4-b200-vllm-agentic-mtp-lowconc key (vllm/vllm-openai digest-pinned) with TP8 c4 and TP2 x DP4 c48 and c64." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4554 + +- config-keys: + - minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-agg + - minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-disagg + description: + - "Add MiniMax-M3.1-Flash (MiniMaxAI/MiniMax-M3.1-Flash, NVFP4 experts, NVFP4 KV and indexer caches) AgentX on GB200 with Dynamo and the arm64 vLLM image (minimax-m31-flash-arm64-cu134)." + - "DSpark with the built-in drafter, 7 draft tokens (golden AL 4.04) and an fp8 drafter KV cache on every point." + - "Agg: TP4 at conc 1-16, and TP4 with SimpleCPUOffloadConnector (lazy offload, 360 GB) at conc 32 and 40." + - "Disagg: 3 DEP4 prefill workers with a DEP8 decode at conc 256 or a DEP16 decode at conc 280, and 4 DEP4 prefill workers with a DEP16 decode at conc 352; DeepGEMM MegaMoE, NIXL transfer plus a shared MooncakeStore DRAM pool (150 GB per rank), Dynamo KV router." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4464 + scenario-type: [agentic-coding]