diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml new file mode 100644 index 0000000000..c369e0b1af --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml @@ -0,0 +1,89 @@ +base: + schema: 2 + name: minimaxm31flash-fp4-b300-sglang-agentic-tp2 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # Cold loads of the 250 GB checkpoint plus JIT MSA kernel builds and graph capture outlast the default. + health_check: + max_attempts: 1440 + interval_seconds: 5 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + # Serve flags follow the cookbook's B300 cells at + # https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1. + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + trust-remote-code: true + reasoning-parser: minimax-m3 + tool-call-parser: minimax-m3 + # Seven-token DSpark, the bundled draft's dspark_block_size. Throughput runs + # switch to synthetic acceptance at the golden acceptance length. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 7 + speculative-use-block-verification: true + # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. + speculative-draft-kv-cache-dtype: fp8_e4m3 + cuda-graph-backend-prefill: breakable + tokenizer-worker-num: 8 + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + FMHA_SM100_ALLOW_JIT: '1' + benchmark: + env: + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +zip_override_tp2_hicache: + roles: + agg: + args: + tensor-parallel-size: 2 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 32768 + # 2x CONC (AgentX subagent fan-out); the decode graph batch matches it. + max-running-requests: [64, 72, 80] + cuda-graph-max-bs-decode: [64, 72, 80] + mem-fraction-static: 0.87 + enable-hierarchical-cache: true + hicache-size: 370 + hicache-write-policy: write_back + benchmark: + env: + CONC: ['32', '36', '40'] + KV_OFFLOADING: dram + +zip_override_tp2_hicache_lpm: + roles: + agg: + args: + tensor-parallel-size: 2 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 32768 + max-running-requests: [88, 92, 96] + cuda-graph-max-bs-decode: [88, 92, 96] + mem-fraction-static: 0.87 + enable-hierarchical-cache: true + hicache-size: 370 + hicache-write-policy: write_back + schedule-policy: lpm + benchmark: + env: + CONC: ['44', '46', '48'] + KV_OFFLOADING: dram diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml index 019c01651e..2e8aab2a96 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml @@ -33,6 +33,8 @@ base: speculative-algorithm: DSPARK speculative-dspark-block-size: 7 speculative-use-block-verification: true + # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. + speculative-draft-kv-cache-dtype: fp8_e4m3 cuda-graph-backend-prefill: breakable tokenizer-worker-num: 8 watchdog-timeout: 3600 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 5ae75951e6..721ff93853 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9690,6 +9690,9 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: - dram-utilization: 0.80 search-space: - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 64, 80, 96], srt-recipe: agentic.yaml } + - dram-utilization: 1.0 + search-space: + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 36, 40, 44, 46, 48], srt-recipe: agentic-tp2.yaml } # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 920eaaaf82..fb060e76d5 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11499,3 +11499,12 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4509 scenario-type: [agentic-coding] append-only: true + +- config-keys: + - minimaxm31flash-fp4-b300-sglang-agentic-dspark + description: + - "Add TP2 points with DRAM KV offload (L2 HiCache) at conc 32 to 48." + - "Store the DSpark draft KV cache in FP8 (speculative-draft-kv-cache-dtype fp8_e4m3) on every point instead of the bf16 default, matching the vLLM DSpark drafter; this frees about 10 GB of HBM per rank and about 35 GB of the HiCache host pool for target KV." + - "Rerun the whole curve so every point uses the FP8 draft KV cache." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4524 + scenario-type: [agentic-coding]