diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-tp2.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-tp2.yaml index ab1788f364..300703a0e5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-tp2.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-tp2.yaml @@ -33,6 +33,8 @@ base: speculative-algorithm: DSPARK speculative-dspark-block-size: 7 speculative-use-block-verification: true + # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. + speculative-draft-kv-cache-dtype: fp8_e4m3 cuda-graph-backend-prefill: breakable tokenizer-worker-num: 8 watchdog-timeout: 3600 diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml index ebad930fc4..eec61fb43b 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -33,6 +33,8 @@ base: speculative-algorithm: DSPARK speculative-dspark-block-size: 7 speculative-use-block-verification: true + # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. + speculative-draft-kv-cache-dtype: fp8_e4m3 cuda-graph-backend-prefill: breakable tokenizer-worker-num: 8 watchdog-timeout: 3600 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 920eaaaf82..c4f05e64fc 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11499,3 +11499,11 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4509 scenario-type: [agentic-coding] append-only: true + +- config-keys: + - minimaxm31flash-fp4-gb300-sglang-agentic-dspark + description: + - "Store the DSpark draft KV cache in FP8 (speculative-draft-kv-cache-dtype fp8_e4m3) on every point except the batch-size-1 MonoKernel Low-Latency point, instead of the bf16 default, matching the vLLM DSpark drafter; this frees HBM and HiCache host pool space for target KV." + - "Rerun the whole curve so it is measured on one recipe revision." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4600 + scenario-type: [agentic-coding]