From d3678124601e7f22fd539e29a7b6c9b7a8dbf8ba Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 27 Oct 2026 21:16:30 -0700 Subject: [PATCH 1/2] Store DSpark draft KV in FP8 on the GB200 SGLang AgentX curve Set speculative-draft-kv-cache-dtype fp8_e4m3, matching the vLLM DSpark drafter, so the bf16 draft KV no longer takes HBM and HiCache host pool space from target KV. The whole curve reruns. Co-Authored-By: Claude Opus 5.5 --- .../minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml | 2 ++ inferencex-e2e/perf-changelog.yaml | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml index 8f4fe1d22f..0c63ca368d 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb200-fp4-mtp/agentic.yaml @@ -33,6 +33,8 @@ base: speculative-algorithm: DSPARK speculative-dspark-block-size: 7 speculative-use-block-verification: true + # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. + speculative-draft-kv-cache-dtype: fp8_e4m3 cuda-graph-backend-prefill: breakable tokenizer-worker-num: 8 watchdog-timeout: 3600 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 907bd21213..3cc6949ec8 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11489,3 +11489,11 @@ - "Update the MI355X DeepSeek-V4-Pro-0813 SGLang MoRI disaggregated AgentX serving image from lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261019 to lmsysorg/sglang-rocm:v0.5.22-rocm10-mi35x-20261026, the latest published MI35x ROCm 10 nightly (digest sha256:edabad83728ab04e3eae39f315eb16261eeb64dbdb051112bd22fc88560b60c4)." - "Preserve all eight concurrency points (1, 16, 32, 48, 192, 256, 384, 512), prefill/decode topology, DSpark settings, UMBP offload, MoRI transfer, and benchmark client image. The workload binder supplies the serving image to every selected recipe variant." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4571 + +- config-keys: + - minimaxm31flash-fp4-gb200-sglang-agentic-dspark + description: + - "Store the DSpark draft KV cache in FP8 (speculative-draft-kv-cache-dtype fp8_e4m3) instead of the bf16 default, matching the vLLM DSpark drafter; this frees HBM and HiCache host pool space for target KV." + - "Rerun the whole curve so every point uses the FP8 draft KV cache." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + scenario-type: [agentic-coding] From 84ee06a118a39a384e86a28fbb6769fccd169383 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 27 Oct 2026 21:17:38 -0700 Subject: [PATCH 2/2] Set the changelog PR link Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3cc6949ec8..8a6d12b9e7 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11495,5 +11495,5 @@ description: - "Store the DSpark draft KV cache in FP8 (speculative-draft-kv-cache-dtype fp8_e4m3) instead of the bf16 default, matching the vLLM DSpark drafter; this frees HBM and HiCache host pool space for target KV." - "Rerun the whole curve so every point uses the FP8 draft KV cache." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4599 scenario-type: [agentic-coding]