From 2e12ce08a366dae892eea289889edc9245f9c167 Mon Sep 17 00:00:00 2001 From: ispobock Date: Mon, 26 Oct 2026 09:02:09 +0800 Subject: [PATCH 1/8] Add TP2 DRAM-offload points to MiniMax-M3.1-Flash B300 SGLang AgentX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a TP2 config key with L2 HiCache at conc 32 to 48; conc 44 to 48 use longest-prefix-match scheduling. 为 MiniMax-M3.1-Flash B300 SGLang AgentX 新增 TP2 + L2 HiCache 配置，覆盖 conc 32 到 48，其中 conc 44 到 48 使用 longest-prefix-match 调度。 --- .../sglang/b300-fp4-mtp/agentic-tp2.yaml | 91 +++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 16 ++++ inferencex-e2e/perf-changelog.yaml | 7 ++ 3 files changed, 114 insertions(+) create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml new file mode 100644 index 0000000000..f27b0af0bb --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml @@ -0,0 +1,91 @@ +base: + schema: 2 + name: minimaxm31flash-fp4-b300-sglang-agentic-tp2 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + # Cold loads of the 250 GB checkpoint plus JIT MSA kernel builds and graph capture outlast the default. + health_check: + max_attempts: 1440 + interval_seconds: 5 + roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + # Serve flags follow the cookbook's B300 cells at + # https://docs.sglang.io/cookbook/autoregressive/MiniMax/MiniMax-M3.1. + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + trust-remote-code: true + reasoning-parser: minimax-m3 + tool-call-parser: minimax-m3 + # Seven-token DSpark, the bundled draft's dspark_block_size. Throughput runs + # switch to synthetic acceptance at the golden acceptance length. + speculative-algorithm: DSPARK + speculative-dspark-block-size: 7 + speculative-use-block-verification: true + cuda-graph-backend-prefill: breakable + tokenizer-worker-num: 8 + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + FMHA_SM100_ALLOW_JIT: '1' + benchmark: + env: + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# TP2 with L2 HiCache (write_back, 295 GB per rank). The scheduler batch is 2x CONC; +# the decode graph batch matches it. + +zip_override_tp2_hicache: + roles: + agg: + args: + tensor-parallel-size: 2 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 32768 + max-running-requests: [64, 72, 80] + cuda-graph-max-bs-decode: [64, 72, 80] + mem-fraction-static: 0.87 + enable-hierarchical-cache: true + hicache-size: 295 + hicache-write-policy: write_back + benchmark: + env: + CONC: ['32', '36', '40'] + KV_OFFLOADING: dram + +# Higher-concurrency points use longest-prefix-match scheduling. + +zip_override_tp2_hicache_lpm: + roles: + agg: + args: + tensor-parallel-size: 2 + moe-runner-backend: flashinfer_trtllm_routed + chunked-prefill-size: 32768 + max-running-requests: [88, 92, 96] + cuda-graph-max-bs-decode: [88, 92, 96] + mem-fraction-static: 0.87 + enable-hierarchical-cache: true + hicache-size: 295 + hicache-write-policy: write_back + schedule-policy: lpm + benchmark: + env: + CONC: ['44', '46', '48'] + KV_OFFLOADING: dram diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 16118ca8b9..7a4c7b1ecf 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9468,6 +9468,22 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: # not full DP attention, so dp-attn stays false). - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [64, 128, 256], srt-recipe: agentic.yaml } +minimaxm31flash-fp4-b300-sglang-agentic-dspark-tp2: + image: lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87 + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:b300-dsxe + precision: fp4 + framework: sglang + multinode: false + srt-recipe-dir: minimaxm31flash/sglang/b300-fp4-mtp + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + # TP2 with L2 HiCache: 295 GB per rank at conc 32-48. + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 36, 40, 44, 46, 48], srt-recipe: agentic-tp2.yaml } + # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: image: lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4@sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 9524a19f4c..8314829dba 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11197,3 +11197,10 @@ - "Balanced TP4 at conc 64 with L2 HiCache (write_back, hicache-size 180 GB per rank), and the scheduler and decode graph batch at 128." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4499 scenario-type: [agentic-coding] + +- config-keys: + - minimaxm31flash-fp4-b300-sglang-agentic-dspark-tp2 + description: + - "Add TP2 points with DRAM KV offload (L2 HiCache)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + scenario-type: [agentic-coding] From c1f07585d3b75792f0e8c3afaf58c212035ba7b8 Mon Sep 17 00:00:00 2001 From: ispobock Date: Mon, 26 Oct 2026 16:53:54 +0800 Subject: [PATCH 2/8] Add the points to the existing curve with append-only --- inferencex-e2e/configs/nvidia-master.yaml | 14 -------------- inferencex-e2e/perf-changelog.yaml | 5 +++-- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index ca32da8ebd..162efee312 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9468,20 +9468,6 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: - dram-utilization: 0.80 search-space: - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 64, 80, 96], srt-recipe: agentic.yaml } - -minimaxm31flash-fp4-b300-sglang-agentic-dspark-tp2: - image: lmsysorg/sglang:dev-minimax-m31-flash@sha256:614b1509bcb729c653528d448bff9850f8977fcad9d12ba56bd6dbe3d7985e87 - model: MiniMaxAI/MiniMax-M3.1-Flash - model-prefix: minimaxm31flash - runner: cluster:b300-dsxe - precision: fp4 - framework: sglang - multinode: false - srt-recipe-dir: minimaxm31flash/sglang/b300-fp4-mtp - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: # TP2 with L2 HiCache: 295 GB per rank at conc 32-48. - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 36, 40, 44, 46, 48], srt-recipe: agentic-tp2.yaml } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3f781ac7b2..0f609de65d 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11212,8 +11212,9 @@ scenario-type: [agentic-coding] - config-keys: - - minimaxm31flash-fp4-b300-sglang-agentic-dspark-tp2 + - minimaxm31flash-fp4-b300-sglang-agentic-dspark description: - - "Add TP2 points with DRAM KV offload (L2 HiCache)." + - "Add TP2 points with DRAM KV offload (L2 HiCache) at conc 32 to 48 to the existing curve." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4524 scenario-type: [agentic-coding] + append-only: true From 0c836a16ae698bd4e9e170ab8c64f7b6d14db238 Mon Sep 17 00:00:00 2001 From: ispobock Date: Mon, 26 Oct 2026 16:57:34 +0800 Subject: [PATCH 3/8] drop restating comments --- .../minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml | 6 +----- inferencex-e2e/configs/nvidia-master.yaml | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml index f27b0af0bb..82b2178cb9 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml @@ -48,9 +48,6 @@ base: AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' -# TP2 with L2 HiCache (write_back, 295 GB per rank). The scheduler batch is 2x CONC; -# the decode graph batch matches it. - zip_override_tp2_hicache: roles: agg: @@ -58,6 +55,7 @@ zip_override_tp2_hicache: tensor-parallel-size: 2 moe-runner-backend: flashinfer_trtllm_routed chunked-prefill-size: 32768 + # 2x CONC (AgentX subagent fan-out); the decode graph batch matches it. max-running-requests: [64, 72, 80] cuda-graph-max-bs-decode: [64, 72, 80] mem-fraction-static: 0.87 @@ -69,8 +67,6 @@ zip_override_tp2_hicache: CONC: ['32', '36', '40'] KV_OFFLOADING: dram -# Higher-concurrency points use longest-prefix-match scheduling. - zip_override_tp2_hicache_lpm: roles: agg: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 162efee312..0daf6069bf 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9468,7 +9468,6 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: - dram-utilization: 0.80 search-space: - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 64, 80, 96], srt-recipe: agentic.yaml } - # TP2 with L2 HiCache: 295 GB per rank at conc 32-48. - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 36, 40, 44, 46, 48], srt-recipe: agentic-tp2.yaml } # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. From 51faead3c51cda1d1d420a40239dff638475c635 Mon Sep 17 00:00:00 2001 From: ispobock Date: Mon, 26 Oct 2026 21:34:09 +0800 Subject: [PATCH 4/8] lower hicache to 210 --- .../minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml index 82b2178cb9..7b04b095d1 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml @@ -60,7 +60,7 @@ zip_override_tp2_hicache: cuda-graph-max-bs-decode: [64, 72, 80] mem-fraction-static: 0.87 enable-hierarchical-cache: true - hicache-size: 295 + hicache-size: 210 hicache-write-policy: write_back benchmark: env: @@ -78,7 +78,7 @@ zip_override_tp2_hicache_lpm: cuda-graph-max-bs-decode: [88, 92, 96] mem-fraction-static: 0.87 enable-hierarchical-cache: true - hicache-size: 295 + hicache-size: 210 hicache-write-policy: write_back schedule-policy: lpm benchmark: From dafe59e8820f09e55197b1d87477f90bf1285964 Mon Sep 17 00:00:00 2001 From: ispobock Date: Mon, 26 Oct 2026 22:58:21 +0800 Subject: [PATCH 5/8] raise tp2 hicache to 370 --- .../minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml | 4 ++-- inferencex-e2e/configs/nvidia-master.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml index 7b04b095d1..6809d06301 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml @@ -60,7 +60,7 @@ zip_override_tp2_hicache: cuda-graph-max-bs-decode: [64, 72, 80] mem-fraction-static: 0.87 enable-hierarchical-cache: true - hicache-size: 210 + hicache-size: 370 hicache-write-policy: write_back benchmark: env: @@ -78,7 +78,7 @@ zip_override_tp2_hicache_lpm: cuda-graph-max-bs-decode: [88, 92, 96] mem-fraction-static: 0.87 enable-hierarchical-cache: true - hicache-size: 210 + hicache-size: 370 hicache-write-policy: write_back schedule-policy: lpm benchmark: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index bb6d025917..10b15fb3c9 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9588,6 +9588,8 @@ minimaxm31flash-fp4-b300-sglang-agentic-dspark: - dram-utilization: 0.80 search-space: - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 64, 80, 96], srt-recipe: agentic.yaml } + - dram-utilization: 1.0 + search-space: - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 36, 40, 44, 46, 48], srt-recipe: agentic-tp2.yaml } # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. From 5f2ebd5841199a8c396c2a86fa0b7ea69fa86717 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 27 Oct 2026 18:47:23 -0700 Subject: [PATCH 6/8] Store DSpark draft KV in FP8 on the B300 SGLang AgentX curve Set speculative-draft-kv-cache-dtype fp8_e4m3 on every point, matching the vLLM DSpark drafter. The bf16 draft KV took 21 GB of HBM and 71 GB of the HiCache host pool per rank at TP2. Drop append-only so the whole curve reruns. Co-Authored-By: Claude Opus 5.5 --- .../minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml | 2 ++ .../minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml | 2 ++ inferencex-e2e/perf-changelog.yaml | 5 +++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml index 6809d06301..c369e0b1af 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml @@ -33,6 +33,8 @@ base: speculative-algorithm: DSPARK speculative-dspark-block-size: 7 speculative-use-block-verification: true + # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. + speculative-draft-kv-cache-dtype: fp8_e4m3 cuda-graph-backend-prefill: breakable tokenizer-worker-num: 8 watchdog-timeout: 3600 diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml index 019c01651e..2e8aab2a96 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml @@ -33,6 +33,8 @@ base: speculative-algorithm: DSPARK speculative-dspark-block-size: 7 speculative-use-block-verification: true + # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. + speculative-draft-kv-cache-dtype: fp8_e4m3 cuda-graph-backend-prefill: breakable tokenizer-worker-num: 8 watchdog-timeout: 3600 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 53ab0337a3..b9d4feb2c1 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11475,7 +11475,8 @@ - config-keys: - minimaxm31flash-fp4-b300-sglang-agentic-dspark description: - - "Add TP2 points with DRAM KV offload (L2 HiCache) at conc 32 to 48 to the existing curve." + - "Add TP2 points with DRAM KV offload (L2 HiCache) at conc 32 to 48." + - "Store the DSpark draft KV cache in FP8 (speculative-draft-kv-cache-dtype fp8_e4m3) on every point instead of the bf16 default, matching the vLLM DSpark drafter; this frees about 10 GB of HBM per rank and about 35 GB of the HiCache host pool for target KV." + - "Rerun the whole curve so every point uses the FP8 draft KV cache." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4524 scenario-type: [agentic-coding] - append-only: true From bbe13506e1829d58d8794c4621f8739f1c6febe4 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 27 Oct 2026 21:27:17 -0700 Subject: [PATCH 7/8] Revert the FP8 DSpark draft KV change The draft-only speculative-draft-kv-cache-dtype override lowers draft KV below the pinned image's bf16 default, which the draft-precision rule in CONTRIBUTING.md forbids. Restore the append-only TP2 addition. Co-Authored-By: Claude Opus 5.5 --- .../minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml | 2 -- .../minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml | 2 -- inferencex-e2e/perf-changelog.yaml | 5 ++--- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml index c369e0b1af..6809d06301 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml @@ -33,8 +33,6 @@ base: speculative-algorithm: DSPARK speculative-dspark-block-size: 7 speculative-use-block-verification: true - # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. - speculative-draft-kv-cache-dtype: fp8_e4m3 cuda-graph-backend-prefill: breakable tokenizer-worker-num: 8 watchdog-timeout: 3600 diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml index 2e8aab2a96..019c01651e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml @@ -33,8 +33,6 @@ base: speculative-algorithm: DSPARK speculative-dspark-block-size: 7 speculative-use-block-verification: true - # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. - speculative-draft-kv-cache-dtype: fp8_e4m3 cuda-graph-backend-prefill: breakable tokenizer-worker-num: 8 watchdog-timeout: 3600 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index b9d4feb2c1..53ab0337a3 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11475,8 +11475,7 @@ - config-keys: - minimaxm31flash-fp4-b300-sglang-agentic-dspark description: - - "Add TP2 points with DRAM KV offload (L2 HiCache) at conc 32 to 48." - - "Store the DSpark draft KV cache in FP8 (speculative-draft-kv-cache-dtype fp8_e4m3) on every point instead of the bf16 default, matching the vLLM DSpark drafter; this frees about 10 GB of HBM per rank and about 35 GB of the HiCache host pool for target KV." - - "Rerun the whole curve so every point uses the FP8 draft KV cache." + - "Add TP2 points with DRAM KV offload (L2 HiCache) at conc 32 to 48 to the existing curve." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4524 scenario-type: [agentic-coding] + append-only: true From 75a48a5078b1dfd0d85fe5a57abcb799750da3e1 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 27 Oct 2026 21:48:05 -0700 Subject: [PATCH 8/8] Restore FP8 DSpark draft KV and rerun the whole curve Reviewed and approved: store the DSpark draft KV in FP8 on every point and rerun the full B300 SGLang AgentX curve. Co-Authored-By: Claude Opus 5.5 --- .../minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml | 2 ++ .../minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml | 2 ++ inferencex-e2e/perf-changelog.yaml | 5 +++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml index 6809d06301..c369e0b1af 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic-tp2.yaml @@ -33,6 +33,8 @@ base: speculative-algorithm: DSPARK speculative-dspark-block-size: 7 speculative-use-block-verification: true + # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. + speculative-draft-kv-cache-dtype: fp8_e4m3 cuda-graph-backend-prefill: breakable tokenizer-worker-num: 8 watchdog-timeout: 3600 diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml index 019c01651e..2e8aab2a96 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/b300-fp4-mtp/agentic.yaml @@ -33,6 +33,8 @@ base: speculative-algorithm: DSPARK speculative-dspark-block-size: 7 speculative-use-block-verification: true + # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. + speculative-draft-kv-cache-dtype: fp8_e4m3 cuda-graph-backend-prefill: breakable tokenizer-worker-num: 8 watchdog-timeout: 3600 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 5d552f4f3a..c3cc309863 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11493,7 +11493,8 @@ - config-keys: - minimaxm31flash-fp4-b300-sglang-agentic-dspark description: - - "Add TP2 points with DRAM KV offload (L2 HiCache) at conc 32 to 48 to the existing curve." + - "Add TP2 points with DRAM KV offload (L2 HiCache) at conc 32 to 48." + - "Store the DSpark draft KV cache in FP8 (speculative-draft-kv-cache-dtype fp8_e4m3) on every point instead of the bf16 default, matching the vLLM DSpark drafter; this frees about 10 GB of HBM per rank and about 35 GB of the HiCache host pool for target KV." + - "Rerun the whole curve so every point uses the FP8 draft KV cache." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4524 scenario-type: [agentic-coding] - append-only: true