diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml index 5e3e101597..49224d2ece 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml @@ -49,7 +49,10 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' # One variant per point. Admission is 2x CONC; the decode graph batch follows it -# up to 64. The TP4 HiCache tier is 180 GB per rank. +# up to 64. The TP4 HiCache tier is 180 GB per rank. TP8 c4 is the exception: +# its slowest decodes are the ones sharing a batch, so admission is capped +# below CONC to trade TTFT for p90 interactivity; the graph batch stays at +# 2x CONC. override_tp4_c4_hicache: roles: agg: @@ -187,7 +190,7 @@ override_tp8_c4: args: tensor-parallel-size: 8 expert-parallel-size: 1 - max-running-requests: 8 + max-running-requests: 2 cuda-graph-max-bs-decode: 8 benchmark: env: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 18e8e72c5c..6ff163fb54 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11499,6 +11499,16 @@ - "Preserve all eight concurrency points (1, 16, 32, 48, 192, 256, 384, 512), prefill/decode topology, DSpark settings, UMBP offload, MoRI transfer, and benchmark client image. The workload binder supplies the serving image to every selected recipe variant." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4571 +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "TP8 concurrency 4 drops max-running-requests from 8 to 2 and keeps cuda-graph-max-bs-decode at 8. No other point moves: TP8 concurrency 1 and 2 and every TP4 HiCache point keep admission at 2x CONC." + - "At this point the slowest decodes are the ones that share a decode batch, so admission above the in-flight tail only widens the batch that tail runs in. A max-running-requests sweep at TP8 concurrency 4 measured interactivity at p90 TPOT 222.3 to 255.0 output tokens/s/user and the slowest decile's mean concurrent decode partners 1.68 to 0.71, against TTFT p90 1.07 to 1.22 s and 3911 to 3754 tokens/s/GPU; requests over the cap wait for a slot." + - "The image, EAGLE shape (5 steps, 6 draft tokens), chunked-prefill-size, mem-fraction-static, HiCache tier and the DSA triton prefill and decode backends are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4630 + - config-keys: - minimaxm31flash-fp4-b200-sglang-agentic-dspark - minimaxm31flash-fp4-gb200-sglang-agentic-dspark