From b54115a2d136e4fc7f0bf5bc2a71adeea91c92c5 Mon Sep 17 00:00:00 2001 From: "Eric.Chin.AMD" Date: Wed, 28 Oct 2026 16:04:32 +0000 Subject: [PATCH] perf(amd): cap GLM-5.2 FP4 MI355X SGLang TP8 c4 admission at 2 At TP8 concurrency 4 the slowest decodes are the ones that share a decode batch, so admission above the in-flight tail only widens the batch that tail runs in. Drop max-running-requests from 8 to 2 and keep the decode graph batch at 8; requests over the cap wait for a slot. Every other point keeps 2x CONC. --- .../glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml | 7 +++++-- inferencex-e2e/perf-changelog.yaml | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml index 5e3e101597..49224d2ece 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/mi355x-fp4-mtp/agentic.yaml @@ -49,7 +49,10 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' # One variant per point. Admission is 2x CONC; the decode graph batch follows it -# up to 64. The TP4 HiCache tier is 180 GB per rank. +# up to 64. The TP4 HiCache tier is 180 GB per rank. TP8 c4 is the exception: +# its slowest decodes are the ones sharing a batch, so admission is capped +# below CONC to trade TTFT for p90 interactivity; the graph batch stays at +# 2x CONC. override_tp4_c4_hicache: roles: agg: @@ -187,7 +190,7 @@ override_tp8_c4: args: tensor-parallel-size: 8 expert-parallel-size: 1 - max-running-requests: 8 + max-running-requests: 2 cuda-graph-max-bs-decode: 8 benchmark: env: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 907bd21213..3938a9c910 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11489,3 +11489,13 @@ - "Update the MI355X DeepSeek-V4-Pro-0813 SGLang MoRI disaggregated AgentX serving image from lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261019 to lmsysorg/sglang-rocm:v0.5.22-rocm10-mi35x-20261026, the latest published MI35x ROCm 10 nightly (digest sha256:edabad83728ab04e3eae39f315eb16261eeb64dbdb051112bd22fc88560b60c4)." - "Preserve all eight concurrency points (1, 16, 32, 48, 192, 256, 384, 512), prefill/decode topology, DSpark settings, UMBP offload, MoRI transfer, and benchmark client image. The workload binder supplies the serving image to every selected recipe variant." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4571 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "TP8 concurrency 4 drops max-running-requests from 8 to 2 and keeps cuda-graph-max-bs-decode at 8. No other point moves: TP8 concurrency 1 and 2 and every TP4 HiCache point keep admission at 2x CONC." + - "At this point the slowest decodes are the ones that share a decode batch, so admission above the in-flight tail only widens the batch that tail runs in. A max-running-requests sweep at TP8 concurrency 4 measured interactivity at p90 TPOT 222.3 to 255.0 output tokens/s/user and the slowest decile's mean concurrent decode partners 1.68 to 0.71, against TTFT p90 1.07 to 1.22 s and 3911 to 3754 tokens/s/GPU; requests over the cap wait for a slot." + - "The image, EAGLE shape (5 steps, 6 draft tokens), chunked-prefill-size, mem-fraction-static, HiCache tier and the DSA triton prefill and decode backends are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX