diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml index efae40964b..b245698534 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml @@ -138,7 +138,7 @@ override_tp4_hicache_c16: cuda-graph-max-bs-decode: 32 enable-hierarchical-cache: true hicache-ratio: 1.5 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -156,7 +156,7 @@ override_tp4_hicache_c18: cuda-graph-max-bs-decode: 36 enable-hierarchical-cache: true hicache-ratio: 1.5 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -174,7 +174,7 @@ override_tp4_hicache_c20: cuda-graph-max-bs-decode: 40 enable-hierarchical-cache: true hicache-ratio: 1.5 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -192,7 +192,7 @@ override_tp4_hicache_c22: cuda-graph-max-bs-decode: 44 enable-hierarchical-cache: true hicache-ratio: 1.5 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -210,7 +210,7 @@ override_tp4_hicache_c24: cuda-graph-max-bs-decode: 48 enable-hierarchical-cache: true hicache-ratio: 1.5 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -228,7 +228,7 @@ override_tp4_hicache_c28: cuda-graph-max-bs-decode: 56 enable-hierarchical-cache: true hicache-ratio: 1.5 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -247,7 +247,7 @@ override_tp4_hicache_c32: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 253 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -266,7 +266,7 @@ override_tp4_hicache_c36: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 253 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -285,7 +285,7 @@ override_tp4_hicache_c40: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 253 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -304,7 +304,7 @@ override_tp4_hicache_c48: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 253 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -323,7 +323,7 @@ override_tp4_hicache_c56: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 253 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -342,7 +342,7 @@ override_tp4_hicache_c64: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 374 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -361,7 +361,7 @@ override_tp4_hicache_c72: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 374 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -380,7 +380,7 @@ override_tp4_hicache_c80: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 374 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index ed18bec2d6..f56d9c9b03 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -238,7 +238,7 @@ qwen3.5-fp8-mi355x-sglang-mtp: # checkpoint leaves too little KV headroom there for the 256k-capped AgentX # corpus. qwen3.5-fp8-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260929 + image: lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261020 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 907bd21213..94ea0af14e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11489,3 +11489,14 @@ - "Update the MI355X DeepSeek-V4-Pro-0813 SGLang MoRI disaggregated AgentX serving image from lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261019 to lmsysorg/sglang-rocm:v0.5.22-rocm10-mi35x-20261026, the latest published MI35x ROCm 10 nightly (digest sha256:edabad83728ab04e3eae39f315eb16261eeb64dbdb051112bd22fc88560b60c4)." - "Preserve all eight concurrency points (1, 16, 32, 48, 192, 256, 384, 512), prefill/decode topology, DSpark settings, UMBP offload, MoRI transfer, and benchmark client image. The workload binder supplies the serving image to every selected recipe variant." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4571 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the MI355X Qwen3.5 FP8 AgentX image from lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260929 to lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261020 (amd64 digest sha256:3a75c91365cc4f4e1194bf5d64b92f2f1ef134d8b9045da165fba28a8dada464)." + - "Switch every HiCache override on this arm from hicache-write-policy write_through_selective to write_through. GPU-resident points and HiCache sizes stay the same." + - "At concurrency 80 the overall prefix cache hit rate is flat: 93.5% of prompt tokens on the previous image and policy (run 36743632401) against 94.0% here (run 39763900198). What moves is the tier split -- the HiCache host tier serves 49.1% of prompt tokens instead of 43.3%, and the GPU tier 44.9% instead of 50.2%. Per-GPU throughput goes from 59,233 to 60,839 tok/s (+2.7%), p90 end-to-end latency from 50.9 s to 47.9 s, and mean TTFT from 3.12 s to 2.68 s." + - "That concurrency 80 comparison is not a single-variable A/B: the 36743632401 baseline ran hicache-size 400 on lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260929, while this point runs hicache-size 374 on the v0.5.22 image, so the image, the write policy and the host pool size all differ." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4374