From d0fb7a4c490c85bfce728d5fea06deed2a175536 Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Wed, 21 Oct 2026 16:41:01 +0000 Subject: [PATCH 1/4] [AMD][Qwen3.5] [Perf] Recover MI355X FP8 AgentX concurrency 80 with HiCache write_through on rocm/sgl-dev v0.5.22 --- .../sglang/mi355x-fp8-mtp/agentic.yaml | 28 +++++++++---------- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 9 ++++++ 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml index efae40964b..b245698534 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp8-mtp/agentic.yaml @@ -138,7 +138,7 @@ override_tp4_hicache_c16: cuda-graph-max-bs-decode: 32 enable-hierarchical-cache: true hicache-ratio: 1.5 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -156,7 +156,7 @@ override_tp4_hicache_c18: cuda-graph-max-bs-decode: 36 enable-hierarchical-cache: true hicache-ratio: 1.5 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -174,7 +174,7 @@ override_tp4_hicache_c20: cuda-graph-max-bs-decode: 40 enable-hierarchical-cache: true hicache-ratio: 1.5 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -192,7 +192,7 @@ override_tp4_hicache_c22: cuda-graph-max-bs-decode: 44 enable-hierarchical-cache: true hicache-ratio: 1.5 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -210,7 +210,7 @@ override_tp4_hicache_c24: cuda-graph-max-bs-decode: 48 enable-hierarchical-cache: true hicache-ratio: 1.5 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -228,7 +228,7 @@ override_tp4_hicache_c28: cuda-graph-max-bs-decode: 56 enable-hierarchical-cache: true hicache-ratio: 1.5 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -247,7 +247,7 @@ override_tp4_hicache_c32: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 253 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -266,7 +266,7 @@ override_tp4_hicache_c36: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 253 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -285,7 +285,7 @@ override_tp4_hicache_c40: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 253 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -304,7 +304,7 @@ override_tp4_hicache_c48: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 253 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -323,7 +323,7 @@ override_tp4_hicache_c56: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 253 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -342,7 +342,7 @@ override_tp4_hicache_c64: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 374 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -361,7 +361,7 @@ override_tp4_hicache_c72: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 374 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: @@ -380,7 +380,7 @@ override_tp4_hicache_c80: enable-hierarchical-cache: true hicache-ratio: 1.5 hicache-size: 374 - hicache-write-policy: write_through_selective + hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first benchmark: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index ce2306b905..54c9697ca3 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -238,7 +238,7 @@ qwen3.5-fp8-mi355x-sglang-mtp: # checkpoint leaves too little KV headroom there for the 256k-capped AgentX # corpus. qwen3.5-fp8-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260929 + image: rocm/sgl-dev:v0.5.22-rocm720-mi35x-20261021 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index b58497f003..84b77a4cad 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10491,3 +10491,12 @@ description: - 'Update GB300 Kimi-K3 AgentX coverage with low-latency aggregated runs and DCP8 disaggregation: 1P2D, 1P3D, 1P4D, 2P3D, and 2P5D.' pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4047 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the MI355X Qwen3.5 FP8 AgentX image from lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260929 to rocm/sgl-dev:v0.5.22-rocm720-mi35x-20261021 (amd64 digest sha256:1f2de36c4f019cbd380fb87bade0cf525b41a08b466e797ad3e342a0130a9614)." + - "Switch every HiCache override on this arm from hicache-write-policy write_through_selective to write_through. At concurrency 80 the cache hit rate moves from 51% to about 95%. GPU-resident points and HiCache sizes stay the same." + pr-link: PLACEHOLDER From a419122ffffdb544411f82b7a51038e28fcbe7c7 Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Wed, 21 Oct 2026 11:49:10 -0500 Subject: [PATCH 2/4] [AMD][Qwen3.5] Pin the MI355X FP8 AgentX image to lmsysorg/sglang-rocm v0.5.22 --- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 54c9697ca3..308b74a96d 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -238,7 +238,7 @@ qwen3.5-fp8-mi355x-sglang-mtp: # checkpoint leaves too little KV headroom there for the 256k-capped AgentX # corpus. qwen3.5-fp8-mi355x-sglang-agentic-mtp: - image: rocm/sgl-dev:v0.5.22-rocm720-mi35x-20261021 + image: lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261021 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 84b77a4cad..d767347d3e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10497,6 +10497,6 @@ scenario-type: - agentic-coding description: - - "Update the MI355X Qwen3.5 FP8 AgentX image from lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260929 to rocm/sgl-dev:v0.5.22-rocm720-mi35x-20261021 (amd64 digest sha256:1f2de36c4f019cbd380fb87bade0cf525b41a08b466e797ad3e342a0130a9614)." + - "Update the MI355X Qwen3.5 FP8 AgentX image from lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260929 to lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261021 (amd64 digest sha256:1f2de36c4f019cbd380fb87bade0cf525b41a08b466e797ad3e342a0130a9614)." - "Switch every HiCache override on this arm from hicache-write-policy write_through_selective to write_through. At concurrency 80 the cache hit rate moves from 51% to about 95%. GPU-resident points and HiCache sizes stay the same." pr-link: PLACEHOLDER From 165e56c6fc50316b5f44e4f6bbe45db304486c15 Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Thu, 22 Oct 2026 09:47:47 +0000 Subject: [PATCH 3/4] [AMD][Qwen3.5] Pin MI355X FP8 AgentX to lmsysorg/sglang-rocm v0.5.22-20261020 --- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 308b74a96d..faf9112885 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -238,7 +238,7 @@ qwen3.5-fp8-mi355x-sglang-mtp: # checkpoint leaves too little KV headroom there for the 256k-capped AgentX # corpus. qwen3.5-fp8-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261021 + image: lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261020 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index d767347d3e..15b8dcc406 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10497,6 +10497,6 @@ scenario-type: - agentic-coding description: - - "Update the MI355X Qwen3.5 FP8 AgentX image from lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260929 to lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261021 (amd64 digest sha256:1f2de36c4f019cbd380fb87bade0cf525b41a08b466e797ad3e342a0130a9614)." + - "Update the MI355X Qwen3.5 FP8 AgentX image from lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260929 to lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261020 (amd64 digest sha256:3a75c91365cc4f4e1194bf5d64b92f2f1ef134d8b9045da165fba28a8dada464)." - "Switch every HiCache override on this arm from hicache-write-policy write_through_selective to write_through. At concurrency 80 the cache hit rate moves from 51% to about 95%. GPU-resident points and HiCache sizes stay the same." - pr-link: PLACEHOLDER + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4374 From 786c796a33ead6a6e83022093ec50584bc9deb6d Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Wed, 28 Oct 2026 07:39:16 +0000 Subject: [PATCH 4/4] [AMD][Qwen3.5] Correct the FP8 AgentX changelog cache-hit claim at concurrency 80 --- inferencex-e2e/perf-changelog.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index ca5e35f544..94ea0af14e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11496,5 +11496,7 @@ - agentic-coding description: - "Update the MI355X Qwen3.5 FP8 AgentX image from lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260929 to lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261020 (amd64 digest sha256:3a75c91365cc4f4e1194bf5d64b92f2f1ef134d8b9045da165fba28a8dada464)." - - "Switch every HiCache override on this arm from hicache-write-policy write_through_selective to write_through. At concurrency 80 the cache hit rate moves from 51% to about 95%. GPU-resident points and HiCache sizes stay the same." + - "Switch every HiCache override on this arm from hicache-write-policy write_through_selective to write_through. GPU-resident points and HiCache sizes stay the same." + - "At concurrency 80 the overall prefix cache hit rate is flat: 93.5% of prompt tokens on the previous image and policy (run 36743632401) against 94.0% here (run 39763900198). What moves is the tier split -- the HiCache host tier serves 49.1% of prompt tokens instead of 43.3%, and the GPU tier 44.9% instead of 50.2%. Per-GPU throughput goes from 59,233 to 60,839 tok/s (+2.7%), p90 end-to-end latency from 50.9 s to 47.9 s, and mean TTFT from 3.12 s to 2.68 s." + - "That concurrency 80 comparison is not a single-variable A/B: the 36743632401 baseline ran hicache-size 400 on lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260929, while this point runs hicache-size 374 on the v0.5.22 image, so the image, the write policy and the host pool size all differ." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4374