diff --git a/.claude/skills/write-inferencex-blog/SKILL.md b/.claude/skills/write-inferencex-blog/SKILL.md index 5aebab7be..f13bf3807 100644 --- a/.claude/skills/write-inferencex-blog/SKILL.md +++ b/.claude/skills/write-inferencex-blog/SKILL.md @@ -5,7 +5,7 @@ description: Author an InferenceX benchmark blog post in MDX. Codifies the struc # Writing an InferenceX blog post -InferenceX blog posts are short, evidence-dense technical writeups that anchor a single headline number (e.g. "1.41x cheaper", "3.13x throughput", "7.7x peak throughput in 25 days") to (a) a Pareto chart from the live dashboard, (b) verified per-concurrency tables, and (c) the upstream PR that drove the change. +InferenceX blog posts are short, evidence-dense technical writeups that anchor a single headline number (e.g. "29.6% lower cost", "3.13x throughput", "7.7x peak throughput in 25 days") to (a) a Pareto chart from the live dashboard, (b) verified per-concurrency tables, and (c) the upstream PR that drove the change. Posts live at `packages/app/content/blog/{slug}.mdx`. Images live at `packages/app/public/images/{slug}/`. The slug matches the MDX filename without `.mdx`. @@ -38,6 +38,7 @@ Common gotchas: - **Model spec verification**: never invent parameter counts. Always `WebSearch` the model's released specs (total params, active params, expert count, attention type) before writing the architecture paragraph. Cite sources. GLM-5 is _not_ GLM-4.5 — the numbers changed. - **TCO values**: use the dashboard's [SemiAnalysis AI Cloud TCO Model](https://semianalysis.com/ai-cloud-tco-model/) rates, `costh` (owning at hyperscaler volume) in `HW_REGISTRY` (`packages/constants/src/gpu-keys.ts`), so table costs match the chart. - **Cost per million tokens formula**: `$/M tok = TCO_$/GPU/hr * 1e6 / (3600 * tput_per_gpu)`. Equivalently in Python: `cost = tco / (3600 * tput / 1e6)`. Throughput is per-GPU, so GPU count cancels out for aggregated configs. +- **Ratios and reductions have different denominators.** For cost ratio `r = reference cost / candidate cost`, candidate cost saving is `1 - 1/r`, not `r - 1`. Compute ratios and savings from unrounded costs. A 1.421x cost ratio is about 29.6% lower cost; it is not a 42.1% reduction. - **Bandwidth units — keep uni-di vs bi-di and GB/s vs Gbit/s consistent.** This is the single most common factor-of-two or factor-of-eight error in scale-up vs scale-out comparisons, and Cursor Bugbot will catch it. NVLink 5 per-GPU is **900 GB/s uni-directional** (1.8 TB/s bi-directional). ConnectX-7 InfiniBand / RoCEv2 Ethernet per-GPU is **400 Gbit/s = 50 GB/s uni-directional** (100 GB/s bi-di). The NVLink-to-IB/RoCE ratio is **18x in either direction**, not 36x. A previous post (gb200-nvl72-kimi-k2-5-vllm-wide-ep-3x-vs-b200.mdx) shipped with 36x because it compared NVLink bi-di against IB uni-di — flag this if you encounter it in older posts. House rule for new posts: always state "uni-directional" or "uni-di" explicitly, and convert Gbit/s to GB/s in the same sentence so readers can audit the math. ### Iso-interactivity interpolation — match the chart, not your shell script @@ -100,7 +101,7 @@ Sections, in order: Lead with the headline number and the workload, both in the first sentence: -> "14 weeks after GLM-5's release, AMD MI355X SGLang FP8 undercuts NVIDIA B200 SGLang FP8 on cost per million tokens across the entire single-node Pareto frontier on the 8k/1k workload. The peak gap is **1.41x at 18 tok/s/user with MTP** ($0.30/M on B200 vs $0.22/M on MI355X — a 40% reduction)..." +> "14 weeks after GLM-5's release, AMD MI355X SGLang FP8 undercuts NVIDIA B200 SGLang FP8 on cost per million tokens across much of the shared single-node operating range on the 8k/1k workload. The largest interpolated saving is **about 29.6% near 24.3 tok/s/user with MTP** ($0.3388/M on B200 vs $0.2384/M on MI355X; B200 / MI355X cost is 1.421x)..." Bold the peak ratio in the lede. Second paragraph: name the upstream PRs that made it happen, then end with a short framing line ("Speed is the moat.", "Software is the moat.", etc.) if it fits — don't force it. diff --git a/packages/app/content/blog/mi355x-glm5-fp8-sglang-40-cheaper-than-b200.mdx b/packages/app/content/blog/mi355x-glm5-fp8-sglang-40-cheaper-than-b200.mdx index b4212a6a3..adb28e1cc 100644 --- a/packages/app/content/blog/mi355x-glm5-fp8-sglang-40-cheaper-than-b200.mdx +++ b/packages/app/content/blog/mi355x-glm5-fp8-sglang-40-cheaper-than-b200.mdx @@ -1,9 +1,10 @@ --- -title: 'AMD MI355X GLM-5 Inference: Up to 40% Cheaper per Million Tokens than B200 on SGLang FP8' +title: 'AMD MI355X GLM-5 Inference: Up to 29.6% Lower Cost per Million Tokens than B200 on SGLang FP8' subtitle: '14 weeks after GLM-5 launched, AMD landed both MTP and non-MTP SGLang FP8 recipes on MI355X — fused MLA + FP8 KV cache via TileLang flips the single-node FP8 cost curve in AMD favor across most of the performance Pareto' -seoDescription: 'AMD MI355X runs GLM-5 up to 40% cheaper per million tokens than B200 on SGLang FP8, with fused MLA and FP8 KV cache via TileLang.' +seoDescription: 'AMD MI355X runs GLM-5 at up to 29.6% lower cost per million tokens than B200 on SGLang FP8, with fused MLA and FP8 KV cache via TileLang.' date: '2026-05-25' publishDate: '2026-05-25' +modifiedDate: '2026-10-25' tags: - benchmark - gpu @@ -17,15 +18,15 @@ tags: - rocm --- -14 weeks after GLM-5's release, AMD MI355X SGLang FP8 undercuts NVIDIA B200 SGLang FP8 on cost per million tokens across most of the single-node Pareto frontier on the 8k/1k workload (from ~10 to ~77 tok/s/user; B200 noses back ahead above ~90 tok/s/user). The peak gap is **1.41x at 18 tok/s/user with MTP** ($0.30/M on B200 vs $0.22/M on MI355X — a 40% reduction) and **1.36x at 10 tok/s/user without MTP** ($0.31/M vs $0.23/M). Both runs are on **SGLang v0.12**, where the MI355X ROCm stack is now feature-matched with the CUDA stack on B200: both MTP and non-MTP recipes, both with FP8 KV cache, both on SGLang's latest TileLang-backed MLA path. +14 weeks after GLM-5's release, AMD MI355X SGLang FP8 has lower cost per million tokens than NVIDIA B200 SGLang FP8 across much of their shared single-node operating ranges on the 8k/1k workload. With MTP, the largest interpolated saving is **about 29.6% near 24.3 tok/s/user** ($0.3388/M on B200 vs $0.2384/M on MI355X, a B200 / MI355X cost ratio of 1.421x). Without MTP, the largest saving is **about 23.7% near 11.2 tok/s/user** ($0.3161/M vs $0.2411/M, a 1.311x ratio). Both use **SGLang v0.5.12** and FP8 KV cache. These comparisons use the historical data available as of 2026-05-20. This is the cadence that matters. GLM-5 dropped, and within a quarter AMD shipped an upstream SGLang kernel ([sgl-project/sglang PR #21511](https://github.com/sgl-project/sglang/pull/21511)) along with other optimizations plus the matching InferenceX recipes ([InferenceX PR #1440](https://github.com/SemiAnalysisAI/InferenceX/pull/1440)) that flipped the FP8 single-node cost curve on the model. Speed is the moat. - + Click to see the full InferenceX dashboard → -GLM-5 is ZAI's (Zhipu) MoE flagship, released 2026-02-11 — exactly 14 weeks before the InferenceX run in this post. It's a **744B-parameter sparse MoE with ~40B activated per token**: 256 experts with top-8 routing (~5.9% sparsity) plus shared experts. The published architecture name is `glm_moe_dsa` — the model integrates **DeepSeek Sparse Attention (DSA)** on the decode path, the same sparse-attention pattern DeepSeek introduced in V3.2 and that SGLang's TileLang backend was built around, paired with Multi-head Latent Attention (MLA) for KV-cache compression to support its 200K context window. +GLM-5 is ZAI's (Zhipu) MoE flagship, released 2026-02-11 — about 14 weeks before the snapshot in this post. It's a **744B-parameter sparse MoE with ~40B activated per token**: 256 experts with top-8 routing (~5.9% sparsity) plus shared experts. The published architecture name is `glm_moe_dsa` — the model integrates **DeepSeek Sparse Attention (DSA)** on the decode path, the same sparse-attention pattern DeepSeek introduced in V3.2 and that SGLang's TileLang backend was built around, paired with Multi-head Latent Attention (MLA) for KV-cache compression to support its 200K context window. On MI355X, the equivalent capability landed via SGLang's TileLang backend in mid-April, and the resulting decode throughput moved enough that MI355X's lower per-GPU TCO ($1.48/GPU/hr vs B200 at $1.95/GPU/hr per the [SemiAnalysis AI Cloud TCO Model](https://newsletter.semianalysis.com/p/ai-cloud-economics)) now compounds into a real cost-per-token advantage instead of being swamped by software gaps. @@ -39,7 +40,11 @@ The TileLang dependency was bumped to enable FP8 GEMM on AMD, and a new `sparse_ ## The Numbers -All rows are GLM-5 FP8 at **ISL 8192 / OSL 1024** on a single non-disaggregated node, measured on InferenceX on 2026-05-20 on **SGLang v0.12** for both CUDA (B200) and ROCm (MI355X). Cost per million total tokens is computed as `TCO_$/GPU/hr / (3600 × tput_per_gpu / 1e6)`, with B200 at $1.95/GPU/hr and MI355X at $1.48/GPU/hr. +Correction (2026-10-25): the previous version overstated the cost saving as 40% and included comparisons outside the shared measured ranges. The title, opening claims and tables have been recomputed from the same historical source measurements using the dashboard’s current interpolation. + +All rows are GLM-5 FP8 at **ISL 8192 / OSL 1024** on a single non-disaggregated node, available in the InferenceX snapshot as of 2026-05-20 on **SGLang v0.5.12** for both CUDA (B200) and ROCm (MI355X). Cost per million total tokens is computed as `TCO_$/GPU/hr / (3600 × tput_per_gpu / 1e6)`, with B200 at $1.95/GPU/hr and MI355X at $1.48/GPU/hr. + +The source measurements are from [B200 run 26047946365](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1) on 2026-05-18 and [MI355X run 26046385161](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1) on 2026-05-19. May 20 is the snapshot cutoff, not a shared measurement date. Container images used: @@ -102,27 +107,31 @@ Container images used: ## Iso-Interactivity Cost Comparison -Interpolating both Pareto frontiers at matched interactivity. For MI355X MTP the Pareto frontier is the lower of TP=4 and TP=8 at each interactivity — TP=4 dominates up to ~77 tok/s/user, with TP=8 conc 4 taking over at the high-interactivity end (~90 tok/s/user) where TP=4 can't reach. +Throughput is interpolated on each hardware/MTP Pareto frontier with the dashboard’s monotone cubic Hermite spline; cost is then derived as `TCO_$/GPU/hr × 1e6 / (3600 × interpolated throughput)`. MI355X MTP combines TP=4 and TP=8 points into one frontier. Ratios and percentage savings are calculated from unrounded costs: `saving = 1 − MI355X cost / B200 cost`. + +The common measured ranges are approximately 18.052–90.435 tok/s/user with MTP and 10.722–42.032 tok/s/user without MTP. An MTP comparison at 18 tok/s/user or a non-MTP comparison at 10 tok/s/user would require extrapolation, so neither is reported. The interpolated MTP maximum is near 24.32 tok/s/user; the table uses 24.3. MI355X MTP remains cheaper up to the crossover near 89.13 tok/s/user, after which B200 is cheaper. The negative saving at 90 tok/s/user reflects that reversal. **MTP:** -| Interactivity (tok/s/user) | B200 SGLang MTP $/M tok | MI355X SGLang MTP $/M tok | B200 / MI355X | -| -------------------------- | ----------------------- | ------------------------- | ------------- | -| **18** | **$0.30** | **$0.22** | **1.41x** | -| 24 | $0.34 | $0.24 | 1.40x | -| 35 | $0.40 | $0.34 | 1.17x | -| 55 | $0.55 | $0.45 | 1.22x | -| 77 | $0.82 | $0.66 | 1.25x | -| 90 | $1.08 | $1.10 | 0.98x | +| Interactivity (tok/s/user) | B200 SGLang MTP $/M tok | MI355X SGLang MTP $/M tok | B200 / MI355X | MI355X cost saving | +| -------------------------- | ----------------------- | ------------------------- | ------------- | ------------------ | +| 18.1 | $0.2987 | $0.2159 | 1.384x | 27.7% | +| 24 | $0.3371 | $0.2373 | 1.420x | 29.6% | +| **24.3** | **$0.3388** | **$0.2384** | **1.421x** | **29.6%** | +| 35 | $0.3982 | $0.3325 | 1.197x | 16.5% | +| 55 | $0.5500 | $0.4538 | 1.212x | 17.5% | +| 77 | $0.8203 | $0.6608 | 1.241x | 19.4% | +| 90 | $1.0475 | $1.0765 | 0.973x | -2.8% | **Non-MTP:** -| Interactivity (tok/s/user) | B200 SGLang $/M tok | MI355X SGLang $/M tok | B200 / MI355X | -| -------------------------- | ------------------- | --------------------- | ------------- | -| 15 | $0.37 | $0.28 | 1.31x | -| 20 | $0.45 | $0.35 | 1.27x | -| 30 | $0.66 | $0.58 | 1.14x | -| 40 | $1.07 | $1.03 | 1.05x | +| Interactivity (tok/s/user) | B200 SGLang $/M tok | MI355X SGLang $/M tok | B200 / MI355X | MI355X cost saving | +| -------------------------- | ------------------- | --------------------- | ------------- | ------------------ | +| **11.2** | **$0.3161** | **$0.2411** | **1.311x** | **23.7%** | +| 15 | $0.3669 | $0.2817 | 1.302x | 23.2% | +| 20 | $0.4449 | $0.3450 | 1.289x | 22.5% | +| 30 | $0.6450 | $0.5723 | 1.127x | 11.3% | +| 40 | $1.0390 | $0.9822 | 1.058x | 5.5% |
-[Live chart](https://inferencex.semianalysis.com/inference?g_model=GLM-5&i_prec=fp8&g_rundate=2026-05-20&g_runid=26187777287&i_active=b200_sglang%2Cb200_sglang_mtp%2Cmi355x_sglang%2Cmi355x_sglang_mtp&i_metric=y_costh&i_linelabel=1), pre-filtered to GLM-5 FP8 on B200 and MI355X SGLang for the 2026-05-20 run. +[Live chart](https://inferencex.semianalysis.com/inference?g_model=GLM-5&i_prec=fp8&g_rundate=2026-05-20&i_active=b200_sglang%2Cb200_sglang_mtp%2Cmi355x_sglang%2Cmi355x_sglang_mtp&i_metric=y_costh&i_linelabel=1), pre-filtered to GLM-5 FP8 on B200 and MI355X SGLang for the snapshot as of 2026-05-20. ## What's Next for MI355X on GLM-5 @@ -144,7 +153,7 @@ This result is single-node, aggregated, FP8 only. Two gaps still need closing: This recipe loop moved fast because [Anush Elangovan](https://x.com/AnushElangovan), [HaiShaw](https://github.com/HaiShaw), and the broader AMD AI team landed both the upstream SGLang TileLang fused MLA + FP8 KV kernel in a 14-week window after GLM-5 dropped. The SGLang maintainers reviewed and shipped the kernel within days of submission. Speed of the upstream-to-benchmark loop is the moat. - + Click to see the full InferenceX dashboard → @@ -157,7 +166,7 @@ This recipe loop moved fast because [Anush Elangovan](https://x.com/AnushElangov "name": "How much cheaper is AMD MI355X than NVIDIA B200 on GLM-5 FP8 single-node serving?", "acceptedAnswer": { "@type": "Answer", - "text": "On GLM-5 FP8 at 8k/1k sequence length with SGLang MTP, MI355X serves at up to 1.41x lower cost per million tokens than B200 — a 40% reduction. The peak gap is at 18 tok/s/user, where B200 costs $0.30 per million tokens and MI355X costs $0.22. Without MTP, the peak gap is 1.36x at 10 tok/s/user ($0.31 vs $0.23). The MI355X cost advantage holds across the entire MTP curve from ~12 to ~77 tok/s/user. The headline configuration is a 4-GPU TP=4 MTP recipe on MI355X with TCO at $1.48/GPU/hr vs B200 at $1.95/GPU/hr. Measured on InferenceX as of 2026-05-20." + "text": "On GLM-5 FP8 at 8k/1k with SGLang MTP, MI355X has an interpolated cost saving of up to about 29.6% near 24.3 tok/s/user: $0.2384 per million total tokens versus $0.3388 for B200. B200 / MI355X cost is 1.421x, which corresponds to a 29.6% reduction. Without MTP, the largest saving is about 23.7% near 11.2 tok/s/user ($0.2411 versus $0.3161, a 1.311x ratio). The MTP comparison is supported only from approximately 18.052 to 90.435 tok/s/user; MI355X is cheaper up to about 89.13 tok/s/user. TCO assumptions are $1.48/GPU/hr for MI355X and $1.95/GPU/hr for B200. The May 20 snapshot contains B200 measurements from May 18 and MI355X measurements from May 19, 2026." } }, { @@ -170,10 +179,10 @@ This recipe loop moved fast because [Anush Elangovan](https://x.com/AnushElangov }, { "@type": "Question", - "name": "Why is the MI355X cost gap biggest at 18 tok/s/user?", + "name": "Why is the MI355X MTP cost advantage largest near 24.3 tok/s/user?", "acceptedAnswer": { "@type": "Answer", - "text": "Two effects compound. First, the MI355X SGLang MTP recipe ships in a 4-GPU TP=4 variant that has no direct 4-GPU B200 equivalent at FP8, so MI355X spreads the same throughput across fewer GPUs at lower per-GPU TCO ($1.48 vs $1.95/GPU/hr). Second, the MI355X TP=4 throughput curve plateaus at $0.22 per million tokens across concurrency 64 and 128, both delivering ~18 tok/s/user, while B200 at the same interactivity is interpolating between conc 128 ($0.28 at 13.8 tok/s/user) and conc 64 ($0.34 at 23.6 tok/s/user). The compounded effect peaks at 18 tok/s/user (B200 $0.30 vs MI355X $0.22, 1.41x or 40% cheaper). Above 90 tok/s/user the comparison flips marginally back to B200 because there is no MI355X recipe matching B200's TP=8 conc 4 at 100+ tok/s/user." + "text": "MI355X MTP combines a 4-GPU TP=4 configuration with the lower assumed per-GPU TCO of $1.48/GPU/hr, versus $1.95/GPU/hr for B200 TP=8. Near 24.3 tok/s/user, the dashboard interpolates the MI355X throughput between its TP=4 concurrency 64 and 32 points, while B200 is interpolated between concurrency 64 and 32 on TP=8. Deriving cost from those interpolated throughputs gives $0.2384/M for MI355X and $0.3388/M for B200: about 29.6% lower cost and a 1.421x B200 / MI355X cost ratio. At 90 tok/s/user the comparison reverses: B200 is $1.0475/M and MI355X is $1.0765/M. The interpolated crossover is near 89.13 tok/s/user." } }, { @@ -181,7 +190,7 @@ This recipe loop moved fast because [Anush Elangovan](https://x.com/AnushElangov "name": "Is MI355X also cheaper than B200 on GLM-5 FP8 without MTP?", "acceptedAnswer": { "@type": "Answer", - "text": "Yes, but the gap narrows at higher interactivity. Without MTP, MI355X SGLang FP8 (TP=4) is 1.36x cheaper than B200 SGLang FP8 (TP=8) at 10 tok/s/user ($0.23 vs $0.31 per million tokens), 1.27x cheaper at 20 tok/s/user ($0.35 vs $0.45), and 1.05x cheaper at 40 tok/s/user ($1.03 vs $1.07). MTP widens the cheap-end gap because the speculative decoding pass lifts effective per-step throughput on MI355X TP=4 by ~1.34x: at concurrency 32, TPOT drops from 53.9 ms (non-MTP) to 40.3 ms (MTP) and tok/s/GPU rises from 1,274 to 1,707." + "text": "Yes. Without MTP, MI355X SGLang FP8 TP=4 has lower interpolated cost than B200 TP=8 throughout their shared measured range of 10.722 to 42.032 tok/s/user. The largest saving is about 23.7% near 11.2 tok/s/user ($0.2411 versus $0.3161 per million total tokens). At 20 tok/s/user the saving is 22.5% ($0.3450 versus $0.4449), and at 40 tok/s/user it is 5.5% ($0.9822 versus $1.0390). A comparison at 10 tok/s/user is unavailable because B200 has no measured frontier there. On MI355X TP=4 at concurrency 32, MTP increases throughput by about 34%: TPOT falls from 53.9 ms to 40.3 ms and tok/s/GPU rises from 1,274 to 1,707." } }, { diff --git a/packages/app/content/blog/zh/mi355x-glm5-fp8-sglang-40-cheaper-than-b200.mdx b/packages/app/content/blog/zh/mi355x-glm5-fp8-sglang-40-cheaper-than-b200.mdx index a708678ef..7180ef42d 100644 --- a/packages/app/content/blog/zh/mi355x-glm5-fp8-sglang-40-cheaper-than-b200.mdx +++ b/packages/app/content/blog/zh/mi355x-glm5-fp8-sglang-40-cheaper-than-b200.mdx @@ -1,9 +1,10 @@ --- -title: 'AMD MI355X GLM-5 推理：SGLang FP8 每百万 token 成本比 B200 最高低 40%' +title: 'AMD MI355X GLM-5 推理：SGLang FP8 每百万 token 成本比 B200 最高低 29.6%' subtitle: 'GLM-5 发布 14 周后，AMD 在 MI355X 上同时实现了 SGLang FP8 的 MTP 和非 MTP 方案 — 通过 TileLang 实现的融合 MLA + FP8 KV 缓存在大部分性能 Pareto 前沿上将单节点 FP8 成本曲线翻转为 AMD 占优' -seoDescription: 'AMD MI355X 在 SGLang FP8 上运行 GLM-5，每百万 token 成本比 B200 最高低 40%——TileLang 融合 MLA 与 FP8 KV 缓存。' +seoDescription: 'AMD MI355X 在 SGLang FP8 上运行 GLM-5，每百万 token 成本比 B200 最高低 29.6%——TileLang 融合 MLA 与 FP8 KV 缓存。' date: '2026-05-25' publishDate: '2026-05-25' +modifiedDate: '2026-10-25' tags: - benchmark - gpu @@ -17,15 +18,15 @@ tags: - rocm --- -GLM-5 发布 14 周后，AMD MI355X SGLang FP8 在 8k/1k 工作负载的大部分单节点 Pareto 前沿上，每百万 token 成本低于 NVIDIA B200 SGLang FP8（从约 10 到约 77 tok/s/user；B200 在约 90 tok/s/user 以上重新反超）。峰值差距为**使用 MTP 时在 18 tok/s/user 下达到 1.41 倍**（B200 $0.30/M vs MI355X $0.22/M — 降低 40%），**不使用 MTP 时在 10 tok/s/user 下达到 1.36 倍**（$0.31/M vs $0.23/M）。两项测试均使用 **SGLang v0.12**，MI355X 的 ROCm 软件栈在此版本上已与 B200 的 CUDA 软件栈功能对齐：均支持 MTP 和非 MTP 方案，均支持 FP8 KV 缓存，均基于 SGLang 最新的 TileLang MLA 路径。 +GLM-5 发布 14 周后，在 8k/1k 工作负载下，AMD MI355X SGLang FP8 在双方共有的单节点运行区间内，大部分交互性水平上的每百万 token 成本低于 NVIDIA B200 SGLang FP8。使用 MTP 时，插值得到的最大成本降幅约为 **29.6%，出现在约 24.3 tok/s/user**（B200 $0.3388/M，MI355X $0.2384/M；B200 / MI355X 成本比为 1.421 倍）。不使用 MTP 时，最大降幅约为 **23.7%，出现在约 11.2 tok/s/user**（$0.3161/M vs $0.2411/M；成本比为 1.311 倍）。双方均使用 **SGLang v0.5.12** 和 FP8 KV 缓存。本次比较采用截至 2026-05-20 的历史数据快照。 这正是关键的节奏。GLM-5 发布后，一个季度内 AMD 就完成了上游 SGLang 内核的合入（[sgl-project/sglang PR #21511](https://github.com/sgl-project/sglang/pull/21511)）及其他优化，并提交了配套的 InferenceX 方案（[InferenceX PR #1440](https://github.com/SemiAnalysisAI/InferenceX/pull/1440)），将该模型的 FP8 单节点成本曲线翻转为 AMD 占优。速度就是护城河。 - + 点击查看完整 InferenceX 仪表板 → -GLM-5 是智谱（ZAI）的 MoE 旗舰模型，于 2026-02-11 发布 — 距本文所述的 InferenceX 测试正好 14 周。该模型拥有 **744B 参数的稀疏 MoE 架构，每 token 激活约 40B**：256 个专家采用 top-8 路由（约 5.9% 稀疏度），外加共享专家。公开的架构名称为 `glm_moe_dsa` — 模型在解码路径中集成了 **DeepSeek 稀疏注意力（DSA）**，这与 DeepSeek 在 V3.2 中引入的稀疏注意力模式相同，也是 SGLang 的 TileLang 后端所围绕构建的核心，同时采用多头潜在注意力（MLA）进行 KV 缓存压缩以支持其 200K 上下文窗口。 +GLM-5 是智谱（ZAI）的 MoE 旗舰模型，于 2026-02-11 发布 — 距本文采用的快照约 14 周。该模型拥有 **744B 参数的稀疏 MoE 架构，每 token 激活约 40B**：256 个专家采用 top-8 路由（约 5.9% 稀疏度），外加共享专家。公开的架构名称为 `glm_moe_dsa` — 模型在解码路径中集成了 **DeepSeek 稀疏注意力（DSA）**，这与 DeepSeek 在 V3.2 中引入的稀疏注意力模式相同，也是 SGLang 的 TileLang 后端所围绕构建的核心，同时采用多头潜在注意力（MLA）进行 KV 缓存压缩以支持其 200K 上下文窗口。 在 MI355X 上，等效能力在四月中旬通过 SGLang 的 TileLang 后端落地，由此带来的解码吞吐量提升使得 MI355X 较低的单 GPU TCO（$1.48/GPU/hr，B200 为 $1.95/GPU/hr，数据来源 [SemiAnalysis AI Cloud TCO 模型](https://newsletter.semianalysis.com/p/ai-cloud-economics)）得以转化为真正的每 token 成本优势，而非被软件差距所淹没。 @@ -39,7 +40,11 @@ TileLang 依赖版本已更新以在 AMD 上启用 FP8 GEMM，并新增了 `spar ## 基准测试数据 -所有数据均为 GLM-5 FP8，**ISL 8192 / OSL 1024**，单节点非分离式部署，于 2026-05-20 在 InferenceX 上测量，CUDA（B200）和 ROCm（MI355X）均使用 **SGLang v0.12**。每百万总 token 成本计算方式为 `TCO_$/GPU/hr / (3600 × tput_per_gpu / 1e6)`，B200 为 $1.95/GPU/hr，MI355X 为 $1.48/GPU/hr。 +更正（2026-10-25）：此前版本将成本降幅误写为 40%，且部分比较超出了双方共有的实测区间。本文标题、开篇结论和下表均基于同一批历史测量数据，按仪表板当前的插值方法重新计算。 + +所有数据均为 GLM-5 FP8，**ISL 8192 / OSL 1024**，单节点非分离式部署，来自截至 2026-05-20 的 InferenceX 数据快照，CUDA（B200）和 ROCm（MI355X）均使用 **SGLang v0.5.12**。每百万总 token 成本计算方式为 `TCO_$/GPU/hr / (3600 × tput_per_gpu / 1e6)`，B200 为 $1.95/GPU/hr，MI355X 为 $1.48/GPU/hr。 + +原始测量分别来自 2026-05-18 的 [B200 运行 26047946365](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1) 和 2026-05-19 的 [MI355X 运行 26046385161](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1)。5 月 20 日是快照截止日期，并非双方共同的测量日期。 容器镜像： @@ -102,27 +107,31 @@ TileLang 依赖版本已更新以在 AMD 上启用 FP8 GEMM，并新增了 `spar ## 等交互性成本对比 -对两条 Pareto 前沿在匹配交互性下进行插值。对于 MI355X MTP，Pareto 前沿取 TP=4 和 TP=8 在每个交互性水平上的较低值 — TP=4 在约 77 tok/s/user 以下占优，TP=8 并发数 4 在高交互性端（约 90 tok/s/user）接管，因为 TP=4 无法达到该区间。 +对每种硬件与 MTP 设置，在其吞吐量 Pareto 前沿上使用与仪表板相同的单调三次 Hermite 样条插值，再按 `TCO_$/GPU/hr × 1e6 / (3600 × interpolated throughput)` 推导成本。MI355X MTP 将 TP=4 与 TP=8 数据点合并为一条前沿。成本比和降幅均使用未舍入的成本计算：`saving = 1 − MI355X cost / B200 cost`。 + +双方共有的实测交互性范围约为：使用 MTP 时 18.052–90.435 tok/s/user，不使用 MTP 时 10.722–42.032 tok/s/user。MTP 在 18 tok/s/user 或非 MTP 在 10 tok/s/user 的比较均需要外推，因此不报告这两个比较。MTP 的最大插值降幅出现在约 24.32 tok/s/user，表中采用 24.3。MI355X MTP 在约 89.13 tok/s/user 的交点之前成本更低，此后 B200 更低；90 tok/s/user 行的负降幅表示这一反转。 **MTP：** -| 交互性 (tok/s/user) | B200 SGLang MTP $/M tok | MI355X SGLang MTP $/M tok | B200 / MI355X | -| ------------------- | ----------------------- | ------------------------- | ------------- | -| **18** | **$0.30** | **$0.22** | **1.41x** | -| 24 | $0.34 | $0.24 | 1.40x | -| 35 | $0.40 | $0.34 | 1.17x | -| 55 | $0.55 | $0.45 | 1.22x | -| 77 | $0.82 | $0.66 | 1.25x | -| 90 | $1.08 | $1.10 | 0.98x | +| 交互性 (tok/s/user) | B200 SGLang MTP $/M tok | MI355X SGLang MTP $/M tok | B200 / MI355X | MI355X 成本降幅 | +| ------------------- | ----------------------- | ------------------------- | ------------- | --------------- | +| 18.1 | $0.2987 | $0.2159 | 1.384x | 27.7% | +| 24 | $0.3371 | $0.2373 | 1.420x | 29.6% | +| **24.3** | **$0.3388** | **$0.2384** | **1.421x** | **29.6%** | +| 35 | $0.3982 | $0.3325 | 1.197x | 16.5% | +| 55 | $0.5500 | $0.4538 | 1.212x | 17.5% | +| 77 | $0.8203 | $0.6608 | 1.241x | 19.4% | +| 90 | $1.0475 | $1.0765 | 0.973x | -2.8% | **非 MTP：** -| 交互性 (tok/s/user) | B200 SGLang $/M tok | MI355X SGLang $/M tok | B200 / MI355X | -| ------------------- | ------------------- | --------------------- | ------------- | -| 15 | $0.37 | $0.28 | 1.31x | -| 20 | $0.45 | $0.35 | 1.27x | -| 30 | $0.66 | $0.58 | 1.14x | -| 40 | $1.07 | $1.03 | 1.05x | +| 交互性 (tok/s/user) | B200 SGLang $/M tok | MI355X SGLang $/M tok | B200 / MI355X | MI355X 成本降幅 | +| ------------------- | ------------------- | --------------------- | ------------- | --------------- | +| **11.2** | **$0.3161** | **$0.2411** | **1.311x** | **23.7%** | +| 15 | $0.3669 | $0.2817 | 1.302x | 23.2% | +| 20 | $0.4449 | $0.3450 | 1.289x | 22.5% | +| 30 | $0.6450 | $0.5723 | 1.127x | 11.3% | +| 40 | $1.0390 | $0.9822 | 1.058x | 5.5% |
-[实时图表](https://inferencex.semianalysis.com/inference?g_model=GLM-5&i_prec=fp8&g_rundate=2026-05-20&g_runid=26187777287&i_active=b200_sglang%2Cb200_sglang_mtp%2Cmi355x_sglang%2Cmi355x_sglang_mtp&i_metric=y_costh&i_linelabel=1)，预筛选为 2026-05-20 测试中 B200 和 MI355X SGLang 上的 GLM-5 FP8。 +[实时图表](https://inferencex.semianalysis.com/inference?g_model=GLM-5&i_prec=fp8&g_rundate=2026-05-20&i_active=b200_sglang%2Cb200_sglang_mtp%2Cmi355x_sglang%2Cmi355x_sglang_mtp&i_metric=y_costh&i_linelabel=1)，预筛选为截至 2026-05-20 快照中 B200 和 MI355X SGLang 上的 GLM-5 FP8。 ## MI355X 在 GLM-5 上的后续展望 @@ -144,7 +153,7 @@ TileLang 依赖版本已更新以在 AMD 上启用 FP8 GEMM，并新增了 `spar 该方案的快速落地得益于 [Anush Elangovan](https://x.com/AnushElangovan)、[HaiShaw](https://github.com/HaiShaw) 及更广泛的 AMD AI 团队在 GLM-5 发布后 14 周内完成了上游 SGLang TileLang 融合 MLA + FP8 KV 内核的提交。SGLang 维护者在提交后数天内即完成了审查与合入。从上游到基准测试的闭环速度就是护城河。 - + 点击查看完整 InferenceX 仪表板 → @@ -157,7 +166,7 @@ TileLang 依赖版本已更新以在 AMD 上启用 FP8 GEMM，并新增了 `spar "name": "AMD MI355X 在 GLM-5 FP8 单节点推理上比 NVIDIA B200 便宜多少？", "acceptedAnswer": { "@type": "Answer", - "text": "在 GLM-5 FP8、8k/1k 序列长度、SGLang MTP 条件下，MI355X 的每百万 token 成本比 B200 最高低 1.41 倍 — 降低 40%。峰值差距出现在 18 tok/s/user，B200 每百万 token 成本为 $0.30，MI355X 为 $0.22。不使用 MTP 时，峰值差距为 1.36 倍，出现在 10 tok/s/user（$0.31 vs $0.23）。MI355X 的成本优势覆盖整个 MTP 曲线的约 12 至约 77 tok/s/user 区间。核心配置为 MI355X 上 4-GPU TP=4 MTP 方案，TCO 为 $1.48/GPU/hr，B200 为 $1.95/GPU/hr。数据基于 2026-05-20 InferenceX 测量。" + "text": "在 GLM-5 FP8、8k/1k、SGLang MTP 条件下，MI355X 的最大插值成本降幅约为 29.6%，出现在约 24.3 tok/s/user：每百万总 token 成本为 $0.2384，B200 为 $0.3388。B200 / MI355X 成本比为 1.421 倍，对应成本降低 29.6%。不使用 MTP 时，最大降幅约为 23.7%，出现在约 11.2 tok/s/user（$0.2411 vs $0.3161，成本比为 1.311 倍）。MTP 比较仅在约 18.052 至 90.435 tok/s/user 范围内有实测前沿支撑；MI355X 在约 89.13 tok/s/user 之前成本更低。TCO 假设为 MI355X $1.48/GPU/hr、B200 $1.95/GPU/hr。5 月 20 日快照包含的 B200 测量来自 2026 年 5 月 18 日，MI355X 测量来自 5 月 19 日。" } }, { @@ -170,10 +179,10 @@ TileLang 依赖版本已更新以在 AMD 上启用 FP8 GEMM，并新增了 `spar }, { "@type": "Question", - "name": "为什么 MI355X 的成本差距在 18 tok/s/user 时最大？", + "name": "为什么 MI355X MTP 的成本优势在约 24.3 tok/s/user 时最大？", "acceptedAnswer": { "@type": "Answer", - "text": "两个效应叠加。首先，MI355X SGLang MTP 方案提供了 4-GPU TP=4 变体，在 FP8 下没有直接对应的 4-GPU B200 方案，因此 MI355X 以更少的 GPU 和更低的单 GPU TCO（$1.48 vs $1.95/GPU/hr）分摊相同的吞吐量。其次，MI355X TP=4 的吞吐量曲线在并发数 64 和 128 时稳定在 $0.22/M，两者均提供约 18 tok/s/user，而 B200 在相同交互性下正在并发数 128（$0.28，13.8 tok/s/user）和并发数 64（$0.34，23.6 tok/s/user）之间插值。叠加效应在 18 tok/s/user 时达到峰值（B200 $0.30 vs MI355X $0.22，1.41 倍即便宜 40%）。在 90 tok/s/user 以上，对比略微翻转回 B200 有利，因为没有 MI355X 方案能匹配 B200 的 TP=8 并发数 4 在 100+ tok/s/user 的表现。" + "text": "MI355X MTP 将 4-GPU TP=4 配置与较低的单 GPU TCO 假设相结合：MI355X 为 $1.48/GPU/hr，B200 TP=8 为 $1.95/GPU/hr。在约 24.3 tok/s/user 处，仪表板在 MI355X TP=4 的并发数 64 与 32 之间插值吞吐量；B200 则在 TP=8 的并发数 64 与 32 之间插值。由吞吐量推导成本后，MI355X 为 $0.2384/M，B200 为 $0.3388/M，即成本约低 29.6%，B200 / MI355X 成本比为 1.421 倍。在 90 tok/s/user 处比较反转：B200 为 $1.0475/M，MI355X 为 $1.0765/M，插值交点约为 89.13 tok/s/user。" } }, { @@ -181,7 +190,7 @@ TileLang 依赖版本已更新以在 AMD 上启用 FP8 GEMM，并新增了 `spar "name": "MI355X 在不使用 MTP 的 GLM-5 FP8 上也比 B200 便宜吗？", "acceptedAnswer": { "@type": "Answer", - "text": "是的，但差距在更高交互性下缩小。不使用 MTP 时，MI355X SGLang FP8（TP=4）在 10 tok/s/user 下比 B200 SGLang FP8（TP=8）便宜 1.36 倍（$0.23 vs $0.31/M），在 20 tok/s/user 下便宜 1.27 倍（$0.35 vs $0.45），在 40 tok/s/user 下便宜 1.05 倍（$1.03 vs $1.07）。MTP 拉大了低成本端的差距，因为投机解码提升了 MI355X TP=4 的每步有效吞吐量约 1.34 倍：在并发数 32 时，TPOT 从 53.9 ms（非 MTP）降至 40.3 ms（MTP），tok/s/GPU 从 1,274 升至 1,707。" + "text": "是的。不使用 MTP 时，在双方共有的 10.722 至 42.032 tok/s/user 实测范围内，MI355X SGLang FP8 TP=4 的插值成本始终低于 B200 TP=8。最大降幅约为 23.7%，出现在约 11.2 tok/s/user（每百万总 token 为 $0.2411 vs $0.3161）。在 20 tok/s/user 处，降幅为 22.5%（$0.3450 vs $0.4449）；在 40 tok/s/user 处，降幅为 5.5%（$0.9822 vs $1.0390）。B200 的实测前沿不覆盖 10 tok/s/user，因此无法在该点比较。在 MI355X TP=4、并发数 32 时，MTP 使吞吐量提高约 34%：TPOT 从 53.9 ms 降至 40.3 ms，tok/s/GPU 从 1,274 升至 1,707。" } }, { diff --git a/packages/app/cypress/e2e/blog.cy.ts b/packages/app/cypress/e2e/blog.cy.ts index f7f0f835d..7dca2a2e3 100644 --- a/packages/app/cypress/e2e/blog.cy.ts +++ b/packages/app/cypress/e2e/blog.cy.ts @@ -33,6 +33,47 @@ describe('Blog', () => { } }); + describe('GLM-5 historical cost correction', () => { + for (const locale of ['', '/zh']) { + for (const width of [390, 1280]) { + it(`keeps the verified cost comparison and metadata aligned on ${locale || '/en'} at ${width}px`, () => { + cy.viewport(width, 900); + cy.visit(`${locale}/blog/mi355x-glm5-fp8-sglang-40-cheaper-than-b200`); + cy.get('h1').should('contain.text', '29.6%'); + cy.get('meta[name="description"]').should('have.attr', 'content').and('include', '29.6%'); + cy.contains('article.prose table', 'B200 / MI355X').within(() => { + cy.contains('tbody tr', '24.3') + .find('td') + .then(($cells) => { + expect([...$cells].map((cell) => cell.textContent?.trim())).to.deep.equal([ + '24.3', + '$0.3388', + '$0.2384', + '1.421x', + '29.6%', + ]); + }); + cy.contains('tbody tr', '90').should('contain.text', '-2.8%'); + }); + cy.get('script[type="application/ld+json"]').then(($scripts) => { + const article = [...$scripts] + .map((script) => JSON.parse(script.textContent ?? '{}')) + .find((value) => value['@type'] === 'BlogPosting'); + expect(article.headline).to.include('29.6%'); + expect(article.description).to.include('29.6%'); + expect(article.dateModified).to.equal('2026-10-25T00:00:00Z'); + }); + cy.get('article.prose a[href*="g_rundate=2026-05-20"]').each(($link) => { + expect($link.attr('href')).not.to.include('g_runid='); + }); + cy.document().then((doc) => { + expect(doc.documentElement.scrollWidth).to.be.at.most(doc.documentElement.clientWidth); + }); + }); + } + } + }); + describe('Blog listing page', () => { beforeEach(() => { cy.intercept( diff --git a/packages/app/src/lib/blog/__fixtures__/glm5-cost-2026-05-20.json b/packages/app/src/lib/blog/__fixtures__/glm5-cost-2026-05-20.json new file mode 100644 index 000000000..e3610a09c --- /dev/null +++ b/packages/app/src/lib/blog/__fixtures__/glm5-cost-2026-05-20.json @@ -0,0 +1,804 @@ +{ + "sourceUrl": "https://inferencex.semianalysis.com/api/v1/benchmarks?model=GLM-5&date=2026-05-20", + "responseSha256": "a576e88bfeaa260169ca397c852aff234259913b9dc0d9825a3e2219c7a55b19", + "scope": { + "model": "glm5", + "framework": "sglang", + "precision": "fp8", + "isl": 8192, + "osl": 1024, + "benchmarkType": "single_turn", + "disagg": false, + "offloadMode": "off" + }, + "tcoUsdPerGpuHour": { + "b200": 1.95, + "mi355x": 1.48 + }, + "rows": [ + { + "id": "412400", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 4, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 54.24537542613569, + "tput_per_gpu": 231.26304414870341, + "median_tpot": 0.018434751204951475 + } + }, + { + "id": "412401", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 46.06894098077538, + "tput_per_gpu": 382.38813648034966, + "median_tpot": 0.021706598387345198 + } + }, + { + "id": "412390", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 16, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 36.651571934675744, + "tput_per_gpu": 613.1904541082405, + "median_tpot": 0.027283959383305696 + } + }, + { + "id": "412409", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 32, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 27.472662793722215, + "tput_per_gpu": 933.7016600273083, + "median_tpot": 0.03639982070571296 + } + }, + { + "id": "412405", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 64, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 18.42246707372037, + "tput_per_gpu": 1291.7848985199907, + "median_tpot": 0.05428154633135423 + } + }, + { + "id": "412391", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 128, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 11.871840814598112, + "tput_per_gpu": 1669.0849794487451, + "median_tpot": 0.08423293536503271 + } + }, + { + "id": "412393", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 256, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 10.721980046228646, + "tput_per_gpu": 1746.0615266138582, + "median_tpot": 0.09326635525233423 + } + }, + { + "id": "413077", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 4, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 90.4348013475264, + "tput_per_gpu": 373.3946225143232, + "median_tpot": 0.01105769001644799 + } + }, + { + "id": "413080", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 65.04882654868278, + "tput_per_gpu": 534.1808232533447, + "median_tpot": 0.015373067479573923 + } + }, + { + "id": "412410", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 4, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 100.84785554759166, + "tput_per_gpu": 416.95543943151426, + "median_tpot": 0.009915927260625632 + } + }, + { + "id": "412403", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 77.81560058966446, + "tput_per_gpu": 650.10261622864, + "median_tpot": 0.012850893553764089 + } + }, + { + "id": "412399", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 16, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 56.928586385956976, + "tput_per_gpu": 952.7343392926402, + "median_tpot": 0.017565867404827705 + } + }, + { + "id": "412392", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 32, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 38.158798210202235, + "tput_per_gpu": 1296.8302948927694, + "median_tpot": 0.026206276059622793 + } + }, + { + "id": "412394", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 64, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 23.555056357832044, + "tput_per_gpu": 1619.253391375855, + "median_tpot": 0.04245372988324439 + } + }, + { + "id": "412404", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 128, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 13.776335469740541, + "tput_per_gpu": 1929.4973046802902, + "median_tpot": 0.07258824396346046 + } + }, + { + "id": "412395", + "hardware": "b200", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "decode_tp": 8, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 256, + "offload_mode": "off", + "image": "lmsysorg/sglang:v0.5.12-cu130", + "date": "2026-05-18", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1", + "metrics": { + "median_intvty": 11.884138423026432, + "tput_per_gpu": 1947.3241344467342, + "median_tpot": 0.08414577181820965 + } + }, + { + "id": "413083", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 4, + "decode_tp": 4, + "num_prefill_gpu": 4, + "num_decode_gpu": 4, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 4, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 76.7953586691819, + "tput_per_gpu": 625.4685640503618, + "median_tpot": 0.013021620281868696 + } + }, + { + "id": "413071", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 4, + "decode_tp": 4, + "num_prefill_gpu": 4, + "num_decode_gpu": 4, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 54.58970949911198, + "tput_per_gpu": 911.7248187766098, + "median_tpot": 0.018318470810258974 + } + }, + { + "id": "413070", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 4, + "decode_tp": 4, + "num_prefill_gpu": 4, + "num_decode_gpu": 4, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 16, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 35.82165233491425, + "tput_per_gpu": 1208.0782780615016, + "median_tpot": 0.02791607686464343 + } + }, + { + "id": "413072", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 4, + "decode_tp": 4, + "num_prefill_gpu": 4, + "num_decode_gpu": 4, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 32, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 24.833931236465183, + "tput_per_gpu": 1707.4049410888426, + "median_tpot": 0.04026748687020759 + } + }, + { + "id": "413058", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 4, + "decode_tp": 4, + "num_prefill_gpu": 4, + "num_decode_gpu": 4, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 64, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 18.18597527246096, + "tput_per_gpu": 1895.0037505636333, + "median_tpot": 0.05498742767534171 + } + }, + { + "id": "413082", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 4, + "decode_tp": 4, + "num_prefill_gpu": 4, + "num_decode_gpu": 4, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 128, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 18.052019666793043, + "tput_per_gpu": 1911.7240239628152, + "median_tpot": 0.055395463690941726 + } + }, + { + "id": "413068", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 4, + "decode_tp": 4, + "num_prefill_gpu": 4, + "num_decode_gpu": 4, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 4, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 42.03240852199166, + "tput_per_gpu": 358.75026574016493, + "median_tpot": 0.023791165797144187 + } + }, + { + "id": "413084", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 4, + "decode_tp": 4, + "num_prefill_gpu": 4, + "num_decode_gpu": 4, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 34.68385782762802, + "tput_per_gpu": 579.5877397319746, + "median_tpot": 0.028831856161151515 + } + }, + { + "id": "413056", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 4, + "decode_tp": 4, + "num_prefill_gpu": 4, + "num_decode_gpu": 4, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 16, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 25.861738506998154, + "tput_per_gpu": 870.7946938012432, + "median_tpot": 0.03866716074518352 + } + }, + { + "id": "413079", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 4, + "decode_tp": 4, + "num_prefill_gpu": 4, + "num_decode_gpu": 4, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 32, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 18.567946168238887, + "tput_per_gpu": 1274.0548241409356, + "median_tpot": 0.05385625264847733 + } + }, + { + "id": "413085", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 4, + "decode_tp": 4, + "num_prefill_gpu": 4, + "num_decode_gpu": 4, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 64, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 11.825685186107302, + "tput_per_gpu": 1660.114957218454, + "median_tpot": 0.08456169636367372 + } + }, + { + "id": "413059", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 4, + "decode_tp": 4, + "num_prefill_gpu": 4, + "num_decode_gpu": 4, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 128, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 7.333498783107506, + "tput_per_gpu": 2071.4105602853188, + "median_tpot": 0.13636055988765827 + } + }, + { + "id": "413063", + "hardware": "mi355x", + "framework": "sglang", + "model": "glm5", + "precision": "fp8", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 4, + "decode_tp": 4, + "num_prefill_gpu": 4, + "num_decode_gpu": 4, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 256, + "offload_mode": "off", + "image": "lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517", + "date": "2026-05-19", + "run_url": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1", + "metrics": { + "median_intvty": 6.691098178675371, + "tput_per_gpu": 2189.45217555414, + "median_tpot": 0.14945229815742575 + } + } + ], + "recordedAt": "2026-10-25T09:53:01.593939+00:00" +} diff --git a/packages/app/src/lib/blog/glm5-cost-comparison.test.ts b/packages/app/src/lib/blog/glm5-cost-comparison.test.ts new file mode 100644 index 000000000..1e7f9bafe --- /dev/null +++ b/packages/app/src/lib/blog/glm5-cost-comparison.test.ts @@ -0,0 +1,163 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +import { interpolateMetricAtInteractivity } from '@/components/inference/hooks/interpolated-trend-core'; +import type { InferenceData } from '@/components/inference/types'; + +import evidence from './__fixtures__/glm5-cost-2026-05-20.json'; + +const SLUG = 'mi355x-glm5-fp8-sglang-40-cheaper-than-b200'; +const CONTENT = path.resolve(import.meta.dirname, '../../../content/blog'); +type Hardware = keyof typeof evidence.tcoUsdPerGpuHour; +type Mode = 'mtp' | 'none'; + +const metric = (y: number) => ({ y, roof: false }); + +function points(hardware: Hardware, mode: Mode): InferenceData[] { + return evidence.rows + .filter((row) => row.hardware === hardware && row.spec_method === mode) + .map((row) => { + const throughput = row.metrics.tput_per_gpu; + const pointCost = (evidence.tcoUsdPerGpuHour[hardware] * 1e6) / (3600 * throughput); + return { + x: row.metrics.median_intvty, + y: throughput, + hwKey: `${hardware}_sglang${mode === 'mtp' ? '_mtp' : ''}`, + date: row.date, + tp: row.decode_tp, + conc: row.conc, + precision: row.precision, + tpPerGpu: metric(throughput), + tpPerMw: metric(0), + costh: metric(pointCost), + costr: metric(0), + costhi: metric(0), + costri: metric(0), + }; + }); +} + +function cost(hardware: Hardware, mode: Mode, target: number): number | null { + return interpolateMetricAtInteractivity(points(hardware, mode), target, 'costh'); +} + +function comparison(mode: Mode, target: number): string[] { + const b200 = cost('b200', mode, target); + const mi355x = cost('mi355x', mode, target); + if (b200 === null || mi355x === null) throw new Error('Target outside common measured range'); + return [ + String(target), + `$${b200.toFixed(4)}`, + `$${mi355x.toFixed(4)}`, + `${(b200 / mi355x).toFixed(3)}x`, + `${((1 - mi355x / b200) * 100).toFixed(1)}%`, + ]; +} + +function tableRows(source: string): string[][][] { + return source + .split(/\n\s*\n/u) + .filter((block) => /^\|.*B200 \/ MI355X/u.test(block)) + .map((block) => + block + .split('\n') + .slice(2) + .map((line) => + line + .split('|') + .slice(1, -1) + .map((cell) => cell.replaceAll('**', '').trim()), + ), + ); +} + +it('retains the exact historical population and original producer identities', () => { + expect(evidence.rows).toHaveLength(29); + expect(new Set(evidence.rows.map((row) => row.id)).size).toBe(29); + for (const row of evidence.rows) { + expect(row).toMatchObject({ + model: 'glm5', + framework: 'sglang', + precision: 'fp8', + benchmark_type: 'single_turn', + isl: 8192, + osl: 1024, + disagg: false, + offload_mode: 'off', + }); + expect([row.date, row.run_url]).toEqual( + row.hardware === 'b200' + ? [ + '2026-05-18', + 'https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26047946365/attempts/1', + ] + : [ + '2026-05-19', + 'https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26046385161/attempts/1', + ], + ); + } +}); + +it('rejects the former extrapolated comparisons instead of manufacturing a saving', () => { + expect(cost('mi355x', 'mtp', 18)).toBeNull(); + expect(cost('b200', 'mtp', 18)).toBeGreaterThan(0); + expect(cost('b200', 'none', 10)).toBeNull(); + expect(cost('mi355x', 'none', 10)).toBeGreaterThan(0); +}); + +it('keeps the non-MTP saving positive across the complete common measured range', () => { + const low = 10.721980046228646; + const high = 42.03240852199166; + for (let sample = 0; sample <= 100; sample += 1) { + const target = low + ((high - low) * sample) / 100; + expect(cost('b200', 'none', target)! - cost('mi355x', 'none', target)!).toBeGreaterThan(0); + } +}); + +it('uses unrounded costs and the reference-cost denominator for percentage savings', () => { + const b200 = cost('b200', 'mtp', 24.3195310505088)!; + const mi355x = cost('mi355x', 'mtp', 24.3195310505088)!; + expect(b200).toBeCloseTo(0.33889205243689297, 12); + expect(mi355x).toBeCloseTo(0.23850119029670874, 12); + expect(b200 / mi355x).toBeCloseTo(1.4209239459781833, 12); + expect((1 - mi355x / b200) * 100).toBeCloseTo(29.62325655567818, 10); + expect(comparison('mtp', 90)).toEqual(['90', '$1.0475', '$1.0765', '0.973x', '-2.8%']); +}); + +for (const locale of ['en', 'zh'] as const) { + describe(`${locale} GLM-5 article numerical parity`, () => { + const source = fs.readFileSync( + path.join(CONTENT, locale === 'zh' ? 'zh' : '', `${SLUG}.mdx`), + 'utf8', + ); + + it('publishes only reachable table rows computed by the dashboard interpolation', () => { + expect(tableRows(source)).toEqual([ + [18.1, 24, 24.3, 35, 55, 77, 90].map((target) => comparison('mtp', target)), + [11.2, 15, 20, 30, 40].map((target) => comparison('none', target)), + ]); + }); + + it('keeps metadata and structured answers aligned with the recomputed costs', () => { + const title = source.match(/^title: (?.*)$/mu)!.groups!.value; + const description = source.match(/^seoDescription: (?.*)$/mu)!.groups!.value; + const saving = comparison('mtp', 24.3)[4]; + expect(title).toContain(saving); + expect(description).toContain(saving); + const jsonLd = source.split('{`')[1].split('`}')[0]; + const faq = JSON.parse(jsonLd).mainEntity; + for (const index of [0, 2]) { + const answer: string = faq[index].acceptedAnswer.text; + for (const figure of comparison('mtp', 24.3).slice(0, 3)) { + expect(answer).toContain(figure); + } + expect(answer).toContain(saving); + } + expect(faq[3].acceptedAnswer.text).toContain(comparison('none', 11.2)[4]); + expect(source).not.toContain('g_runid=26187777287'); + }); + }); +}