diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/mi355x-fp4-mtp/agentic.yaml index 979f69af96..1ea4e7f22b 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/mi355x-fp4-mtp/agentic.yaml @@ -60,81 +60,219 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' override_tp8_c1: + roles: + agg: + args: + monokernel-backend: amd + max-running-requests: 1 + cuda-graph-max-bs-decode: 1 benchmark: env: CONC: '1' KV_OFFLOADING: none -override_tp8_c2: - benchmark: - env: - CONC: '2' - KV_OFFLOADING: none - override_tp8_c4: + roles: + agg: + args: + monokernel-backend: amd + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + prefill-decode-interval: 64 benchmark: env: CONC: '4' KV_OFFLOADING: none override_tp8_c8: + roles: + agg: + args: + monokernel-backend: amd + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + prefill-decode-interval: 64 benchmark: env: CONC: '8' KV_OFFLOADING: none -override_tp8_c16: +override_tp8_c13: roles: agg: args: + monokernel-backend: amd + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + prefill-decode-interval: 64 + benchmark: + env: + CONC: '13' + KV_OFFLOADING: none + +override_tp4_c10: + roles: + agg: + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first hicache-size: 253 + swa-full-tokens-ratio: 0.1 + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + prefill-decode-interval: 64 + mem-fraction-static: 0.88 + env: + PYTORCH_HIP_ALLOC_CONF: expandable_segments:True + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + gpus: 4 + benchmark: + env: + CONC: '10' + KV_OFFLOADING: dram + +override_tp4_c16: + roles: + agg: + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + enable-hierarchical-cache: true hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first + hicache-size: 253 + swa-full-tokens-ratio: 0.1 + schedule-policy: shortest-prefill-first + max-running-requests: 12 + cuda-graph-max-bs-decode: 12 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + prefill-decode-interval: 96 + env: + PYTORCH_HIP_ALLOC_CONF: expandable_segments:True + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + gpus: 4 benchmark: env: CONC: '16' KV_OFFLOADING: dram -override_tp8_c32: +override_tp4_c24: roles: agg: args: + tensor-parallel-size: 4 + expert-parallel-size: 4 enable-hierarchical-cache: true + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first hicache-size: 253 + swa-full-tokens-ratio: 0.1 + schedule-policy: shortest-prefill-first + max-running-requests: 48 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + prefill-decode-interval: 48 + mem-fraction-static: 0.96 + env: + PYTORCH_HIP_ALLOC_CONF: expandable_segments:True + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + gpus: 4 + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + +override_tp4_c28: + roles: + agg: + args: + tensor-parallel-size: 4 + expert-parallel-size: 4 + enable-hierarchical-cache: true hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first + hicache-size: 310 + swa-full-tokens-ratio: 0.1 + schedule-policy: shortest-prefill-first + max-running-requests: 56 + cuda-graph-max-bs-decode: 56 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + prefill-decode-interval: 80 + mem-fraction-static: 0.86 + env: + PYTORCH_HIP_ALLOC_CONF: expandable_segments:True + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_AITER_HONOR_EXPLICIT_MEM_FRACTION: '1' + gpus: 4 benchmark: env: - CONC: '32' + CONC: '28' KV_OFFLOADING: dram -override_tp8_c48: +override_tp4_c36: roles: agg: args: + tensor-parallel-size: 4 + expert-parallel-size: 4 enable-hierarchical-cache: true - hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first + hicache-size: 310 + swa-full-tokens-ratio: 0.1 + schedule-policy: shortest-prefill-first + max-running-requests: 72 + cuda-graph-max-bs-decode: 72 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + prefill-decode-interval: 24 + mem-fraction-static: 0.86 + env: + PYTORCH_HIP_ALLOC_CONF: expandable_segments:True + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_AITER_HONOR_EXPLICIT_MEM_FRACTION: '1' + gpus: 4 benchmark: env: - CONC: '48' + CONC: '36' KV_OFFLOADING: dram -override_tp8_c64: +override_tp4_c40: roles: agg: args: + tensor-parallel-size: 4 + expert-parallel-size: 4 enable-hierarchical-cache: true - hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first + hicache-size: 310 + mem-fraction-static: 0.86 + swa-full-tokens-ratio: 0.1 + schedule-policy: shortest-prefill-first + max-running-requests: 80 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + prefill-decode-interval: 80 + env: + SGLANG_AITER_HONOR_EXPLICIT_MEM_FRACTION: '1' + PYTORCH_HIP_ALLOC_CONF: expandable_segments:True + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + gpus: 4 benchmark: env: - CONC: '64' + CONC: '40' KV_OFFLOADING: dram diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index ed18bec2d6..27debe3e43 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1698,7 +1698,7 @@ dsv41flash-fp4-mi355x-sglang-agentic-dspark: - { tp: 2, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [32, 64], srt-recipe: agentic.yaml } musespark-fp4-mi355x-sglang-agentic-dflash: - image: lmsysorg/sglang-rocm:muse-spark-open-1020@sha256:7343002d0dc2c1e525c0563f0c5c0377b471f966b3d604343b93a6366c2f7935 + image: lmsysorg/sglang-rocm:muse-spark-open-1027@sha256:7184d3ae2bfdc75a5bb8eb497b768e370ef7b213879cb4d97fbaaa3e8e5f5040 model: amd/Muse-Spark-Open-816B-A42B-Quark-MXFP4 model-prefix: musespark runner: cluster:mi355x-amds @@ -1710,8 +1710,8 @@ musespark-fp4-mi355x-sglang-agentic-dflash: agentic-coding: - dram-utilization: 0.85 search-space: - - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8], srt-recipe: agentic.yaml } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [16, 32, 48, 64], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 8, 13], srt-recipe: agentic.yaml } + - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [10, 16, 24, 28, 36, 40], srt-recipe: agentic.yaml } # Native Kimi-K3 PD with TP8 workers and MoRIIO READ. kimik3-fp4-mi355x-vllm-disagg-agentic: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 907bd21213..431af32a67 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11489,3 +11489,12 @@ - "Update the MI355X DeepSeek-V4-Pro-0813 SGLang MoRI disaggregated AgentX serving image from lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261019 to lmsysorg/sglang-rocm:v0.5.22-rocm10-mi35x-20261026, the latest published MI35x ROCm 10 nightly (digest sha256:edabad83728ab04e3eae39f315eb16261eeb64dbdb051112bd22fc88560b60c4)." - "Preserve all eight concurrency points (1, 16, 32, 48, 192, 256, 384, 512), prefill/decode topology, DSpark settings, UMBP offload, MoRI transfer, and benchmark client image. The workload binder supplies the serving image to every selected recipe variant." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4571 + +- config-keys: + - musespark-fp4-mi355x-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Bump to lmsysorg/sglang-rocm:muse-spark-open-1027." + - "Concurrency 1, 4, 8 and 13: TP8/EP8 with the AMD monokernel backend and GPU-only KV; 10 to 40: TP4/EP4 with a 253 to 310 GB-per-rank HiCache DRAM tier." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4606