From fb6378fab86120b6c2257f1cf124bb0c196fbb96 Mon Sep 17 00:00:00 2001 From: Guanbao Yu Date: Mon, 19 Oct 2026 04:34:08 -0500 Subject: [PATCH 1/9] feat(agentx): Kimi-K3 ATOM mono decode at C1/C2 and on-demand state cache at C48+ - Concurrency 1 and the new concurrency 2 run ATOM mono decode, with the fp32 SSM state and BF16 projection excludes it requires; every other point pins ATOM_MONO_ENABLE=0. - Concurrency 48, 56 and 72 enable on-demand state checkpoints, a 224 GB per-rank LMCache pool and a 128 GB state cap inside it (dram-utilization 0.6 -> TOTAL_CPU_DRAM_GB 1799). Image unchanged: mono at C1/C2 needs ROCm/ATOM#2626, which is not yet in a nightly. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../kimik3/atom/mi355x-fp4-mtp/agentic.yaml | 40 ++++++++++++++++--- inferencex-e2e/configs/amd-master.yaml | 23 ++++++----- inferencex-e2e/perf-changelog.yaml | 10 +++++ 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml index 1884b3a997..6f9b29930c 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml @@ -59,6 +59,8 @@ base: # Anchor-only state checkpointing. ATOM_STATE_CHECKPOINT_DEMAND: '0' ATOM_GDN_SSM_DTYPE: fp16 + # ATOM defaults mono decode on; only concurrency 1 and 2 opt in. + ATOM_MONO_ENABLE: '0' # FlyDSL FP8 prefill attention; ATOM defaults it off. ATOM_USE_FLYDSL_FP8_PREFILL_ATTN: '1' benchmark: @@ -79,10 +81,28 @@ override_tp8_c1: agg: args: cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' + # Mono loads in_proj, f_b_proj, o_proj and the two shared-expert + # projections as plain BF16 and refuses them quantized. + online_quant_config: &mono_quant '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*self_attn.[qkv]_conv1d*","*block_sparse_moe.experts*","*block_sparse_moe.routed_expert_*","*vision_tower*","*mm_projector*","layers.*.self_attn.fused_qkv_a_proj","layers.*.self_attn.q_b_proj","layers.*.self_attn.kv_b_proj","layers.*.self_attn.o_proj","layers.*.mlp.gate_up_proj","layers.*.mlp.down_proj","context_proj","*self_attn.in_proj*","*self_attn.f_b_proj*","*self_attn.o_proj*","*shared_experts.gate_up_proj*","*shared_experts.down_proj*"]}' + env: &mono_env + ATOM_MONO_ENABLE: '1' + # The mono KDA kernel reads the SSM state pool as fp32 and refuses fp16. + ATOM_GDN_SSM_DTYPE: fp32 benchmark: env: CONC: '1' +override_tp8_c2: + roles: + agg: + args: + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' + online_quant_config: *mono_quant + env: *mono_env + benchmark: + env: + CONC: '2' + override_tp8_c4: roles: agg: @@ -159,7 +179,11 @@ override_tp8_dcp8_c48_lmcache: extra-kv-connectors: *lmcache_offload env: <<: *lmcache_env - LMCACHE_MAX_LOCAL_CPU_SIZE: '128' + # On-demand state checkpoints at shared-prefix targets. The state cap + # is a limit inside the shared LMCache pool, not a second pool. + ATOM_STATE_CHECKPOINT_DEMAND: '1' + LMCACHE_MAX_LOCAL_CPU_SIZE: '224' + OFFLOAD_STATE_CPU_SIZE: '128' ATOM_ENABLE_REPLAYSSM: '1' ATOM_PREFILL_DECODE_INTERVAL: '4' ATOM_PREFILL_DELAYER_MAX_QUEUE_MS: '5000' @@ -168,7 +192,7 @@ override_tp8_dcp8_c48_lmcache: CONC: '48' SPEC_DECODING: mtp KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1028' + TOTAL_CPU_DRAM_GB: '1799' override_tp8_dcp8_c56_lmcache: roles: @@ -181,7 +205,9 @@ override_tp8_dcp8_c56_lmcache: extra-kv-connectors: *lmcache_offload env: <<: *lmcache_env - LMCACHE_MAX_LOCAL_CPU_SIZE: '192' + ATOM_STATE_CHECKPOINT_DEMAND: '1' + LMCACHE_MAX_LOCAL_CPU_SIZE: '224' + OFFLOAD_STATE_CPU_SIZE: '128' ATOM_ENABLE_REPLAYSSM: '1' AITER_REUSE_IDENTICAL_COMM_GROUPS: '1' ATOM_PREFILL_DECODE_INTERVAL: '4' @@ -191,7 +217,7 @@ override_tp8_dcp8_c56_lmcache: CONC: '56' SPEC_DECODING: mtp KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1538' + TOTAL_CPU_DRAM_GB: '1799' override_tp8_dcp8_c72_lmcache: roles: @@ -204,7 +230,9 @@ override_tp8_dcp8_c72_lmcache: extra-kv-connectors: *lmcache_offload env: <<: *lmcache_env - LMCACHE_MAX_LOCAL_CPU_SIZE: '192' + ATOM_STATE_CHECKPOINT_DEMAND: '1' + LMCACHE_MAX_LOCAL_CPU_SIZE: '224' + OFFLOAD_STATE_CPU_SIZE: '128' ATOM_ENABLE_REPLAYSSM: '1' AITER_REUSE_IDENTICAL_COMM_GROUPS: '1' ATOM_PREFILL_DECODE_INTERVAL: '4' @@ -214,4 +242,4 @@ override_tp8_dcp8_c72_lmcache: CONC: '72' SPEC_DECODING: mtp KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1538' + TOTAL_CPU_DRAM_GB: '1799' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index a8ea888081..8c7c64324d 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -696,24 +696,25 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding. Acceptance is pinned to the committed golden curve in # infx/golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml -# at both draft lengths used here: 7 draft tokens -> AL 3.84 at concurrency 1 and 4, +# at both draft lengths used here: 7 draft tokens -> AL 3.84 at concurrency 1, 2 and 4, # 4 draft tokens -> AL 3.36 from concurrency 14 up. # Companion to kimik3-fp4-mi355x-vllm-agentic-mtp: same checkpoint, same runner. # TP8 only -- the 1.56 TB MXFP4 checkpoint is ~195 GB/GPU and TP4 cannot load, # which is also why there is no DP-attention arm. # Three serving bands, tracking the ATOM recipe: -# interactive (1, 4) the latency floor, GPU-resident, dcp 1. +# interactive (1, 2, 4) the latency floor, GPU-resident, dcp 1; 1 and 2 +# run mono decode. # mid (14, 16) decode is KV-bandwidth-bound over 100k+ token # contexts, so dcp-size 8 shards the KV read across # all 8 GPUs and the LMCache DRAM tier backs the # paged KV. -# throughput (48, 56, 72) same shape, with the draft model. +# throughput (48, 56, 72) same shape, with on-demand state checkpoints +# and a 128 GB state cap inside the LMCache pool. # Two utilization blocks because the per-rank LMCache pool differs, and the # launcher divides the aggregate budget by TP: -# 0.343 -> 1028 GB aggregate -> 128 GB/rank (conc 14 through 48). -# 0.513 -> 1538 GB aggregate -> 192 GB/rank (conc 56 and 72), where the -# wider in-flight window needs the deeper pool to keep the paged KV -# resident. +# 0.343 -> 1028 GB aggregate -> 128 GB/rank (conc 14 and 16). +# 0.6 -> 1799 GB aggregate -> 224 GB/rank (conc 48, 56 and 72), so the +# paged KV keeps ~96 GB/rank once the state tier reaches its cap. kimik3-fp4-mi355x-atom-agentic-mtp: image: rocm/atom-dev:nightly_202610131505 model: moonshotai/Kimi-K3 @@ -727,11 +728,11 @@ kimik3-fp4-mi355x-atom-agentic-mtp: agentic-coding: - dram-utilization: 0.343 search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } - - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [14, 16, 48], spec-decoding: mtp, srt-recipe: agentic.yaml } - - dram-utilization: 0.513 + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [14, 16], spec-decoding: mtp, srt-recipe: agentic.yaml } + - dram-utilization: 0.6 search-space: - - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [56, 72], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [48, 56, 72], spec-decoding: mtp, srt-recipe: agentic.yaml } # Kimi-K3 MXFP4 agentic-coding on MI355X via vLLM with DSpark speculative # decoding. Companion to kimik3-fp4-mi355x-atom-agentic-mtp: same checkpoint and diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index b88d216b21..2b1f8db88f 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10139,3 +10139,13 @@ - "Re-sweep Muse Spark Open B200 AgentX on the full uncapped trace corpus (semianalysis_cc_traces_weka_062126) after #4287 dropped the Muse Spark 256k exception; image and search space are unchanged." - "Set HF_HUB_OFFLINE=1 so every rank resolves the DFlash2 draft from the shared HF cache, matching MI355X; concurrent launches otherwise hit the Hub 429 API rate limit." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4291 + +- config-keys: + - kimik3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add concurrency 2; the published set becomes [1, 2, 4, 14, 16, 48, 56, 72]. Concurrency 2 runs the 7-token DSpark draft at AL 3.84 with a 1-32 CUDA-graph ladder." + - "Serve concurrency 1 and 2 with ATOM mono decode (ATOM_MONO_ENABLE=1). Mono requires fp32 SSM state (ATOM_GDN_SSM_DTYPE=fp32) and BF16 in_proj, f_b_proj, o_proj and shared-expert projections, which are added to the online-quant exclude list. Every other point sets ATOM_MONO_ENABLE=0." + - "At concurrency 48, 56 and 72, enable on-demand state checkpoints (ATOM_STATE_CHECKPOINT_DEMAND=1), raise the per-rank LMCache pool to 224 GB, and cap retained state at 128 GB inside that pool (OFFLOAD_STATE_CPU_SIZE=128). dram-utilization 0.6 gives TOTAL_CPU_DRAM_GB 1799." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 038c3bc88fadfa21333aa2c3840ab6ca6c169ea3 Mon Sep 17 00:00:00 2001 From: Guanbao Yu Date: Mon, 19 Oct 2026 04:38:36 -0500 Subject: [PATCH 2/9] chore(changelog): set the pr-link for #4307 Co-Authored-By: Claude Opus 5.5 (1M context) --- inferencex-e2e/perf-changelog.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 2b1f8db88f..0d2af5e612 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4307 - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp @@ -9724,7 +9724,7 @@ - "Replace the TP4/EP1 and TP2/EP1 C1-C128 sweeps with TP4/EP4 points at concurrency 1, 4, 8 and 16, plus aggregated TP4/EP4 points with DP attention (DP4) at concurrency 32, 64, 128 and 192 behind sglang-router 0.3.2." - "Apply the point-specific prefill-decode interval, prefill chunking, admission, memory and SWA prefix-tail settings from the single-node recipe; all points use the default fcfs schedule policy." - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4.1-Flash runs as shipped with block size 5." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4307 - config-keys: - kimik3-fp4-mi355x-vllm-disagg-agentic @@ -10148,4 +10148,4 @@ - "Add concurrency 2; the published set becomes [1, 2, 4, 14, 16, 48, 56, 72]. Concurrency 2 runs the 7-token DSpark draft at AL 3.84 with a 1-32 CUDA-graph ladder." - "Serve concurrency 1 and 2 with ATOM mono decode (ATOM_MONO_ENABLE=1). Mono requires fp32 SSM state (ATOM_GDN_SSM_DTYPE=fp32) and BF16 in_proj, f_b_proj, o_proj and shared-expert projections, which are added to the online-quant exclude list. Every other point sets ATOM_MONO_ENABLE=0." - "At concurrency 48, 56 and 72, enable on-demand state checkpoints (ATOM_STATE_CHECKPOINT_DEMAND=1), raise the per-rank LMCache pool to 224 GB, and cap retained state at 128 GB inside that pool (OFFLOAD_STATE_CPU_SIZE=128). dram-utilization 0.6 gives TOTAL_CPU_DRAM_GB 1799." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4307 From b8fb29020f831e684719d9dccf51baad091b2198 Mon Sep 17 00:00:00 2001 From: Guanbao Yu Date: Mon, 19 Oct 2026 04:39:06 -0500 Subject: [PATCH 3/9] fix(changelog): restore the two existing XXX placeholders Only this PR's entry takes the #4307 link. Co-Authored-By: Claude Opus 5.5 (1M context) --- inferencex-e2e/perf-changelog.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 0d2af5e612..0d0e607d0e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4307 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp @@ -9724,7 +9724,7 @@ - "Replace the TP4/EP1 and TP2/EP1 C1-C128 sweeps with TP4/EP4 points at concurrency 1, 4, 8 and 16, plus aggregated TP4/EP4 points with DP attention (DP4) at concurrency 32, 64, 128 and 192 behind sglang-router 0.3.2." - "Apply the point-specific prefill-decode interval, prefill chunking, admission, memory and SWA prefix-tail settings from the single-node recipe; all points use the default fcfs schedule policy." - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4.1-Flash runs as shipped with block size 5." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4307 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - kimik3-fp4-mi355x-vllm-disagg-agentic From 163620693d7d9ab599a0da5b147e02b0a134a6f3 Mon Sep 17 00:00:00 2001 From: Guanbao Yu Date: Mon, 19 Oct 2026 10:46:27 -0500 Subject: [PATCH 4/9] feat(agentx): move Kimi-K3 ATOM AgentX to nightly_202610191501 ATOM 53e4fb524 carries Kimi-K3 mono decode and the one-token f_b_proj fix it depends on (ROCm/ATOM#2626). Recipe frontend pinned to the tag's digest. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml | 2 +- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml index 6f9b29930c..f90922d828 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884 + container_image: rocm/atom-dev@sha256:566aa3dc9d5d98a737f9f5c1b775a651e44dfd8e878ab77f2536a76df21f687e enable_multiple_frontends: false observability: enabled: false diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 8c7c64324d..93dad58248 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -716,7 +716,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: # 0.6 -> 1799 GB aggregate -> 224 GB/rank (conc 48, 56 and 72), so the # paged KV keeps ~96 GB/rank once the state tier reaches its cap. kimik3-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202610131505 + image: rocm/atom-dev:nightly_202610191501 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 8e2ce7ceb7..7d952197a5 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10205,6 +10205,7 @@ scenario-type: - agentic-coding description: + - "Move the image from rocm/atom-dev:nightly_202610131505 (ATOM 661ef7661) to rocm/atom-dev:nightly_202610191501 (ATOM 53e4fb524), and pin the recipe frontend to that tag's digest sha256:566aa3dc9d5d98a737f9f5c1b775a651e44dfd8e878ab77f2536a76df21f687e. The new image adds Kimi-K3 mono decode and the one-token f_b_proj fix it depends on (ROCm/ATOM#2626)." - "Add concurrency 2; the published set becomes [1, 2, 4, 14, 16, 48, 56, 72]. Concurrency 2 runs the 7-token DSpark draft at AL 3.84 with a 1-32 CUDA-graph ladder." - "Serve concurrency 1 and 2 with ATOM mono decode (ATOM_MONO_ENABLE=1). Mono requires fp32 SSM state (ATOM_GDN_SSM_DTYPE=fp32) and BF16 in_proj, f_b_proj, o_proj and shared-expert projections, which are added to the online-quant exclude list. Every other point sets ATOM_MONO_ENABLE=0." - "At concurrency 48, 56 and 72, enable on-demand state checkpoints (ATOM_STATE_CHECKPOINT_DEMAND=1), raise the per-rank LMCache pool to 224 GB, and cap retained state at 128 GB inside that pool (OFFLOAD_STATE_CPU_SIZE=128). dram-utilization 0.6 gives TOTAL_CPU_DRAM_GB 1799." From 76a2b7edfce651f5866a2c65d0a97ae101b939da Mon Sep 17 00:00:00 2001 From: Guanbao Yu Date: Tue, 20 Oct 2026 22:02:17 -0500 Subject: [PATCH 5/9] feat(agentx): drop the Kimi-K3 ATOM concurrency 2 point Concurrency 1 keeps mono decode; the published set returns to [1, 4, 14, 16, 48, 56, 72]. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../kimik3/atom/mi355x-fp4-mtp/agentic.yaml | 17 +++-------------- inferencex-e2e/configs/amd-master.yaml | 8 ++++---- inferencex-e2e/perf-changelog.yaml | 3 +-- 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml index f90922d828..af8a866b00 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml @@ -59,7 +59,7 @@ base: # Anchor-only state checkpointing. ATOM_STATE_CHECKPOINT_DEMAND: '0' ATOM_GDN_SSM_DTYPE: fp16 - # ATOM defaults mono decode on; only concurrency 1 and 2 opt in. + # ATOM defaults mono decode on; only concurrency 1 opts in. ATOM_MONO_ENABLE: '0' # FlyDSL FP8 prefill attention; ATOM defaults it off. ATOM_USE_FLYDSL_FP8_PREFILL_ATTN: '1' @@ -83,8 +83,8 @@ override_tp8_c1: cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' # Mono loads in_proj, f_b_proj, o_proj and the two shared-expert # projections as plain BF16 and refuses them quantized. - online_quant_config: &mono_quant '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*self_attn.[qkv]_conv1d*","*block_sparse_moe.experts*","*block_sparse_moe.routed_expert_*","*vision_tower*","*mm_projector*","layers.*.self_attn.fused_qkv_a_proj","layers.*.self_attn.q_b_proj","layers.*.self_attn.kv_b_proj","layers.*.self_attn.o_proj","layers.*.mlp.gate_up_proj","layers.*.mlp.down_proj","context_proj","*self_attn.in_proj*","*self_attn.f_b_proj*","*self_attn.o_proj*","*shared_experts.gate_up_proj*","*shared_experts.down_proj*"]}' - env: &mono_env + online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*self_attn.[qkv]_conv1d*","*block_sparse_moe.experts*","*block_sparse_moe.routed_expert_*","*vision_tower*","*mm_projector*","layers.*.self_attn.fused_qkv_a_proj","layers.*.self_attn.q_b_proj","layers.*.self_attn.kv_b_proj","layers.*.self_attn.o_proj","layers.*.mlp.gate_up_proj","layers.*.mlp.down_proj","context_proj","*self_attn.in_proj*","*self_attn.f_b_proj*","*self_attn.o_proj*","*shared_experts.gate_up_proj*","*shared_experts.down_proj*"]}' + env: ATOM_MONO_ENABLE: '1' # The mono KDA kernel reads the SSM state pool as fp32 and refuses fp16. ATOM_GDN_SSM_DTYPE: fp32 @@ -92,17 +92,6 @@ override_tp8_c1: env: CONC: '1' -override_tp8_c2: - roles: - agg: - args: - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]' - online_quant_config: *mono_quant - env: *mono_env - benchmark: - env: - CONC: '2' - override_tp8_c4: roles: agg: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index a2b37d6138..d588207b51 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -696,14 +696,14 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding. Acceptance is pinned to the committed golden curve in # infx/golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml -# at both draft lengths used here: 7 draft tokens -> AL 3.84 at concurrency 1, 2 and 4, +# at both draft lengths used here: 7 draft tokens -> AL 3.84 at concurrency 1 and 4, # 4 draft tokens -> AL 3.36 from concurrency 14 up. # Companion to kimik3-fp4-mi355x-vllm-agentic-mtp: same checkpoint, same runner. # TP8 only -- the 1.56 TB MXFP4 checkpoint is ~195 GB/GPU and TP4 cannot load, # which is also why there is no DP-attention arm. # Three serving bands, tracking the ATOM recipe: -# interactive (1, 2, 4) the latency floor, GPU-resident, dcp 1; 1 and 2 -# run mono decode. +# interactive (1, 4) the latency floor, GPU-resident, dcp 1; 1 runs +# mono decode. # mid (14, 16) decode is KV-bandwidth-bound over 100k+ token # contexts, so dcp-size 8 shards the KV read across # all 8 GPUs and the LMCache DRAM tier backs the @@ -728,7 +728,7 @@ kimik3-fp4-mi355x-atom-agentic-mtp: agentic-coding: - dram-utilization: 0.343 search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 8, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } - { tp: 8, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [14, 16], spec-decoding: mtp, srt-recipe: agentic.yaml } - dram-utilization: 0.6 search-space: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 5d85357f49..1a342e8f57 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10328,7 +10328,6 @@ - agentic-coding description: - "Move the image from rocm/atom-dev:nightly_202610131505 (ATOM 661ef7661) to rocm/atom-dev:nightly_202610191501 (ATOM 53e4fb524), and pin the recipe frontend to that tag's digest sha256:566aa3dc9d5d98a737f9f5c1b775a651e44dfd8e878ab77f2536a76df21f687e. The new image adds Kimi-K3 mono decode and the one-token f_b_proj fix it depends on (ROCm/ATOM#2626)." - - "Add concurrency 2; the published set becomes [1, 2, 4, 14, 16, 48, 56, 72]. Concurrency 2 runs the 7-token DSpark draft at AL 3.84 with a 1-32 CUDA-graph ladder." - - "Serve concurrency 1 and 2 with ATOM mono decode (ATOM_MONO_ENABLE=1). Mono requires fp32 SSM state (ATOM_GDN_SSM_DTYPE=fp32) and BF16 in_proj, f_b_proj, o_proj and shared-expert projections, which are added to the online-quant exclude list. Every other point sets ATOM_MONO_ENABLE=0." + - "Serve concurrency 1 with ATOM mono decode (ATOM_MONO_ENABLE=1). Mono requires fp32 SSM state (ATOM_GDN_SSM_DTYPE=fp32) and BF16 in_proj, f_b_proj, o_proj and shared-expert projections, which are added to the online-quant exclude list. Every other point sets ATOM_MONO_ENABLE=0." - "At concurrency 48, 56 and 72, enable on-demand state checkpoints (ATOM_STATE_CHECKPOINT_DEMAND=1), raise the per-rank LMCache pool to 224 GB, and cap retained state at 128 GB inside that pool (OFFLOAD_STATE_CPU_SIZE=128). dram-utilization 0.6 gives TOTAL_CPU_DRAM_GB 1799." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4307 From e88d648e2aa3e0323e236ec2931a5c86d8896a92 Mon Sep 17 00:00:00 2001 From: Guanbao Yu Date: Tue, 20 Oct 2026 22:17:59 -0500 Subject: [PATCH 6/9] feat(agentx): enable AITER FlyDSL herd at every Kimi-K3 ATOM band AITER_FLYDSL_USE_HERD=1 and AITER_FLYDSL_HERD_MIN_M=64 in the recipe base env. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml | 2 ++ inferencex-e2e/perf-changelog.yaml | 1 + 2 files changed, 3 insertions(+) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml index af8a866b00..4d0d336dcc 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml @@ -56,6 +56,8 @@ base: AITER_SITUV2_A4W4: '1' AITER_QUICK_REDUCE_QUANTIZATION: INT4 AITER_FLYDSL_STAGE2_FP8: '1' + AITER_FLYDSL_USE_HERD: '1' + AITER_FLYDSL_HERD_MIN_M: '64' # Anchor-only state checkpointing. ATOM_STATE_CHECKPOINT_DEMAND: '0' ATOM_GDN_SSM_DTYPE: fp16 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 9825fb3e14..69ff49f5b1 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10499,5 +10499,6 @@ description: - "Move the image from rocm/atom-dev:nightly_202610131505 (ATOM 661ef7661) to rocm/atom-dev:nightly_202610191501 (ATOM 53e4fb524), and pin the recipe frontend to that tag's digest sha256:566aa3dc9d5d98a737f9f5c1b775a651e44dfd8e878ab77f2536a76df21f687e. The new image adds Kimi-K3 mono decode and the one-token f_b_proj fix it depends on (ROCm/ATOM#2626)." - "Serve concurrency 1 with ATOM mono decode (ATOM_MONO_ENABLE=1). Mono requires fp32 SSM state (ATOM_GDN_SSM_DTYPE=fp32) and BF16 in_proj, f_b_proj, o_proj and shared-expert projections, which are added to the online-quant exclude list. Every other point sets ATOM_MONO_ENABLE=0." + - "Set AITER_FLYDSL_USE_HERD=1 and AITER_FLYDSL_HERD_MIN_M=64 at every concurrency." - "At concurrency 48, 56 and 72, enable on-demand state checkpoints (ATOM_STATE_CHECKPOINT_DEMAND=1), raise the per-rank LMCache pool to 224 GB, and cap retained state at 128 GB inside that pool (OFFLOAD_STATE_CPU_SIZE=128). dram-utilization 0.6 gives TOTAL_CPU_DRAM_GB 1799." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4307 From bcfbfe18fac0885abfd823994b5545d3c2b938b5 Mon Sep 17 00:00:00 2001 From: Guanbao Yu Date: Mon, 26 Oct 2026 10:07:31 -0500 Subject: [PATCH 7/9] feat(agentx): serve Kimi-K3 ATOM concurrency 1 one request at a time PrefillDelayer (ATOM_PREFILL_DECODE_INTERVAL=1000000), max-num-seqs 2 and CUDA-graph sizes [1, 2], per the Kimi-K3 recipe in ROCm/ATOM#2659. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../kimik3/atom/mi355x-fp4-mtp/agentic.yaml | 7 ++++++- inferencex-e2e/perf-changelog.yaml | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml index 4d0d336dcc..e1d6c40f6c 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml @@ -82,7 +82,9 @@ override_tp8_c1: roles: agg: args: - cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' + # max-num-seqs 1 fails the DSpark drafter's warmup, which runs two sequences. + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' # Mono loads in_proj, f_b_proj, o_proj and the two shared-expert # projections as plain BF16 and refuses them quantized. online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*self_attn.[qkv]_conv1d*","*block_sparse_moe.experts*","*block_sparse_moe.routed_expert_*","*vision_tower*","*mm_projector*","layers.*.self_attn.fused_qkv_a_proj","layers.*.self_attn.q_b_proj","layers.*.self_attn.kv_b_proj","layers.*.self_attn.o_proj","layers.*.mlp.gate_up_proj","layers.*.mlp.down_proj","context_proj","*self_attn.in_proj*","*self_attn.f_b_proj*","*self_attn.o_proj*","*shared_experts.gate_up_proj*","*shared_experts.down_proj*"]}' @@ -90,6 +92,9 @@ override_tp8_c1: ATOM_MONO_ENABLE: '1' # The mono KDA kernel reads the SSM state pool as fp32 and refuses fp16. ATOM_GDN_SSM_DTYPE: fp32 + # Holds new prefill while any request decodes, so requests mostly decode + # one at a time. + ATOM_PREFILL_DECODE_INTERVAL: '1000000' benchmark: env: CONC: '1' diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 69ff49f5b1..edbfabe590 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10499,6 +10499,7 @@ description: - "Move the image from rocm/atom-dev:nightly_202610131505 (ATOM 661ef7661) to rocm/atom-dev:nightly_202610191501 (ATOM 53e4fb524), and pin the recipe frontend to that tag's digest sha256:566aa3dc9d5d98a737f9f5c1b775a651e44dfd8e878ab77f2536a76df21f687e. The new image adds Kimi-K3 mono decode and the one-token f_b_proj fix it depends on (ROCm/ATOM#2626)." - "Serve concurrency 1 with ATOM mono decode (ATOM_MONO_ENABLE=1). Mono requires fp32 SSM state (ATOM_GDN_SSM_DTYPE=fp32) and BF16 in_proj, f_b_proj, o_proj and shared-expert projections, which are added to the online-quant exclude list. Every other point sets ATOM_MONO_ENABLE=0." + - "At concurrency 1, run one request at a time: the PrefillDelayer (ATOM_PREFILL_DECODE_INTERVAL=1000000) holds new prefill while a request decodes, with max-num-seqs 2 and CUDA-graph sizes [1, 2] (max-num-seqs 1 fails the DSpark drafter warmup)." - "Set AITER_FLYDSL_USE_HERD=1 and AITER_FLYDSL_HERD_MIN_M=64 at every concurrency." - "At concurrency 48, 56 and 72, enable on-demand state checkpoints (ATOM_STATE_CHECKPOINT_DEMAND=1), raise the per-rank LMCache pool to 224 GB, and cap retained state at 128 GB inside that pool (OFFLOAD_STATE_CPU_SIZE=128). dram-utilization 0.6 gives TOTAL_CPU_DRAM_GB 1799." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4307 From b050be2cc238c08aa728ce83faca36d43329f55c Mon Sep 17 00:00:00 2001 From: Guanbao Yu Date: Mon, 26 Oct 2026 11:09:06 -0500 Subject: [PATCH 8/9] feat(agentx): move Kimi-K3 ATOM AgentX to nightly_202610261558 ATOM e893780f4 carries ROCm/ATOM#2659 (16-row Kimi-K3 mono, K2 on the MLA layers). Recipe frontend pinned to the tag's digest. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml | 2 +- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml index e1d6c40f6c..b28e5a5f82 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:566aa3dc9d5d98a737f9f5c1b775a651e44dfd8e878ab77f2536a76df21f687e + container_image: rocm/atom-dev@sha256:57477e7a4b5156fcc7b21bab082eba98577752465f58479ac2ead8514f882a61 enable_multiple_frontends: false observability: enabled: false diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 6205bcdbf8..23bdf7afcc 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -736,7 +736,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: # 0.6 -> 1799 GB aggregate -> 224 GB/rank (conc 48, 56 and 72), so the # paged KV keeps ~96 GB/rank once the state tier reaches its cap. kimik3-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202610191501 + image: rocm/atom-dev:nightly_202610261558 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 6b0588c807..9b26e77829 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11272,7 +11272,7 @@ scenario-type: - agentic-coding description: - - "Move the image from rocm/atom-dev:nightly_202610131505 (ATOM 661ef7661) to rocm/atom-dev:nightly_202610191501 (ATOM 53e4fb524), and pin the recipe frontend to that tag's digest sha256:566aa3dc9d5d98a737f9f5c1b775a651e44dfd8e878ab77f2536a76df21f687e. The new image adds Kimi-K3 mono decode and the one-token f_b_proj fix it depends on (ROCm/ATOM#2626)." + - "Move the image from rocm/atom-dev:nightly_202610131505 (ATOM 661ef7661) to rocm/atom-dev:nightly_202610261558 (ATOM e893780f4), and pin the recipe frontend to that tag's digest sha256:57477e7a4b5156fcc7b21bab082eba98577752465f58479ac2ead8514f882a61. The new image adds Kimi-K3 mono decode and the one-token f_b_proj fix it depends on (ROCm/ATOM#2626), and widens mono to 16-row steps with K2 on the MLA layers (ROCm/ATOM#2659)." - "Serve concurrency 1 with ATOM mono decode (ATOM_MONO_ENABLE=1). Mono requires fp32 SSM state (ATOM_GDN_SSM_DTYPE=fp32) and BF16 in_proj, f_b_proj, o_proj and shared-expert projections, which are added to the online-quant exclude list. Every other point sets ATOM_MONO_ENABLE=0." - "At concurrency 1, run one request at a time: the PrefillDelayer (ATOM_PREFILL_DECODE_INTERVAL=1000000) holds new prefill while a request decodes, with max-num-seqs 2 and CUDA-graph sizes [1, 2] (max-num-seqs 1 fails the DSpark drafter warmup)." - "Set AITER_FLYDSL_USE_HERD=1 and AITER_FLYDSL_HERD_MIN_M=64 at every concurrency." From 8351e8241a6579e7396e9d7e894964ae5823eedc Mon Sep 17 00:00:00 2001 From: Guanbao Yu Date: Wed, 28 Oct 2026 11:22:13 -0500 Subject: [PATCH 9/9] feat(agentx): move Kimi-K3 ATOM AgentX to nightly_202610281457 Its aiter carries ROCm/aiter#7020, which keeps the tuned Kimi-K3 MXFP4 inline-sort MoE rows (C4/C14/C16 decode) under ATOM's real row count. Recipe frontend pinned to the tag's digest. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml | 2 +- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml index b28e5a5f82..8987cdd519 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:57477e7a4b5156fcc7b21bab082eba98577752465f58479ac2ead8514f882a61 + container_image: rocm/atom-dev@sha256:96a9614276518c2619e88800dec5f276bc76a95691331b4abb2210184e1431af enable_multiple_frontends: false observability: enabled: false diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 7708dad92b..6246f103e6 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -736,7 +736,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: # 0.6 -> 1799 GB aggregate -> 224 GB/rank (conc 48, 56 and 72), so the # paged KV keeps ~96 GB/rank once the state tier reaches its cap. kimik3-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202610261558 + image: rocm/atom-dev:nightly_202610281457 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 021f459eaf..a03ff605e1 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11505,7 +11505,7 @@ scenario-type: - agentic-coding description: - - "Move the image from rocm/atom-dev:nightly_202610131505 (ATOM 661ef7661) to rocm/atom-dev:nightly_202610261558 (ATOM e893780f4), and pin the recipe frontend to that tag's digest sha256:57477e7a4b5156fcc7b21bab082eba98577752465f58479ac2ead8514f882a61. The new image adds Kimi-K3 mono decode and the one-token f_b_proj fix it depends on (ROCm/ATOM#2626), and widens mono to 16-row steps with K2 on the MLA layers (ROCm/ATOM#2659)." + - "Move the image from rocm/atom-dev:nightly_202610131505 (ATOM 661ef7661) to rocm/atom-dev:nightly_202610281457 (ATOM 06dbf6692), and pin the recipe frontend to that tag's digest sha256:96a9614276518c2619e88800dec5f276bc76a95691331b4abb2210184e1431af. The new image adds Kimi-K3 mono decode and the one-token f_b_proj fix it depends on (ROCm/ATOM#2626), and widens mono to 16-row steps with K2 on the MLA layers (ROCm/ATOM#2659). Its aiter keeps the tuned Kimi-K3 MXFP4 inline-sort MoE rows when ATOM passes the real row count (ROCm/aiter#7020)." - "Serve concurrency 1 with ATOM mono decode (ATOM_MONO_ENABLE=1). Mono requires fp32 SSM state (ATOM_GDN_SSM_DTYPE=fp32) and BF16 in_proj, f_b_proj, o_proj and shared-expert projections, which are added to the online-quant exclude list. Every other point sets ATOM_MONO_ENABLE=0." - "At concurrency 1, run one request at a time: the PrefillDelayer (ATOM_PREFILL_DECODE_INTERVAL=1000000) holds new prefill while a request decodes, with max-num-seqs 2 and CUDA-graph sizes [1, 2] (max-num-seqs 1 fails the DSpark drafter warmup)." - "Set AITER_FLYDSL_USE_HERD=1 and AITER_FLYDSL_HERD_MIN_M=64 at every concurrency."