diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..b14562284f --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,293 @@ +base: + schema: 2 + name: kimik3-fp4-mi355x-atom-disagg-agentic + slurm: + time_limit: '08:00:00' + sbatch_directives: + cpus-per-task: '128' + mem: '0' + srun_options: + mem: '0' + container-writable: '' + container-remap-root: '' + extra_mount: + - /it-share/data/Inferact-Kimi-K3-DSpark:/models/Inferact-Kimi-K3-DSpark + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: atomesh + enable_multiple_frontends: false + args: + policy: random + log-level: info + disable-circuit-breaker: true + prometheus-port: 29100 + observability: + enabled: false + tachometer: + enabled: false + health_check: + interval_seconds: 10 + # A cold 1.56 TB MXFP4 load from shared storage can take over an hour. + max_attempts: 720 + engine: atom + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: &env + # The image bakes in a build-host proxy that nothing serves on a node. + HTTP_PROXY: '' + HTTPS_PROXY: '' + http_proxy: '' + https_proxy: '' + NO_PROXY: localhost,127.0.0.1,::1 + no_proxy: localhost,127.0.0.1,::1 + PYTHONUNBUFFERED: '1' + PYTHONNOUSERSITE: '1' + PYTHONHASHSEED: '0' + AITER_LOG_LEVEL: WARNING + AITER_SITUV2_A4W4: '1' + AITER_QUICK_REDUCE_QUANTIZATION: INT4 + AITER_FLYDSL_STAGE2_FP8: '1' + AITER_REUSE_IDENTICAL_COMM_GROUPS: '0' + ATOM_GDN_SSM_DTYPE: fp16 + ATOM_USE_FLYDSL_GATHER_KV_B_PROJ: '1' + # FlyDSL FP8 prefill attention; ATOM defaults it off. + ATOM_USE_FLYDSL_FP8_PREFILL_ATTN: '1' + ATOM_ENABLE_REPLAYSSM: '0' + ATOM_STATE_CHECKPOINT_DEMAND: '0' + MC_GID_INDEX: '@fabric.rdma-gid-index' + NCCL_IB_DISABLE: '1' + args: &args + served-model-name: moonshotai/Kimi-K3 + trust-remote-code: true + # Fails closed if the chat template stops rendering a tools payload. + tool-call-parser: kimi_k3 + decode-context-parallel-size: 1 + kv_cache_dtype: fp8 + block-size: 128 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + gpu-memory-utilization: 0.9 + enable_prefix_caching: true + # -1 keeps the prompt-end checkpoint anchor without the fixed-interval grid. + state-checkpoint-interval-tokens: -1 + # Global ptpc_fp8 online quantization; the trailing entries keep the BF16 + # draft (unprefixed layers.* and context_proj) unquantized. + online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*self_attn.[qkv]_conv1d*","*block_sparse_moe.experts*","*block_sparse_moe.routed_expert_*","*vision_tower*","*mm_projector*","layers.*.self_attn.fused_qkv_a_proj","layers.*.self_attn.q_b_proj","layers.*.self_attn.kv_b_proj","layers.*.self_attn.o_proj","layers.*.mlp.gate_up_proj","layers.*.mlp.down_proj","context_proj"]}' + method: dspark + draft-model: /models/Inferact-Kimi-K3-DSpark + num-speculative-tokens: 7 + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: *env + args: + <<: *args + level: 3 + cudagraph-mode: FULL + benchmark: + env: + HTTP_PROXY: '' + HTTPS_PROXY: '' + http_proxy: '' + https_proxy: '' + NO_PROXY: localhost,127.0.0.1,::1 + no_proxy: localhost,127.0.0.1,::1 + # Long agentic turns against a 1M context are prefill-bound on the server. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' + AIPERF_APPLY_CHAT_TEMPLATE: 'true' + +override_1p1d_tp8_c1: + roles: + decode: + args: + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]' + +override_1p2d_dcp8_c16: + # A 4 KiB LMCache pool stays movable after hipHostRegister, and each migrated + # page stalls the rank's GPU queues until KFD re-validates the whole pool, so + # the pools use 2 MiB HugeTLB pages, reserved here on the head node (the + # prefill's): 4 pools per NUMA node, plus 400 GiB left free for model loading. + # A recipe host_setup replaces the cluster default, so setup.sh runs first. + host_setup: + nodes: all + timeout_seconds: 1800 + commands: + - &k3_host_setup_sh IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" + # 4 ranks x 128 GiB per NUMA node, in 2 MiB pages. + - pages_per_numa=262144 + - &k3_reserve_lmcache_hugepages >- + if hostname -I | tr ' ' '\n' | grep -qxF "${SLURM_LAUNCH_NODE_IPADDR:?srun did not set SLURM_LAUNCH_NODE_IPADDR}"; then + echo "[hugepage] $(hostname -s) is the head node; reserving the LMCache pools' pages"; + held=$(awk '/^HugePages_Total:/ {total = $2} /^HugePages_Free:/ {free = $2} /^HugePages_Rsvd:/ {rsvd = $2} END {print total - free + rsvd}' /proc/meminfo); + HUGEPAGES_TARGET=$(( held + 2 * pages_per_numa * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 + bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" || exit 1; + for node in 0 1; do + free=$(cat /sys/devices/system/node/node$node/hugepages/hugepages-2048kB/free_hugepages); + mem_gib=$(awk '/MemFree:/ {print int($4 / 1048576)}' /sys/devices/system/node/node$node/meminfo); + echo "[hugepage] NUMA$node: $free free 2 MiB pages, pools need $pages_per_numa; $mem_gib GiB other memory free, need 400"; + [[ $free -ge $pages_per_numa && $mem_gib -ge 400 ]] || exit 1; + done; fi + teardown: &k3_release_hugepages + - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" + roles: + prefill: + args: + decode-context-parallel-size: 8 + num-speculative-tokens: 7 + max-num-seqs: 32 + extra-kv-connectors: &lmcache_offload + - kv_connector: lmcache_offload + kv_role: offload + env: + <<: &lmcache_env + LMCACHE_LOCAL_CPU: 'True' + # The offload hash block is block-size 128 x DCP 8, so the KV and the + # state-checkpoint grids coincide. + LMCACHE_CHUNK_SIZE: '1024' + # Pin the eight ranks to the two sockets explicitly. + LMCACHE_NUMA_MODE: auto + # Back each pool with 2 MiB HugeTLB pages; host_setup reserves them. + LMCACHE_LOCAL_CPU_USE_HUGEPAGES: 'True' + ATOM_NUMA_BIND: '1' + ATOM_NUMA_NODE: 0,0,0,0,1,1,1,1 + ATOM_AUTO_NUMA_BIND: '0' + OFFLOAD_PROFILE: '1' + # The GPU staging buffer must hold one K3 KDA state entry (~30 MiB). + OFFLOAD_GPU_STAGING_CHUNKS: '32' + ATOM_ENABLE_REPLAYSSM: '1' + LMCACHE_MAX_LOCAL_CPU_SIZE: '128' + AITER_REUSE_IDENTICAL_COMM_GROUPS: '0' + ATOM_STATE_CHECKPOINT_DEMAND: '0' + decode: + nodes: 2 + workers: 2 + args: + decode-context-parallel-size: 8 + num-speculative-tokens: 7 + max-num-seqs: 32 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256]' + env: + ATOM_ENABLE_REPLAYSSM: '1' + AITER_REUSE_IDENTICAL_COMM_GROUPS: '0' + ATOM_STATE_CHECKPOINT_DEMAND: '0' + +override_1p1d_dcp8_c48: + host_setup: &k3_lmcache_224_host_setup + nodes: all + timeout_seconds: 1800 + commands: + - *k3_host_setup_sh + # 4 ranks x 224 GiB per NUMA node, in 2 MiB pages. + - pages_per_numa=458752 + - *k3_reserve_lmcache_hugepages + teardown: *k3_release_hugepages + roles: + prefill: + args: + decode-context-parallel-size: 8 + num-speculative-tokens: 4 + max-num-seqs: 96 + extra-kv-connectors: *lmcache_offload + env: + <<: *lmcache_env + LMCACHE_MAX_LOCAL_CPU_SIZE: '224' + OFFLOAD_STATE_CPU_SIZE: '128' + AITER_REUSE_IDENTICAL_COMM_GROUPS: '0' + ATOM_STATE_CHECKPOINT_DEMAND: '1' + decode: + args: + decode-context-parallel-size: 8 + num-speculative-tokens: 4 + max-num-seqs: 96 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480]' + env: + ATOM_ENABLE_REPLAYSSM: '1' + AITER_REUSE_IDENTICAL_COMM_GROUPS: '0' + ATOM_STATE_CHECKPOINT_DEMAND: '1' + +override_1p2d_dcp8_c48: + host_setup: *k3_lmcache_224_host_setup + roles: + prefill: + args: + decode-context-parallel-size: 8 + num-speculative-tokens: 4 + max-num-seqs: 96 + extra-kv-connectors: *lmcache_offload + env: + <<: *lmcache_env + LMCACHE_MAX_LOCAL_CPU_SIZE: '224' + OFFLOAD_STATE_CPU_SIZE: '128' + AITER_REUSE_IDENTICAL_COMM_GROUPS: '0' + ATOM_STATE_CHECKPOINT_DEMAND: '1' + decode: + nodes: 2 + workers: 2 + args: + decode-context-parallel-size: 8 + num-speculative-tokens: 4 + max-num-seqs: 96 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480]' + env: + ATOM_ENABLE_REPLAYSSM: '1' + AITER_REUSE_IDENTICAL_COMM_GROUPS: '0' + ATOM_STATE_CHECKPOINT_DEMAND: '1' + +override_1p1d_dcp8_c72: + host_setup: *k3_lmcache_224_host_setup + roles: + prefill: + args: + decode-context-parallel-size: 8 + num-speculative-tokens: 4 + max-num-seqs: 144 + extra-kv-connectors: *lmcache_offload + env: + <<: *lmcache_env + LMCACHE_MAX_LOCAL_CPU_SIZE: '224' + OFFLOAD_STATE_CPU_SIZE: '128' + AITER_REUSE_IDENTICAL_COMM_GROUPS: '1' + ATOM_STATE_CHECKPOINT_DEMAND: '1' + decode: + args: + decode-context-parallel-size: 8 + num-speculative-tokens: 4 + max-num-seqs: 144 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720]' + env: + ATOM_ENABLE_REPLAYSSM: '1' + AITER_REUSE_IDENTICAL_COMM_GROUPS: '1' + ATOM_STATE_CHECKPOINT_DEMAND: '1' + +override_1p1d_dcp8_c88: + host_setup: *k3_lmcache_224_host_setup + roles: + prefill: + args: + decode-context-parallel-size: 8 + num-speculative-tokens: 4 + max-num-seqs: 176 + extra-kv-connectors: *lmcache_offload + env: + <<: *lmcache_env + LMCACHE_MAX_LOCAL_CPU_SIZE: '224' + OFFLOAD_STATE_CPU_SIZE: '128' + AITER_REUSE_IDENTICAL_COMM_GROUPS: '1' + ATOM_STATE_CHECKPOINT_DEMAND: '1' + decode: + args: + decode-context-parallel-size: 8 + num-speculative-tokens: 4 + max-num-seqs: 176 + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880]' + env: + ATOM_ENABLE_REPLAYSSM: '1' + AITER_REUSE_IDENTICAL_COMM_GROUPS: '1' + ATOM_STATE_CHECKPOINT_DEMAND: '1' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index a8ea888081..7895943edb 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1727,3 +1727,66 @@ kimik3-fp4-mi355x-vllm-disagg-agentic: # Inherits the same P-only scratch-reclaim setting as 1P1D c48. - TOTAL_CPU_DRAM_GB=1799 decode: { num-worker: 2, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + +kimik3-fp4-mi355x-atom-disagg-agentic: + image: rocm/atom-dev:nightly_202610190554@sha256:9765777e2f56c5c17fefc9e141b37c51f1ab2dea009b158bcf1823f10030cea3 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:mi355x-amds + precision: fp4 + framework: atom-disagg + router: { name: atomesh, version: "daa4f1bb8404be08caa9b61e6930414de9de84a3" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: kimik3/atom/mi355x-fp4/agentx + scenarios: + agentic-coding: + # 0.343 -> 1029 GB over the eight prefill ranks -> LMCACHE_MAX_LOCAL_CPU_SIZE 128. + - dram-utilization: 0.343 + search-space: + - spec-decoding: mtp + conc-list: [1] + kv-offloading: none + srt-recipe: disagg-variants.yaml:override_1p1d_tp8_c1 + prefill: { num-worker: 1, tp: 8, dcp-size: 1, ep: 1, dp-attn: false } + decode: { num-worker: 1, tp: 8, dcp-size: 1, ep: 1, dp-attn: false } + - spec-decoding: mtp + conc-list: [16] + kv-offloading: dram + kv-offload-backend: { name: lmcache } + srt-recipe: disagg-variants.yaml:override_1p2d_dcp8_c16 + prefill: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + decode: { num-worker: 2, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + # 0.598 -> 1794 GB over the eight prefill ranks -> LMCACHE_MAX_LOCAL_CPU_SIZE 224, + # of which OFFLOAD_STATE_CPU_SIZE caps the KDA state images at 128. + - dram-utilization: 0.598 + search-space: + - spec-decoding: mtp + conc-list: [48] + kv-offloading: dram + kv-offload-backend: { name: lmcache } + srt-recipe: disagg-variants.yaml:override_1p1d_dcp8_c48 + prefill: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + decode: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - spec-decoding: mtp + conc-list: [48] + kv-offloading: dram + kv-offload-backend: { name: lmcache } + srt-recipe: disagg-variants.yaml:override_1p2d_dcp8_c48 + prefill: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + decode: { num-worker: 2, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - spec-decoding: mtp + conc-list: [72] + kv-offloading: dram + kv-offload-backend: { name: lmcache } + srt-recipe: disagg-variants.yaml:override_1p1d_dcp8_c72 + prefill: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + decode: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - spec-decoding: mtp + conc-list: [88] + kv-offloading: dram + kv-offload-backend: { name: lmcache } + srt-recipe: disagg-variants.yaml:override_1p1d_dcp8_c88 + prefill: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + decode: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 5821265d3d..f04fe12a4c 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10199,3 +10199,14 @@ description: - "Re-sweep Muse Spark Open MI355X AgentX on the full uncapped trace corpus (semianalysis_cc_traces_weka_062126) after #4287 dropped the Muse Spark 256k exception; image, recipe and search space are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4295 + +- config-keys: + - kimik3-fp4-mi355x-atom-disagg-agentic + scenario-type: + - agentic-coding + description: + - "Add native Kimi-K3 MI355X ATOM P/D AgentX points behind AToMesh with Mooncake P->D transfer: 1P1D TP8 DSpark7 c1, 1P2D TP8+DCP8 DSpark7 c16, 1P1D TP8+DCP8 DSpark4 c48/c72/c88 and 1P2D TP8+DCP8 DSpark4 c48." + - "Prefill keeps the paged KV and the KDA state images in an LMCache CPU tier (lmcache_offload): 224 GB per rank with the state images capped at 128 GB and on-demand checkpoints for the DSpark4 points, 128 GB per rank for c16. The c1 point is GPU-resident." + - "The DSpark draft stays BF16: the ATOM online_quant_config exclusions cover every draft layer." + - "The prefill LMCache pools are backed by 2 MiB HugeTLB pages (LMCACHE_LOCAL_CPU_USE_HUGEPAGES), which host_setup reserves on the prefill node through hugepage_sanity.sh and teardown releases. A 4 KiB pool registered with hipHostRegister stays movable, and each page the kernel migrates stalls the rank's GPU queues, which left LMCache prefills stuck in their startup barrier on mi355x-amds." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4311