weireweire · Oct 28, 2026 · Oct 28, 2026 · Oct 28, 2026 · Oct 28, 2026 · Oct 28, 2026
@@ -5,8 +5,8 @@ base:
   identity:
     model: {}
     frameworks:
-      dynamo: "1.6.0"
-      sglang: "0.0.0.dev1+g656331295"
+      dynamo: "1.7.0.dev20261028"
+      sglang: "0.0.0.dev1+gc10e0e177"
   dynamo:
     install: false
     request_plane: tcp
@@ -66,6 +66,11 @@ base:
       env:
         PYTHONNOUSERSITE: "1"
         PYTHONUNBUFFERED: "1"
+        # Dynamo 1.7 backend admission rejects turns with 529 once the engine is full;
+        # the SGLang scheduler queues them instead.
+        DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: "10000"
+        DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: "10000"
+        DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: "10000"
       args:
         served-model-name: deepseek-ai/DeepSeek-V4.1-Flash
         trust-remote-code: true
@@ -94,26 +99,6 @@ base:
       # The client splits this value on whitespace, so it has none.
       AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}'
 
-override_tp2_c16:
-  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp2ep2-c16
-  roles:
-    agg:
-      gpus: 2
-      args:
-        tensor-parallel-size: 2
-        expert-parallel-size: 2
-        dsv4-attn-backend: flashmla
-        enable-decoder-swa-bounded-replay: true
-        cuda-graph-backend-prefill: disabled
-        prefill-decode-interval: 96
-        swa-prefix-tails: 1024
-        max-running-requests: 32
-      env:
-        SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
-        SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank
-        PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
-        SGLANG_NUMA_BIND_V2: "0"
-
 override_tp4_c1:
   name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp4ep1-c1
   roles:
@@ -142,24 +127,9 @@ override_tp4_c1:
         SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
         SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank
         SGLANG_NVIDIA_MEGAKERNEL_SLOTS: hop_a,moe,gate,attn,embed,two_shot,markov,draft_lm_head,aux_mean_stats,gate_wide,shared_ll,hop_a_batch2,engram,attn_qkv_a,attn_qkv_a_bf16,index_q_mxfp8_in,attn_indexer,hc_prenorm,hop_b,hop_b_wide,early_trigger,hop_b_start_trigger,draft,draft_moe,draft_gate,draft_tail,draft_post,draft_seam,draft_attn,draft_qfront,draft_wo_a,draft_swapab,draft_qkv_a,draft_prenorm,draft_qkv_a_ll,draft_wo_b_ll,sched_ahead,req_pos_once,kv_before_wait,target_lm_head,hop_b_quad_trigger,lm_head_pinned_tactic
-        SGLANG_DSPARK_FUSED_ACCEPT_TAIL: "1"
-        SGLANG_DSPARK_DEFER_CONTEXT_KV: "1"
-        SGLANG_OPT_DSV41_DSPARK_DRAFT_PREP_FORK: "1"
-        SGLANG_OPT_DSV41_DSPARK_POST_FOLD: "1"
-        SGLANG_OPT_DSV41_DSPARK_SKIP_PREFILLED_INPUTS: "1"
-        SGLANG_OPT_DSV41_DSPARK_ONE_LAUNCH_COMMIT_STAGE: "1"
-        SGLANG_OPT_DSV41_DSPARK_PREPARE_INPUTS_ONCE: "1"
-        SGLANG_OPT_DSV41_DSPARK_MID_FOLD: "1"
-        SGLANG_OPT_DSV41_TARGET_VERIFY_METADATA_GLUE: "1"
-        SGLANG_OPT_DSV41_TARGET_METADATA_EARLY: "1"
-        SGLANG_OPT_SKIP_GLUED_REPLAY_VIEW: "1"
-        SGLANG_OPT_DSV41_POST_FORWARD_CUDA_GRAPH: "1"
-        SGLANG_OPT_DSV41_BLOCKWISE_REJECTION: "1"
-        SGLANG_OPT_DSV41_SAMPLING_ACCEPT_GRAPH: "1"
+        SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION: "1"
         SGLANG_SIMULATE_ACC_GREEDY: "0"
         SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD: "1"
-        SGLANG_OPT_MXFP4_DEFERRED_FINALIZED_SHAPE: "1"
-        SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH: "1"
         NCCL_CUMEM_ENABLE: "0"
         NCCL_MNNVL_ENABLE: "0"
 

@@ -1,12 +1,12 @@
 schema: 2
 
 base:
-  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg
+  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache
   identity:
     model: {}
     frameworks:
-      dynamo: "1.6.0"
-      sglang: "0.0.0.dev1+g656331295"
+      dynamo: "1.7.0.dev20261027"
+      sglang: "0.0.0.dev1+g586355a83"
   dynamo:
     install: false
     request_plane: tcp
@@ -32,12 +32,6 @@ base:
       type: etcd
       placement:
         node: infra
-    - name: mooncake-master
-      type: mooncake-master
-      args:
-        - --eviction_high_watermark_ratio=0.90
-        - --client_ttl=60
-        - --rpc_thread_num=64
     # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it.
     - name: otel-collector
       type: otel-collector
@@ -46,7 +40,7 @@ base:
     type: dynamo
     enable_multiple_frontends: false
     numa_bind: true
-    # One CPU node hosts the frontend, etcd, the Mooncake master and the AgentX client.
+    # One CPU node hosts the frontend, etcd and the AgentX client.
     placement:
       node: dedicated
     env:
@@ -67,31 +61,34 @@ base:
   engine: sglang
   roles:
     prefill:
+      nodes: 5
       gpus: 2
       env:
         PYTHONUNBUFFERED: "1"
         SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
         SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank
+        SGLANG_DSV4_KV_LAYOUT: v41_new
         SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600"
         SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600"
         SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1"
         SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
-        SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "32768"
+        SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "65536"
+        SGLANG_OPT_DSV41_BF16_INDEX_LOGITS: "1"
         SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: "1"
         NCCL_MNNVL_ENABLE: "1"
         NCCL_CUMEM_ENABLE: "1"
         MC_FORCE_MNNVL: "1"
         MOONCAKE_PROTOCOL: rdma
-        # The store registers host buffers only over RDMA; without a device list it
-        # picks NVLink for same-domain peers.
         MOONCAKE_DEVICE: "@fabric.rdma-devices"
-        MOONCAKE_GLOBAL_SEGMENT_SIZE: 80gb
-        MOONCAKE_STANDALONE_STORAGE: "0"
-        MC_STORE_CLIENT_METRIC_INTERVAL: "5"
         MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
         # Some GB300 nodes list a BMC interface with the same address on every node first.
         MC_TCP_BIND_ADDRESS: "{node}"
         WITH_NVIDIA_PEERMEM: "0"
+        # Dynamo 1.7 backend admission rejects turns with 529 once a DP rank is full;
+        # the prefill scheduler queues them instead.
+        DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: "10000"
+        DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: "10000"
+        DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: "10000"
       args:
         served-model-name: deepseek-ai/DeepSeek-V4.1-Flash
         trust-remote-code: true
@@ -108,14 +105,19 @@ base:
         dsv4-attn-backend: flashmla_mega
         disaggregation-transfer-backend: mooncake
         chunked-prefill-size: 65536
+        max-prefill-tokens: 32768
+        # 64 per DP rank, as vLLM's prefill max-num-seqs; absorbs the client's turn burst.
         max-running-requests: 128
         swa-prefix-tails: 2048
         enable-decoder-swa-bounded-replay: true
-        enable-unified-cache-external-linker: true
-        unified-cache-external-linker-backend: mooncake
-        hicache-storage-backend-extra-config: '{"enable_group_semantics":true}'
-        hicache-ratio: 1.02
+        enable-encoder-swa-bounded-replay: true
+        # Local host-DRAM tier: device-evicted prefixes reload from host memory off the
+        # forward path; an external L3 lookup blocks each layer of the forward.
+        enable-hierarchical-cache: true
+        hicache-ratio: 1.2
         hicache-mem-layout: page_first_direct
+        hicache-io-backend: direct
+        hicache-write-policy: write_back
         mem-fraction-static: 0.85
         speculative-algorithm: DSPARK
         speculative-dspark-block-size: 5
@@ -134,25 +136,30 @@ base:
       gpus: 16
       env:
         PYTHONUNBUFFERED: "1"
-        SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
-        SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank
-        # One request gateway process saturates a CPU core serving 16 DP ranks.
-        DYN_SGL_GATEWAY_WORKERS: "8"
+        # Decode ranks read Engram rows from peer shards instead of a host table.
+        SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS: "1"
+        SGLANG_DSV4_KV_LAYOUT: v41_new
+        # One gateway process saturates a CPU core serving 16 DP ranks; with 8, one
+        # instance stalls at high concurrency.
+        DYN_SGL_GATEWAY_WORKERS: "16"
         SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600"
         SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600"
         SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
+        SGLANG_OPT_DEEPGEMM_MEGA_GATE: "1"
         SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "16384"
         SGLANG_OPT_DSV41_BF16_INDEX_LOGITS: "1"
+        SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: "1"
         NCCL_MNNVL_ENABLE: "1"
         NCCL_CUMEM_ENABLE: "1"
         MC_FORCE_MNNVL: "1"
         MOONCAKE_PROTOCOL: rdma
         MOONCAKE_DEVICE: "@fabric.rdma-devices"
-        MOONCAKE_STANDALONE_STORAGE: "0"
-        MC_STORE_CLIENT_METRIC_INTERVAL: "5"
         MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
         MC_TCP_BIND_ADDRESS: "{node}"
         WITH_NVIDIA_PEERMEM: "0"
+        DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: "10000"
+        DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: "10000"
+        DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: "10000"
       args:
         served-model-name: deepseek-ai/DeepSeek-V4.1-Flash
         trust-remote-code: true
@@ -165,16 +172,19 @@ base:
         enable-dp-attention: true
         enable-dp-lm-head: true
         enable-dp-attention-local-control-broadcast: true
+        enable-fp32-lm-head: true
         moe-a2a-backend: megamoe
         dsv4-attn-backend: flashmla_mega
         disaggregation-transfer-backend: mooncake
-        disaggregation-decode-polling-interval: 8
+        disaggregation-decode-extra-slots: 256
         stream-interval: 32
         swa-prefix-tails: 2048
         enable-decoder-swa-bounded-replay: true
-        enable-unified-cache-external-linker: true
-        unified-cache-external-linker-backend: mooncake
-        mem-fraction-static: 0.85
+        mem-fraction-static: 0.75
+        enable-eplb: true
+        ep-num-redundant-experts: 16
+        eplb-rebalance-num-iterations: 15000
+        expert-distribution-recorder-buffer-size: 1000
         speculative-algorithm: DSPARK
         speculative-dspark-block-size: 5
         speculative-num-steps: 5
@@ -205,50 +215,37 @@ base:
       AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}'
       AIPERF_DATASET_MMAP_CACHE_ENABLED: "true"
 
-override_3p1d_dep2_dep16_c1228:
-  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-3p1d-dep2-dep16-c1228
-  roles:
-    prefill:
-      # Three DEP2 workers leave two GPUs of the second node idle.
-      nodes: 2
-      workers: 3
-    decode:
-      env:
-        MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
-      args:
-        max-running-requests: 384
-        cuda-graph-max-bs-decode: 24
-        disaggregation-decode-extra-slots: 36
-
 override_5p1d_dep2_dep16_c2884:
-  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-5p1d-dep2-dep16-c2884
+  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache-5p1d-dep2-dep16-c2884
   roles:
     prefill:
-      # Five DEP2 workers leave two GPUs of the third node idle.
       nodes: 3
       workers: 5
-      args:
-        max-prefill-tokens: 32768
     decode:
-      env:
-        MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
       args:
         max-running-requests: 640
         cuda-graph-max-bs-decode: 40
         disaggregation-decode-extra-slots: 80
 
 override_8p1d_dep2_dep16_c4096:
-  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-8p1d-dep2-dep16-c4096
+  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache-8p1d-dep2-dep16-c4096
   roles:
     prefill:
       nodes: 4
       workers: 8
-      args:
-        max-prefill-tokens: 32768
     decode:
-      env:
-        MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
       args:
         max-running-requests: 1024
         cuda-graph-max-bs-decode: 64
         disaggregation-decode-extra-slots: 128
+
+override_9p1d_dep2_dep16_b160_c7480:
+  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache-9p1d-dep2-dep16-b160-c7480
+  roles:
+    prefill:
+      # Nine DEP2 workers leave two GPUs of the fifth node idle.
+      workers: 9
+    decode:
+      args:
+        max-running-requests: 2560
+        cuda-graph-max-bs-decode: 160
@@ -8020,13 +8020,13 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark:
       - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128, 160], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml }
       - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 16], srt-recipe: agentic.yaml }
 dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg:
-  image: lmsysorg/sglang:nightly-dev-cu13-20261025-65633129@sha256:069efa1d6a2d58309529f53eeee34c39d69b1cdb455eaf7fb8de83e3fd0be727
+  image: lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17@sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47
   model: deepseek-ai/DeepSeek-V4.1-Flash
   model-prefix: dsv41flash
   runner: cluster:gb300-nv
   precision: fp4
   framework: dynamo-sglang
-  router: { name: dynamo-router, version: "1.6.0" }
+  router: { name: dynamo-router, version: "1.7.0.dev20261028" }
   multinode: true
   disagg: false
   srt-recipe-dir: dsv41flash/sglang/gb300-fp4/agentx
@@ -8061,24 +8061,15 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg:
           tp: 4
           ep: 1
           dp-attn: false
-      - spec-decoding: mtp
-        conc-list: [16]
-        num-nodes: 1
-        srt-recipe: agg-variants.yaml:override_tp2_c16
-        worker:
-          num-worker: 1
-          tp: 2
-          ep: 2
-          dp-attn: false
 
-dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg:
-  image: lmsysorg/sglang:nightly-dev-cu13-20261025-65633129@sha256:069efa1d6a2d58309529f53eeee34c39d69b1cdb455eaf7fb8de83e3fd0be727
+dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache:
+  image: lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90
   model: deepseek-ai/DeepSeek-V4.1-Flash
   model-prefix: dsv41flash
   runner: cluster:gb300-nv
   precision: fp4
   framework: dynamo-sglang
-  router: { name: dynamo-router, version: "1.6.0.dev20261019" }
+  router: { name: dynamo-router, version: "1.7.0.dev20261027" }
   kv-p2p-transfer: mooncake
   multinode: true
   disagg: true
@@ -8088,12 +8079,12 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg:
     - dram-utilization: 0.80
       search-space:
       - spec-decoding: mtp
-        conc-list: [1228]
-        srt-recipe: disagg-variants.yaml:override_3p1d_dep2_dep16_c1228
+        conc-list: [2884]
+        srt-recipe: disagg-dep16-hicache-variants.yaml:override_5p1d_dep2_dep16_c2884
         kv-offloading: dram
-        kv-offload-backend: { name: mooncake, version: "0.3.13.post1" }
+        kv-offload-backend: { name: hicache }
         prefill:
-          num-worker: 3
+          num-worker: 5
           tp: 2
           ep: 2
           dp-attn: true
@@ -8103,12 +8094,12 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg:
           ep: 16
           dp-attn: true
       - spec-decoding: mtp
-        conc-list: [2884]
-        srt-recipe: disagg-variants.yaml:override_5p1d_dep2_dep16_c2884
+        conc-list: [4096]
+        srt-recipe: disagg-dep16-hicache-variants.yaml:override_8p1d_dep2_dep16_c4096
         kv-offloading: dram
-        kv-offload-backend: { name: mooncake, version: "0.3.13.post1" }
+        kv-offload-backend: { name: hicache }
         prefill:
-          num-worker: 5
+          num-worker: 8
           tp: 2
           ep: 2
           dp-attn: true
@@ -8118,12 +8109,12 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg:
           ep: 16
           dp-attn: true
       - spec-decoding: mtp
-        conc-list: [4096]
-        srt-recipe: disagg-variants.yaml:override_8p1d_dep2_dep16_c4096
+        conc-list: [7480]
+        srt-recipe: disagg-dep16-hicache-variants.yaml:override_9p1d_dep2_dep16_b160_c7480
         kv-offloading: dram
-        kv-offload-backend: { name: mooncake, version: "0.3.13.post1" }
+        kv-offload-backend: { name: hicache }
         prefill:
-          num-worker: 8
+          num-worker: 9
           tp: 2
           ep: 2
           dp-attn: true