From f9696501e44902daad1fa9ef3ea1a64f7806b822 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:58:45 +0800 Subject: [PATCH 1/9] Add GB300 DSV4.1-Flash SGLang DEP16 disagg c7480 points with prefill HiCache Root cause: at c7480 the Mooncake external linker reads L3 hits layer by layer on the forward path; after the AgentX turn burst, prefill stalls on those reads (about 40 % of wall time), device prefixes age out, and the deployment does not recover. Fix: a new config key on nightly-dev-cu13-20261027-5884e4fe (Dynamo 1.7.0.dev20261026) with 9 and 10 DEP2 prefill workers and one DEP16 decode. Prefill keeps evicted prefixes in a local host-DRAM HiCache tier (ratio 1.8, write-back, direct I/O); decode keeps SGLang's default polling interval of 1. Dynamo 1.7 backend admission limits are raised so turns queue in the scheduler, and NVIDIA/srt-slurm#629 is backported so the readiness wait asks /health?instances=true when /health is status-only. The agg and dspark-disagg keys are listed so they rerun in the same sweep. Validation: on GB300 with the same recipe and AgentX client, 10 x DEP2 batch 128 gave 924,652 tok/s/GPU at p90 117.2 tok/s/user over 3600 s. --- .../agentx/disagg-dep16-hicache-variants.yaml | 247 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 62 +++++ inferencex-e2e/perf-changelog.yaml | 12 + .../patches/629-dynamo-health-instances.patch | 48 ++++ .../runners/srt-slurm/patches/README.md | 1 + 5 files changed, 370 insertions(+) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml create mode 100644 inferencex-e2e/runners/srt-slurm/patches/629-dynamo-health-instances.patch diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml new file mode 100644 index 0000000000..bc5c156e40 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml @@ -0,0 +1,247 @@ +schema: 2 + +base: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache + identity: + model: {} + frameworks: + dynamo: "1.7.0.dev20261026" + sglang: "0.0.0.dev1+g5884e4fe" + dynamo: + install: false + request_plane: tcp + event_plane: zmq + # gb300-nv serves this model from the Hub; read it from the shared mounted cache. + environment: + HF_HUB_CACHE: /hf_hub_cache + # Tachometer's per-node exporters slow decode steps and add head-node host memory; + # InferenceX does not read its output. + observability: + tachometer: + enabled: false + slurm: + time_limit: "6:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. + - name: otel-collector + type: otel-collector + enabled: false + frontend: + type: dynamo + enable_multiple_frontends: false + numa_bind: true + # One CPU node hosts the frontend, etcd and the AgentX client. + placement: + node: dedicated + env: + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + args: + router-mode: kv + router-session-affinity-ttl-secs: "14400" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + prefill: + nodes: 5 + gpus: 2 + env: + PYTHONUNBUFFERED: "1" + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + SGLANG_DSV4_KV_LAYOUT: v41_new + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "65536" + SGLANG_OPT_DSV41_BF16_INDEX_LOGITS: "1" + SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "@fabric.rdma-devices" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # Some GB300 nodes list a BMC interface with the same address on every node first. + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + # Dynamo 1.7 backend admission rejects turns with 529 once a DP rank is full; + # the prefill scheduler queues them instead. + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: "10000" + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: "10000" + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: "10000" + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + tensor-parallel-size: 2 + tokenizer-backend: fastokens + router-mode: kv + router-session-affinity-ttl-secs: "14400" + data-parallel-size: 2 + expert-parallel-size: 2 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + moe-a2a-backend: megamoe + dsv4-attn-backend: flashmla_mega + disaggregation-transfer-backend: mooncake + chunked-prefill-size: 65536 + max-prefill-tokens: 32768 + # 64 per DP rank, as vLLM's prefill max-num-seqs; absorbs the client's turn burst. + max-running-requests: 128 + swa-prefix-tails: 2048 + enable-decoder-swa-bounded-replay: true + enable-encoder-swa-bounded-replay: true + # Local host-DRAM tier: device-evicted prefixes reload from host memory off the + # forward path; an external L3 lookup blocks each layer of the forward. + enable-hierarchical-cache: true + hicache-ratio: 1.8 + hicache-mem-layout: page_first_direct + hicache-io-backend: direct + hicache-write-policy: write_back + mem-fraction-static: 0.85 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + speculative-num-steps: 5 + reasoning-parser: auto + tool-call-parser: auto + watchdog-timeout: 3600 + enable-metrics: true + weight-loader-prefetch-checkpoints: true + weight-loader-drop-cache-after-load: false + model-loader-extra-config: '{"enable_multithread_load":true}' + kv_events: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + PYTHONUNBUFFERED: "1" + # Decode ranks read Engram rows from peer shards instead of a host table. + SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS: "1" + SGLANG_DSV4_KV_LAYOUT: v41_new + # One gateway process saturates a CPU core serving 16 DP ranks; with 8, one + # instance stalls at high concurrency. + DYN_SGL_GATEWAY_WORKERS: "16" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_DEEPGEMM_MEGA_GATE: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "16384" + SGLANG_OPT_DSV41_BF16_INDEX_LOGITS: "1" + SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "@fabric.rdma-devices" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: "10000" + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: "10000" + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: "10000" + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + tensor-parallel-size: 16 + tokenizer-backend: fastokens + router-mode: round-robin + load-balance-method: total_tokens + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + enable-fp32-lm-head: true + moe-a2a-backend: megamoe + dsv4-attn-backend: flashmla_mega + disaggregation-transfer-backend: mooncake + disaggregation-decode-extra-slots: 256 + stream-interval: 32 + swa-prefix-tails: 2048 + enable-decoder-swa-bounded-replay: true + mem-fraction-static: 0.85 + enable-eplb: true + ep-num-redundant-experts: 16 + eplb-rebalance-num-iterations: 15000 + expert-distribution-recorder-buffer-size: 1000 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + speculative-num-steps: 5 + reasoning-parser: auto + tool-call-parser: auto + watchdog-timeout: 3600 + enable-metrics: true + weight-loader-prefetch-checkpoints: true + weight-loader-drop-cache-after-load: false + model-loader-extra-config: '{"enable_multithread_load":true}' + kv_events: false + sbatch_directives: + mem: "0" + exclusive: "" + srun_options: + container-remap-root: "" + # Engram host tables need the node's full memory in every worker step. + mem: "0" + benchmark: + placement: + node: dedicated + colocate_with_frontend: true + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}' + AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" + +override_9p1d_dep2_dep16_c7480: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache-9p1d-dep2-dep16-c7480 + roles: + prefill: + # Nine DEP2 workers leave two GPUs of the fifth node idle. + workers: 9 + decode: + args: + max-running-requests: 2048 + cuda-graph-max-bs-decode: 128 + +override_10p1d_dep2_dep16_c7480: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache-10p1d-dep2-dep16-c7480 + roles: + prefill: + workers: 10 + decode: + args: + max-running-requests: 2048 + cuda-graph-max-bs-decode: 128 + +override_10p1d_dep2_dep16_b160_c7480: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache-10p1d-dep2-dep16-b160-c7480 + roles: + prefill: + workers: 10 + decode: + args: + max-running-requests: 2560 + cuda-graph-max-bs-decode: 160 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 9ead101629..62bb66529c 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9718,6 +9718,68 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg: ep: 16 dp-attn: true +dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache: + image: lmsysorg/sglang:nightly-dev-cu13-20261027-5884e4fe@sha256:61b9083bdcb7cf3741366f487cb996c05ea33ad2ddddc820d10cbc1c0cfb0a86 + model: deepseek-ai/DeepSeek-V4.1-Flash + model-prefix: dsv41flash + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.7.0.dev20261026" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: dsv41flash/sglang/gb300-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + conc-list: [7480] + srt-recipe: disagg-dep16-hicache-variants.yaml:override_9p1d_dep2_dep16_c7480 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 9 + tp: 2 + ep: 2 + dp-attn: true + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [7480] + srt-recipe: disagg-dep16-hicache-variants.yaml:override_10p1d_dep2_dep16_c7480 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 10 + tp: 2 + ep: 2 + dp-attn: true + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [7480] + srt-recipe: disagg-dep16-hicache-variants.yaml:override_10p1d_dep2_dep16_b160_c7480 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 10 + tp: 2 + ep: 2 + dp-attn: true + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + dsv41flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg: image: vllm/vllm-openai:cu134-nightly-aarch64-dev-20261019 model: deepseek-ai/DeepSeek-V4.1-Flash diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 917db5953b..1ad4d5adb3 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11404,3 +11404,15 @@ - "TP2 c16-c64, the unsearched TP4 c4-c64 recipe variants, the DSpark speculative config (5 drafts, probabilistic drafting) and mono decode are unchanged." - "Draft data types unchanged: the DSpark draft (mtp.0-2 in the target checkpoint) loads with the checkpoint's FP8 config, as shipped (FP8 E4M3 with 32x32 UE8M0 scales for attention, shared experts and main_proj, MXFP4 routed experts, BF16 gate, norms and heads), inherits the target fp8_ds_mla KV cache, and the mono draft kernels read those weights in place." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4480 + +- config-keys: + - dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg + - dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg + - dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache + scenario-type: [agentic-coding] + description: + - "Add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache on nightly-dev-cu13-20261027-5884e4fe (Dynamo 1.7.0.dev20261026), pinned by digest: DEP2 prefill and one DEP16 decode at c7480 with 9 prefill workers (decode CUDA graph batch 128), 10 prefill workers (batch 128) and 10 prefill workers (batch 160, max-running-requests 2560)." + - "Prefill keeps evicted prefixes in a local host-DRAM HiCache tier (ratio 1.8, write-back, direct I/O, mem-fraction-static 0.85) instead of the Mooncake external linker, whose layer-wise L3 reads block the forward pass and stall prefill after the client's turn burst at c7480. Decode admits transferred requests every step (SGLang's default disaggregation-decode-polling-interval of 1; the dspark-disagg key sets 8)." + - "Dynamo 1.7 backend admission limits are raised to 10000 so waiting turns queue in the prefill scheduler instead of failing with 529, and runners/srt-slurm/patches backports NVIDIA/srt-slurm#629 so the readiness wait asks /health?instances=true when /health is status-only." + - "The agg and existing dspark-disagg keys are unchanged; they rerun in the same sweep because AgentX plots all dynamo-sglang points of this model and GPU as one curve." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/inferencex-e2e/runners/srt-slurm/patches/629-dynamo-health-instances.patch b/inferencex-e2e/runners/srt-slurm/patches/629-dynamo-health-instances.patch new file mode 100644 index 0000000000..e0092720d4 --- /dev/null +++ b/inferencex-e2e/runners/srt-slurm/patches/629-dynamo-health-instances.patch @@ -0,0 +1,48 @@ +diff --git a/src/srtctl/frontends/dynamo.py b/src/srtctl/frontends/dynamo.py +--- a/src/srtctl/frontends/dynamo.py ++++ b/src/srtctl/frontends/dynamo.py +@@ -12,6 +12,7 @@ import shlex + import threading + from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast + ++import requests + import yaml + + from srtctl.backends.vllm import VLLMBackend +@@ -42,6 +43,8 @@ logger = logging.getLogger(__name__) + + ROUTER_POLICY_CONFIG_FILENAME = "router_policy_config.yaml" + ROUTER_POLICY_CONFIG_CONTAINER_PATH = f"/logs/{ROUTER_POLICY_CONFIG_FILENAME}" ++# Since ai-dynamo/dynamo#16454 (a789babbe3) /health is status-only unless asked for its instance list. ++HEALTH_INSTANCES_ENDPOINT = "/health?instances=true" + + + def _vllm_mode_config(backend: VLLMBackend, mode: str) -> dict[str, Any]: +@@ -176,9 +179,26 @@ class DynamoFrontend(DynamicFrontend): + expected_prefill: int, + expected_decode: int, + ) -> WorkerHealthResult: +- """Parse dynamo /health endpoint response.""" ++ """Count the generate instances in a Dynamo /health instance listing (see ``probe_ready``).""" + return check_dynamo_health(response_json, expected_prefill, expected_decode) + ++ def probe_ready( ++ self, host: str, port: int, expected_prefill: int, expected_decode: int, config: Any ++ ) -> WorkerHealthResult: ++ """Count instances from ``/health``, asking ``/health?instances=true`` when it answers status-only. ++ ++ Frontends before ai-dynamo/dynamo#16454 always list instances in ``/health`` and get the one ++ request they always got; newer ones answer ``{"status": "healthy", "stage": "ready"}`` there. ++ """ ++ for endpoint in (self.health_endpoint, HEALTH_INSTANCES_ENDPOINT): ++ response = requests.get(f"http://{host}:{port}{endpoint}", timeout=5.0) ++ if response.status_code != 200: ++ return WorkerHealthResult(ready=False, message=f"{endpoint} returned HTTP {response.status_code}") ++ body = response.json() ++ if "instances" in body: ++ break ++ return self.parse_health(body, expected_prefill, expected_decode) ++ + def health_expectations(self, config: Any, processes: list["Process"] | None) -> tuple[int, int, str]: + """Dynamo's /health counts registered generate instances. + diff --git a/inferencex-e2e/runners/srt-slurm/patches/README.md b/inferencex-e2e/runners/srt-slurm/patches/README.md index d9052d1aa8..7881645376 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/README.md +++ b/inferencex-e2e/runners/srt-slurm/patches/README.md @@ -9,3 +9,4 @@ Each patch backports an upstream fix that the shared submodule pin does not incl | Patch | Upstream PR | Fix | |-------|-------------|-----| | `573-participating-gpus.patch` | [NVIDIA/srt-slurm#573](https://github.com/NVIDIA/srt-slurm/pull/573) | Record only the GPUs workers occupy on a worker node; an exclusive node's exporter (DCGM or AMD) reports every GPU, which the collector otherwise flags as `unexpected_device`. The pin includes #573, but upstream merged a different design: it keeps every row and excuses idle worker-node GPUs only after a DCGM worker GPU UUID probe. [`infx/results/power/multinode.py`](../../../infx/results/power/multinode.py) does not apply that excusal yet, so keep this filter until it does. | +| `629-dynamo-health-instances.patch` | [NVIDIA/srt-slurm#629](https://github.com/NVIDIA/srt-slurm/pull/629) | Dynamo frontends since ai-dynamo/dynamo#16454 (1.7 dev builds) answer `/health` status-only and list instances only for `/health?instances=true`, so the readiness wait never counted any workers. `DynamoFrontend.probe_ready` reads `/health` and asks `/health?instances=true` only when the answer has no `instances` key; older frontends get the same single request. | From 7095a6b2de6e9f32dcceea2434269cca04335ece Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:59:59 +0800 Subject: [PATCH 2/9] Set perf-changelog pr-link to #4595 --- inferencex-e2e/perf-changelog.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 1ad4d5adb3..602f273549 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp @@ -9724,7 +9724,7 @@ - "Replace the TP4/EP1 and TP2/EP1 C1-C128 sweeps with TP4/EP4 points at concurrency 1, 4, 8 and 16, plus aggregated TP4/EP4 points with DP attention (DP4) at concurrency 32, 64, 128 and 192 behind sglang-router 0.3.2." - "Apply the point-specific prefill-decode interval, prefill chunking, admission, memory and SWA prefix-tail settings from the single-node recipe; all points use the default fcfs schedule policy." - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4.1-Flash runs as shipped with block size 5." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 - config-keys: - kimik3-fp4-mi355x-vllm-disagg-agentic @@ -10287,7 +10287,7 @@ description: - "Update the image from lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17 to lmsysorg/sglang:nightly-dev-cu13-20261013-7224fce1." - "Set TP4 concurrency 2 max-running-requests to 2 and sweep concurrency 1 to 16 on TP4 and TP8." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 - config-keys: - dsv4-fp4-mi355x-sglang-agentic-mtp @@ -10694,7 +10694,7 @@ description: - "Replace the Muse Spark Open B300 TP8/EP8 search space with per-concurrency AgentX settings: TP8/EP8 with GPU-resident KV at concurrency 1, 4, 8 and 14; TP4/EP4 with GPU-resident KV at concurrency 4; TP4/EP4 with 120-175 GB per-rank DRAM HiCache at concurrency 10, 16, 20, 28, 36 and 40." - "Tune request admission, decode graph batch limits, memory fraction, SWA/full ratio, prefill-decode interval and chunked prefill per concurrency; enable layernorm SP and MegaMoE at the HiCache points and shortest-prefill-first from concurrency 16." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 - config-keys: - musespark-fp4-b200-sglang-agentic-dflash @@ -10703,7 +10703,7 @@ description: - "Configure Muse Spark Open B200 SGLang AgentX request admission, scheduling, memory allocation, HiCache, and draft attention per concurrency, adding TP4/EP4 at concurrency 8, 10, 16, and 24 and TP8 at concurrency 6, 10, and 56." - "Drop TP8 concurrency 12, 16, 24, 32, 48, and 64." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 - config-keys: - qwen3.5-fp8-b300-sglang-mtp @@ -10789,7 +10789,7 @@ - "Disagg: replace the DEP8 prefill + Mooncake store ladder with TP1 x PP8 prefill workers that keep a 2x HiCache host tier, and run W4A4 MegaMoE for the target model on prefill and decode while the DSpark draft MoE stays at the image default (W4A8)." - "Disagg points: 1P+DEP16 at concurrency 80, 160 and 200 with the KV router and token-balanced decode; 1P+DEP16 c300, 2P+DEP16 c720, 2P+DEP12 c800 and 2P+DEP8 c880 with a round-robin frontend and request-balanced decode. DSpark block size 6, 6, 5, 5 and 4, matched on prefill and decode." - "Agg: TP8 over two nodes at concurrency 1; TP4 on one node at concurrency 1 and 4 with the fastokens tokenizer. DSpark block size 6." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 - config-keys: - glm5.2-fp4-gb200-dynamo-sglang-agentic-agg @@ -10810,7 +10810,7 @@ description: - "Update the Muse Spark Open GB300 curve to lmsysorg/sglang:nightly-dev-20261021-c1c287a4 and set SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION=0 and --stream-interval 20 at every concurrency." - "Lower max-running-requests to 3 at concurrency 10, raise prefill-decode-interval to 96 at concurrency 24, switch HiCache to write_back, and replace concurrency 16, 20 and 40 with 28, 30, 44 and 48." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 - config-keys: - musespark-fp4-b200-sglang-agentic-dflash @@ -11241,7 +11241,7 @@ - minimaxm31flash-fp4-gb300-sglang-agentic-dspark description: - "Add TP2 points with DRAM KV offload, and adjust the memory settings on the DRAM KV offload points." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 scenario-type: [agentic-coding] - config-keys: @@ -11415,4 +11415,4 @@ - "Prefill keeps evicted prefixes in a local host-DRAM HiCache tier (ratio 1.8, write-back, direct I/O, mem-fraction-static 0.85) instead of the Mooncake external linker, whose layer-wise L3 reads block the forward pass and stall prefill after the client's turn burst at c7480. Decode admits transferred requests every step (SGLang's default disaggregation-decode-polling-interval of 1; the dspark-disagg key sets 8)." - "Dynamo 1.7 backend admission limits are raised to 10000 so waiting turns queue in the prefill scheduler instead of failing with 529, and runners/srt-slurm/patches backports NVIDIA/srt-slurm#629 so the readiness wait asks /health?instances=true when /health is status-only." - "The agg and existing dspark-disagg keys are unchanged; they rerun in the same sweep because AgentX plots all dynamo-sglang points of this model and GPU as one curve." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 From 4b5e7d9eca182ca468ec83dbdf13bfcb14b16750 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:00:46 +0800 Subject: [PATCH 3/9] Restore historical perf-changelog entries; set only this entry's pr-link --- inferencex-e2e/perf-changelog.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 602f273549..d573f35bb2 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp @@ -9724,7 +9724,7 @@ - "Replace the TP4/EP1 and TP2/EP1 C1-C128 sweeps with TP4/EP4 points at concurrency 1, 4, 8 and 16, plus aggregated TP4/EP4 points with DP attention (DP4) at concurrency 32, 64, 128 and 192 behind sglang-router 0.3.2." - "Apply the point-specific prefill-decode interval, prefill chunking, admission, memory and SWA prefix-tail settings from the single-node recipe; all points use the default fcfs schedule policy." - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4.1-Flash runs as shipped with block size 5." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - kimik3-fp4-mi355x-vllm-disagg-agentic @@ -10287,7 +10287,7 @@ description: - "Update the image from lmsysorg/sglang:nightly-dev-cu13-20260923-06008c17 to lmsysorg/sglang:nightly-dev-cu13-20261013-7224fce1." - "Set TP4 concurrency 2 max-running-requests to 2 and sweep concurrency 1 to 16 on TP4 and TP8." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - dsv4-fp4-mi355x-sglang-agentic-mtp @@ -10694,7 +10694,7 @@ description: - "Replace the Muse Spark Open B300 TP8/EP8 search space with per-concurrency AgentX settings: TP8/EP8 with GPU-resident KV at concurrency 1, 4, 8 and 14; TP4/EP4 with GPU-resident KV at concurrency 4; TP4/EP4 with 120-175 GB per-rank DRAM HiCache at concurrency 10, 16, 20, 28, 36 and 40." - "Tune request admission, decode graph batch limits, memory fraction, SWA/full ratio, prefill-decode interval and chunked prefill per concurrency; enable layernorm SP and MegaMoE at the HiCache points and shortest-prefill-first from concurrency 16." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - musespark-fp4-b200-sglang-agentic-dflash @@ -10703,7 +10703,7 @@ description: - "Configure Muse Spark Open B200 SGLang AgentX request admission, scheduling, memory allocation, HiCache, and draft attention per concurrency, adding TP4/EP4 at concurrency 8, 10, 16, and 24 and TP8 at concurrency 6, 10, and 56." - "Drop TP8 concurrency 12, 16, 24, 32, 48, and 64." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - qwen3.5-fp8-b300-sglang-mtp @@ -10789,7 +10789,7 @@ - "Disagg: replace the DEP8 prefill + Mooncake store ladder with TP1 x PP8 prefill workers that keep a 2x HiCache host tier, and run W4A4 MegaMoE for the target model on prefill and decode while the DSpark draft MoE stays at the image default (W4A8)." - "Disagg points: 1P+DEP16 at concurrency 80, 160 and 200 with the KV router and token-balanced decode; 1P+DEP16 c300, 2P+DEP16 c720, 2P+DEP12 c800 and 2P+DEP8 c880 with a round-robin frontend and request-balanced decode. DSpark block size 6, 6, 5, 5 and 4, matched on prefill and decode." - "Agg: TP8 over two nodes at concurrency 1; TP4 on one node at concurrency 1 and 4 with the fastokens tokenizer. DSpark block size 6." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - glm5.2-fp4-gb200-dynamo-sglang-agentic-agg @@ -10810,7 +10810,7 @@ description: - "Update the Muse Spark Open GB300 curve to lmsysorg/sglang:nightly-dev-20261021-c1c287a4 and set SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION=0 and --stream-interval 20 at every concurrency." - "Lower max-running-requests to 3 at concurrency 10, raise prefill-decode-interval to 96 at concurrency 24, switch HiCache to write_back, and replace concurrency 16, 20 and 40 with 28, 30, 44 and 48." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - musespark-fp4-b200-sglang-agentic-dflash @@ -11241,7 +11241,7 @@ - minimaxm31flash-fp4-gb300-sglang-agentic-dspark description: - "Add TP2 points with DRAM KV offload, and adjust the memory settings on the DRAM KV offload points." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX scenario-type: [agentic-coding] - config-keys: From e480515ce495d9c0ae9cfd96823678b275d7319f Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:18:20 +0800 Subject: [PATCH 4/9] Move the GB300 DSV4.1-Flash Dynamo + SGLang agg row to the 20261028 nightly and drop TP2/EP2 c16 nightly-dev-cu13-20261028-586355a8 includes sgl-project/sglang#46404. Raise the Dynamo 1.7 backend admission limits as on the disagg rows. TP2/EP2 c16 sits far below the line from TP4 c16 to the disaggregated points, so it is removed with its override. --- .../sglang/gb300-fp4/agentx/agg-variants.yaml | 29 +++++-------------- inferencex-e2e/configs/nvidia-master.yaml | 13 ++------- inferencex-e2e/perf-changelog.yaml | 3 +- 3 files changed, 11 insertions(+), 34 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml index 523dee636b..66a9dfc3b7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -5,8 +5,8 @@ base: identity: model: {} frameworks: - dynamo: "1.6.0" - sglang: "0.0.0.dev1+g656331295" + dynamo: "1.7.0.dev20261027" + sglang: "0.0.0.dev1+g586355a83" dynamo: install: false request_plane: tcp @@ -66,6 +66,11 @@ base: env: PYTHONNOUSERSITE: "1" PYTHONUNBUFFERED: "1" + # Dynamo 1.7 backend admission rejects turns with 529 once the engine is full; + # the SGLang scheduler queues them instead. + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: "10000" + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: "10000" + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: "10000" args: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true @@ -94,26 +99,6 @@ base: # The client splits this value on whitespace, so it has none. AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}' -override_tp2_c16: - name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp2ep2-c16 - roles: - agg: - gpus: 2 - args: - tensor-parallel-size: 2 - expert-parallel-size: 2 - dsv4-attn-backend: flashmla - enable-decoder-swa-bounded-replay: true - cuda-graph-backend-prefill: disabled - prefill-decode-interval: 96 - swa-prefix-tails: 1024 - max-running-requests: 32 - env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_NUMA_BIND_V2: "0" - override_tp4_c1: name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp4ep1-c1 roles: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 7616f33e6c..1b5481370a 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9703,13 +9703,13 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark: - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128, 160], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 16], srt-recipe: agentic.yaml } dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20261025-65633129@sha256:069efa1d6a2d58309529f53eeee34c39d69b1cdb455eaf7fb8de83e3fd0be727 + image: lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.6.0" } + router: { name: dynamo-router, version: "1.7.0.dev20261027" } multinode: true disagg: false srt-recipe-dir: dsv41flash/sglang/gb300-fp4/agentx @@ -9744,15 +9744,6 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: tp: 4 ep: 1 dp-attn: false - - spec-decoding: mtp - conc-list: [16] - num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp2_c16 - worker: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg: image: lmsysorg/sglang:nightly-dev-cu13-20261025-65633129@sha256:069efa1d6a2d58309529f53eeee34c39d69b1cdb455eaf7fb8de83e3fd0be727 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 07296046f5..a72278d549 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11499,5 +11499,6 @@ - "Add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache on nightly-dev-cu13-20261027-5884e4fe (Dynamo 1.7.0.dev20261026), pinned by digest: DEP2 prefill and one DEP16 decode at c7480 with 9 prefill workers (decode CUDA graph batch 128), 10 prefill workers (batch 128) and 10 prefill workers (batch 160, max-running-requests 2560)." - "Prefill keeps evicted prefixes in a local host-DRAM HiCache tier (ratio 1.8, write-back, direct I/O, mem-fraction-static 0.85) instead of the Mooncake external linker, whose layer-wise L3 reads block the forward pass and stall prefill after the client's turn burst at c7480. Decode admits transferred requests every step (SGLang's default disaggregation-decode-polling-interval of 1; the dspark-disagg key sets 8)." - "Dynamo 1.7 backend admission limits are raised to 10000 so waiting turns queue in the prefill scheduler instead of failing with 529, and runners/srt-slurm/patches backports NVIDIA/srt-slurm#629 so the readiness wait asks /health?instances=true when /health is status-only." - - "The agg and existing dspark-disagg keys are unchanged; they rerun in the same sweep because AgentX plots all dynamo-sglang points of this model and GPU as one curve." + - "Move dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg to nightly-dev-cu13-20261028-586355a8 (Dynamo 1.7.0.dev20261027), pinned by digest, with the Dynamo 1.7 backend admission limits raised to 10000. The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row). Remove the TP2/EP2 c16 point; TP4 c1/c4/c16 keep their settings." + - "The existing dspark-disagg key is unchanged; it reruns in the same sweep because AgentX plots all dynamo-sglang points of this model and GPU as one curve." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 From c1549c179dcafe81047717627775316a33fe3793 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:35:19 +0800 Subject: [PATCH 5/9] Move the GB300 DSV4.1-Flash Dynamo + SGLang HiCache disagg key to the 20261028 nightly and retire the Mooncake-linker disagg key The hicache key now runs on nightly-dev-cu13-20261028-586355a8, which includes sgl-project/sglang#46404. On that image the KV pool sizes at 890 B/token, so hicache-ratio drops to 1.2 (1.8 exceeds host memory) and decode mem-fraction-static to 0.75 (0.85 runs out of memory at graph capture). Points: 5P c2884 (bs40), 8P c4096 (bs64) and 10P c7480 (bs160). The 9P and 10P bs128 c7480 points are dropped; bs160 is higher at the same interactivity. The dspark-disagg key on 65633129 and its recipe are retired. --- .../agentx/disagg-dep16-hicache-variants.yaml | 33 +-- .../gb300-fp4/agentx/disagg-variants.yaml | 254 ------------------ inferencex-e2e/configs/nvidia-master.yaml | 76 +----- inferencex-e2e/perf-changelog.yaml | 9 +- 4 files changed, 29 insertions(+), 343 deletions(-) delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml index bc5c156e40..b268ff7017 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml @@ -5,8 +5,8 @@ base: identity: model: {} frameworks: - dynamo: "1.7.0.dev20261026" - sglang: "0.0.0.dev1+g5884e4fe" + dynamo: "1.7.0.dev20261027" + sglang: "0.0.0.dev1+g586355a83" dynamo: install: false request_plane: tcp @@ -114,7 +114,7 @@ base: # Local host-DRAM tier: device-evicted prefixes reload from host memory off the # forward path; an external L3 lookup blocks each layer of the forward. enable-hierarchical-cache: true - hicache-ratio: 1.8 + hicache-ratio: 1.2 hicache-mem-layout: page_first_direct hicache-io-backend: direct hicache-write-policy: write_back @@ -180,7 +180,7 @@ base: stream-interval: 32 swa-prefix-tails: 2048 enable-decoder-swa-bounded-replay: true - mem-fraction-static: 0.85 + mem-fraction-static: 0.75 enable-eplb: true ep-num-redundant-experts: 16 eplb-rebalance-num-iterations: 15000 @@ -215,26 +215,29 @@ base: AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}' AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" -override_9p1d_dep2_dep16_c7480: - name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache-9p1d-dep2-dep16-c7480 +override_5p1d_dep2_dep16_c2884: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache-5p1d-dep2-dep16-c2884 roles: prefill: - # Nine DEP2 workers leave two GPUs of the fifth node idle. - workers: 9 + nodes: 3 + workers: 5 decode: args: - max-running-requests: 2048 - cuda-graph-max-bs-decode: 128 + max-running-requests: 640 + cuda-graph-max-bs-decode: 40 + disaggregation-decode-extra-slots: 80 -override_10p1d_dep2_dep16_c7480: - name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache-10p1d-dep2-dep16-c7480 +override_8p1d_dep2_dep16_c4096: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache-8p1d-dep2-dep16-c4096 roles: prefill: - workers: 10 + nodes: 4 + workers: 8 decode: args: - max-running-requests: 2048 - cuda-graph-max-bs-decode: 128 + max-running-requests: 1024 + cuda-graph-max-bs-decode: 64 + disaggregation-decode-extra-slots: 128 override_10p1d_dep2_dep16_b160_c7480: name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache-10p1d-dep2-dep16-b160-c7480 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml deleted file mode 100644 index e793e84ae8..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ /dev/null @@ -1,254 +0,0 @@ -schema: 2 - -base: - name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg - identity: - model: {} - frameworks: - dynamo: "1.6.0" - sglang: "0.0.0.dev1+g656331295" - dynamo: - install: false - request_plane: tcp - event_plane: zmq - # gb300-nv serves this model from the Hub; read it from the shared mounted cache. - environment: - HF_HUB_CACHE: /hf_hub_cache - # Tachometer's per-node exporters slow decode steps and add head-node host memory; - # InferenceX does not read its output. - observability: - tachometer: - enabled: false - slurm: - time_limit: "6:00:00" - health_check: - max_attempts: 1440 - interval_seconds: 10 - resources: - gpu_type: gb300 - gpus_per_node: 4 - services: - - name: etcd - type: etcd - placement: - node: infra - - name: mooncake-master - type: mooncake-master - args: - - --eviction_high_watermark_ratio=0.90 - - --client_ttl=60 - - --rpc_thread_num=64 - # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. - - name: otel-collector - type: otel-collector - enabled: false - frontend: - type: dynamo - enable_multiple_frontends: false - numa_bind: true - # One CPU node hosts the frontend, etcd, the Mooncake master and the AgentX client. - placement: - node: dedicated - env: - DYN_ROUTER_TEMPERATURE: "0" - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - args: - router-mode: kv - router-session-affinity-ttl-secs: "14400" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - engine: sglang - roles: - prefill: - gpus: 2 - env: - PYTHONUNBUFFERED: "1" - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "32768" - SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MOONCAKE_PROTOCOL: rdma - # The store registers host buffers only over RDMA; without a device list it - # picks NVLink for same-domain peers. - MOONCAKE_DEVICE: "@fabric.rdma-devices" - MOONCAKE_GLOBAL_SEGMENT_SIZE: 80gb - MOONCAKE_STANDALONE_STORAGE: "0" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - # Some GB300 nodes list a BMC interface with the same address on every node first. - MC_TCP_BIND_ADDRESS: "{node}" - WITH_NVIDIA_PEERMEM: "0" - args: - served-model-name: deepseek-ai/DeepSeek-V4.1-Flash - trust-remote-code: true - tensor-parallel-size: 2 - tokenizer-backend: fastokens - router-mode: kv - router-session-affinity-ttl-secs: "14400" - data-parallel-size: 2 - expert-parallel-size: 2 - enable-dp-attention: true - enable-dp-lm-head: true - enable-dp-attention-local-control-broadcast: true - moe-a2a-backend: megamoe - dsv4-attn-backend: flashmla_mega - disaggregation-transfer-backend: mooncake - chunked-prefill-size: 65536 - max-running-requests: 128 - swa-prefix-tails: 2048 - enable-decoder-swa-bounded-replay: true - enable-unified-cache-external-linker: true - unified-cache-external-linker-backend: mooncake - hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' - hicache-ratio: 1.02 - hicache-mem-layout: page_first_direct - mem-fraction-static: 0.85 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 5 - speculative-num-steps: 5 - reasoning-parser: auto - tool-call-parser: auto - watchdog-timeout: 3600 - enable-metrics: true - weight-loader-prefetch-checkpoints: true - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - kv_events: true - decode: - nodes: 4 - workers: 1 - gpus: 16 - env: - PYTHONUNBUFFERED: "1" - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - # One request gateway process saturates a CPU core serving 16 DP ranks. - DYN_SGL_GATEWAY_WORKERS: "8" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "16384" - SGLANG_OPT_DSV41_BF16_INDEX_LOGITS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MOONCAKE_PROTOCOL: rdma - MOONCAKE_DEVICE: "@fabric.rdma-devices" - MOONCAKE_STANDALONE_STORAGE: "0" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_TCP_BIND_ADDRESS: "{node}" - WITH_NVIDIA_PEERMEM: "0" - args: - served-model-name: deepseek-ai/DeepSeek-V4.1-Flash - trust-remote-code: true - tensor-parallel-size: 16 - tokenizer-backend: fastokens - router-mode: round-robin - load-balance-method: total_tokens - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - enable-dp-attention-local-control-broadcast: true - moe-a2a-backend: megamoe - dsv4-attn-backend: flashmla_mega - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - stream-interval: 32 - swa-prefix-tails: 2048 - enable-decoder-swa-bounded-replay: true - enable-unified-cache-external-linker: true - unified-cache-external-linker-backend: mooncake - mem-fraction-static: 0.85 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 5 - speculative-num-steps: 5 - reasoning-parser: auto - tool-call-parser: auto - watchdog-timeout: 3600 - enable-metrics: true - weight-loader-prefetch-checkpoints: true - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - kv_events: false - sbatch_directives: - mem: "0" - exclusive: "" - srun_options: - container-remap-root: "" - # Engram host tables need the node's full memory in every worker step. - mem: "0" - benchmark: - placement: - node: dedicated - colocate_with_frontend: true - env: - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - # The client splits this value on whitespace, so it has none. - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}' - AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" - -override_3p1d_dep2_dep16_c1228: - name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-3p1d-dep2-dep16-c1228 - roles: - prefill: - # Three DEP2 workers leave two GPUs of the second node idle. - nodes: 2 - workers: 3 - decode: - env: - MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb - args: - max-running-requests: 384 - cuda-graph-max-bs-decode: 24 - disaggregation-decode-extra-slots: 36 - -override_5p1d_dep2_dep16_c2884: - name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-5p1d-dep2-dep16-c2884 - roles: - prefill: - # Five DEP2 workers leave two GPUs of the third node idle. - nodes: 3 - workers: 5 - args: - max-prefill-tokens: 32768 - decode: - env: - MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb - args: - max-running-requests: 640 - cuda-graph-max-bs-decode: 40 - disaggregation-decode-extra-slots: 80 - -override_8p1d_dep2_dep16_c4096: - name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-8p1d-dep2-dep16-c4096 - roles: - prefill: - nodes: 4 - workers: 8 - args: - max-prefill-tokens: 32768 - decode: - env: - MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb - args: - max-running-requests: 1024 - cuda-graph-max-bs-decode: 64 - disaggregation-decode-extra-slots: 128 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 1b5481370a..ed8d058196 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9745,14 +9745,14 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false -dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20261025-65633129@sha256:069efa1d6a2d58309529f53eeee34c39d69b1cdb455eaf7fb8de83e3fd0be727 +dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache: + image: lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.6.0.dev20261019" } + router: { name: dynamo-router, version: "1.7.0.dev20261027" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -9761,26 +9761,11 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg: agentic-coding: - dram-utilization: 0.80 search-space: - - spec-decoding: mtp - conc-list: [1228] - srt-recipe: disagg-variants.yaml:override_3p1d_dep2_dep16_c1228 - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - spec-decoding: mtp conc-list: [2884] - srt-recipe: disagg-variants.yaml:override_5p1d_dep2_dep16_c2884 + srt-recipe: disagg-dep16-hicache-variants.yaml:override_5p1d_dep2_dep16_c2884 kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } + kv-offload-backend: { name: hicache } prefill: num-worker: 5 tp: 2 @@ -9793,58 +9778,11 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg: dp-attn: true - spec-decoding: mtp conc-list: [4096] - srt-recipe: disagg-variants.yaml:override_8p1d_dep2_dep16_c4096 - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } - prefill: - num-worker: 8 - tp: 2 - ep: 2 - dp-attn: true - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - -dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache: - image: lmsysorg/sglang:nightly-dev-cu13-20261027-5884e4fe@sha256:61b9083bdcb7cf3741366f487cb996c05ea33ad2ddddc820d10cbc1c0cfb0a86 - model: deepseek-ai/DeepSeek-V4.1-Flash - model-prefix: dsv41flash - runner: cluster:gb300-nv - precision: fp4 - framework: dynamo-sglang - router: { name: dynamo-router, version: "1.7.0.dev20261026" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - srt-recipe-dir: dsv41flash/sglang/gb300-fp4/agentx - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - spec-decoding: mtp - conc-list: [7480] - srt-recipe: disagg-dep16-hicache-variants.yaml:override_9p1d_dep2_dep16_c7480 + srt-recipe: disagg-dep16-hicache-variants.yaml:override_8p1d_dep2_dep16_c4096 kv-offloading: dram kv-offload-backend: { name: hicache } prefill: - num-worker: 9 - tp: 2 - ep: 2 - dp-attn: true - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: mtp - conc-list: [7480] - srt-recipe: disagg-dep16-hicache-variants.yaml:override_10p1d_dep2_dep16_c7480 - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 10 + num-worker: 8 tp: 2 ep: 2 dp-attn: true diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index a72278d549..3bca1a13de 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11492,13 +11492,12 @@ - config-keys: - dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg - - dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg - dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache scenario-type: [agentic-coding] description: - - "Add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache on nightly-dev-cu13-20261027-5884e4fe (Dynamo 1.7.0.dev20261026), pinned by digest: DEP2 prefill and one DEP16 decode at c7480 with 9 prefill workers (decode CUDA graph batch 128), 10 prefill workers (batch 128) and 10 prefill workers (batch 160, max-running-requests 2560)." - - "Prefill keeps evicted prefixes in a local host-DRAM HiCache tier (ratio 1.8, write-back, direct I/O, mem-fraction-static 0.85) instead of the Mooncake external linker, whose layer-wise L3 reads block the forward pass and stall prefill after the client's turn burst at c7480. Decode admits transferred requests every step (SGLang's default disaggregation-decode-polling-interval of 1; the dspark-disagg key sets 8)." + - "Add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache on nightly-dev-cu13-20261028-586355a8 (Dynamo 1.7.0.dev20261027), pinned by digest: DEP2 prefill and one DEP16 decode at c2884 with 5 prefill workers (decode CUDA graph batch 40), c4096 with 8 prefill workers (batch 64) and c7480 with 10 prefill workers (batch 160, max-running-requests 2560). The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row)." + - "Prefill keeps evicted prefixes in a local host-DRAM HiCache tier (ratio 1.2, write-back, direct I/O, mem-fraction-static 0.85) instead of the Mooncake external linker, whose layer-wise L3 reads block the forward pass and stall prefill after the client's turn burst. Decode runs at mem-fraction-static 0.75 and admits transferred requests every step (SGLang's default disaggregation-decode-polling-interval of 1)." - "Dynamo 1.7 backend admission limits are raised to 10000 so waiting turns queue in the prefill scheduler instead of failing with 529, and runners/srt-slurm/patches backports NVIDIA/srt-slurm#629 so the readiness wait asks /health?instances=true when /health is status-only." - - "Move dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg to nightly-dev-cu13-20261028-586355a8 (Dynamo 1.7.0.dev20261027), pinned by digest, with the Dynamo 1.7 backend admission limits raised to 10000. The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row). Remove the TP2/EP2 c16 point; TP4 c1/c4/c16 keep their settings." - - "The existing dspark-disagg key is unchanged; it reruns in the same sweep because AgentX plots all dynamo-sglang points of this model and GPU as one curve." + - "Retire dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg (nightly-dev-cu13-20261025-65633129, Mooncake external linker) and its disagg-variants.yaml recipe; the hicache key replaces its c2884 and c4096 points and drops c1228." + - "Move dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg to nightly-dev-cu13-20261028-586355a8 (Dynamo 1.7.0.dev20261027), pinned by digest, with the Dynamo 1.7 backend admission limits raised to 10000. Remove the TP2/EP2 c16 point; TP4 c1/c4/c16 keep their settings." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 From eb4a618735a6a93b92ba9adb053b04514802f48c Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:42:36 +0800 Subject: [PATCH 6/9] Use 9 prefill workers for the c7480 HiCache point At decode batch 160, 9 x DEP2 reaches the same p90 interactivity as 10 x DEP2 with two fewer GPUs, so its throughput per GPU is higher. --- .../gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml | 7 ++++--- inferencex-e2e/configs/nvidia-master.yaml | 4 ++-- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml index b268ff7017..6bfb092ef5 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-dep16-hicache-variants.yaml @@ -239,11 +239,12 @@ override_8p1d_dep2_dep16_c4096: cuda-graph-max-bs-decode: 64 disaggregation-decode-extra-slots: 128 -override_10p1d_dep2_dep16_b160_c7480: - name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache-10p1d-dep2-dep16-b160-c7480 +override_9p1d_dep2_dep16_b160_c7480: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-dep16-hicache-9p1d-dep2-dep16-b160-c7480 roles: prefill: - workers: 10 + # Nine DEP2 workers leave two GPUs of the fifth node idle. + workers: 9 decode: args: max-running-requests: 2560 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index ed8d058196..e2c4d98be6 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9793,11 +9793,11 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache: dp-attn: true - spec-decoding: mtp conc-list: [7480] - srt-recipe: disagg-dep16-hicache-variants.yaml:override_10p1d_dep2_dep16_b160_c7480 + srt-recipe: disagg-dep16-hicache-variants.yaml:override_9p1d_dep2_dep16_b160_c7480 kv-offloading: dram kv-offload-backend: { name: hicache } prefill: - num-worker: 10 + num-worker: 9 tp: 2 ep: 2 dp-attn: true diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3bca1a13de..21fb92c706 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11495,7 +11495,7 @@ - dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache scenario-type: [agentic-coding] description: - - "Add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache on nightly-dev-cu13-20261028-586355a8 (Dynamo 1.7.0.dev20261027), pinned by digest: DEP2 prefill and one DEP16 decode at c2884 with 5 prefill workers (decode CUDA graph batch 40), c4096 with 8 prefill workers (batch 64) and c7480 with 10 prefill workers (batch 160, max-running-requests 2560). The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row)." + - "Add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache on nightly-dev-cu13-20261028-586355a8 (Dynamo 1.7.0.dev20261027), pinned by digest: DEP2 prefill and one DEP16 decode at c2884 with 5 prefill workers (decode CUDA graph batch 40), c4096 with 8 prefill workers (batch 64) and c7480 with 9 prefill workers (batch 160, max-running-requests 2560). The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row)." - "Prefill keeps evicted prefixes in a local host-DRAM HiCache tier (ratio 1.2, write-back, direct I/O, mem-fraction-static 0.85) instead of the Mooncake external linker, whose layer-wise L3 reads block the forward pass and stall prefill after the client's turn burst. Decode runs at mem-fraction-static 0.75 and admits transferred requests every step (SGLang's default disaggregation-decode-polling-interval of 1)." - "Dynamo 1.7 backend admission limits are raised to 10000 so waiting turns queue in the prefill scheduler instead of failing with 529, and runners/srt-slurm/patches backports NVIDIA/srt-slurm#629 so the readiness wait asks /health?instances=true when /health is status-only." - "Retire dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg (nightly-dev-cu13-20261025-65633129, Mooncake external linker) and its disagg-variants.yaml recipe; the hicache key replaces its c2884 and c4096 points and drops c1228." From bb5e630152c30d5e9c56892bf3058d667ea64121 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:45:32 +0800 Subject: [PATCH 7/9] Name the KV layout, precision and Engram changes against the retired recipe in the changelog --- inferencex-e2e/perf-changelog.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 21fb92c706..163e1c783b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11498,6 +11498,7 @@ - "Add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache on nightly-dev-cu13-20261028-586355a8 (Dynamo 1.7.0.dev20261027), pinned by digest: DEP2 prefill and one DEP16 decode at c2884 with 5 prefill workers (decode CUDA graph batch 40), c4096 with 8 prefill workers (batch 64) and c7480 with 9 prefill workers (batch 160, max-running-requests 2560). The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row)." - "Prefill keeps evicted prefixes in a local host-DRAM HiCache tier (ratio 1.2, write-back, direct I/O, mem-fraction-static 0.85) instead of the Mooncake external linker, whose layer-wise L3 reads block the forward pass and stall prefill after the client's turn burst. Decode runs at mem-fraction-static 0.75 and admits transferred requests every step (SGLang's default disaggregation-decode-polling-interval of 1)." - "Dynamo 1.7 backend admission limits are raised to 10000 so waiting turns queue in the prefill scheduler instead of failing with 529, and runners/srt-slurm/patches backports NVIDIA/srt-slurm#629 so the readiness wait asks /health?instances=true when /health is status-only." + - "Compared with the retired dspark-disagg recipe, both roles set SGLANG_DSV4_KV_LAYOUT=v41_new, prefill also sets SGLANG_OPT_DSV41_BF16_INDEX_LOGITS=1, and decode sets enable-fp32-lm-head and reads Engram rows from peer shards (SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS=1) instead of a host table." - "Retire dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg (nightly-dev-cu13-20261025-65633129, Mooncake external linker) and its disagg-variants.yaml recipe; the hicache key replaces its c2884 and c4096 points and drops c1228." - "Move dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg to nightly-dev-cu13-20261028-586355a8 (Dynamo 1.7.0.dev20261027), pinned by digest, with the Dynamo 1.7 backend admission limits raised to 10000. Remove the TP2/EP2 c16 point; TP4 c1/c4/c16 keep their settings." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 From 1b0252fcc507d6c3bdf2bb38494e5fbf428e31f7 Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:34:48 +0800 Subject: [PATCH 8/9] Move the GB300 DSV4.1-Flash Dynamo + SGLang agg row to the 20261029 nightly and use the megakernel runtime-optimization switch at TP4 c1 nightly-dev-cu13-20261029-c10e0e17 bundles sgl-nvidia-mega 54b428921749 (megakernel API 2, sgl-project/sglang#47145), so TP4 c1 starts with --enable-nvidia-megakernel again. SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION (sgl-project/sglang#47105) replaces the 16 runtime-optimization variables the recipe set one by one. --- .../sglang/gb300-fp4/agentx/agg-variants.yaml | 21 +++---------------- inferencex-e2e/configs/nvidia-master.yaml | 4 ++-- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml index 66a9dfc3b7..c7a86bf934 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -5,8 +5,8 @@ base: identity: model: {} frameworks: - dynamo: "1.7.0.dev20261027" - sglang: "0.0.0.dev1+g586355a83" + dynamo: "1.7.0.dev20261028" + sglang: "0.0.0.dev1+gc10e0e177" dynamo: install: false request_plane: tcp @@ -127,24 +127,9 @@ override_tp4_c1: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank SGLANG_NVIDIA_MEGAKERNEL_SLOTS: hop_a,moe,gate,attn,embed,two_shot,markov,draft_lm_head,aux_mean_stats,gate_wide,shared_ll,hop_a_batch2,engram,attn_qkv_a,attn_qkv_a_bf16,index_q_mxfp8_in,attn_indexer,hc_prenorm,hop_b,hop_b_wide,early_trigger,hop_b_start_trigger,draft,draft_moe,draft_gate,draft_tail,draft_post,draft_seam,draft_attn,draft_qfront,draft_wo_a,draft_swapab,draft_qkv_a,draft_prenorm,draft_qkv_a_ll,draft_wo_b_ll,sched_ahead,req_pos_once,kv_before_wait,target_lm_head,hop_b_quad_trigger,lm_head_pinned_tactic - SGLANG_DSPARK_FUSED_ACCEPT_TAIL: "1" - SGLANG_DSPARK_DEFER_CONTEXT_KV: "1" - SGLANG_OPT_DSV41_DSPARK_DRAFT_PREP_FORK: "1" - SGLANG_OPT_DSV41_DSPARK_POST_FOLD: "1" - SGLANG_OPT_DSV41_DSPARK_SKIP_PREFILLED_INPUTS: "1" - SGLANG_OPT_DSV41_DSPARK_ONE_LAUNCH_COMMIT_STAGE: "1" - SGLANG_OPT_DSV41_DSPARK_PREPARE_INPUTS_ONCE: "1" - SGLANG_OPT_DSV41_DSPARK_MID_FOLD: "1" - SGLANG_OPT_DSV41_TARGET_VERIFY_METADATA_GLUE: "1" - SGLANG_OPT_DSV41_TARGET_METADATA_EARLY: "1" - SGLANG_OPT_SKIP_GLUED_REPLAY_VIEW: "1" - SGLANG_OPT_DSV41_POST_FORWARD_CUDA_GRAPH: "1" - SGLANG_OPT_DSV41_BLOCKWISE_REJECTION: "1" - SGLANG_OPT_DSV41_SAMPLING_ACCEPT_GRAPH: "1" + SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION: "1" SGLANG_SIMULATE_ACC_GREEDY: "0" SGLANG_OPT_DSV41_DSPARK_SIM_DRAW_AHEAD: "1" - SGLANG_OPT_MXFP4_DEFERRED_FINALIZED_SHAPE: "1" - SGLANG_OPT_CUDA_GRAPH_FILL_FAST_PATH: "1" NCCL_CUMEM_ENABLE: "0" NCCL_MNNVL_ENABLE: "0" diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 8979d8edc4..3ea3f47266 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9732,13 +9732,13 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark: - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128, 160], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 16], srt-recipe: agentic.yaml } dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20261028-586355a8@sha256:b4f09a2ccbcfcc262d49600c3cd7f8581a793ea7d104947257b6a95500953b90 + image: lmsysorg/sglang:nightly-dev-cu13-20261029-c10e0e17@sha256:78142f9f4b7fb3ee27f59065615dc7bb263f643ed256c9299c3861086ade4d47 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.7.0.dev20261027" } + router: { name: dynamo-router, version: "1.7.0.dev20261028" } multinode: true disagg: false srt-recipe-dir: dsv41flash/sglang/gb300-fp4/agentx diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 814145d563..00ed44e587 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11567,5 +11567,5 @@ - "Dynamo 1.7 backend admission limits are raised to 10000 so waiting turns queue in the prefill scheduler instead of failing with 529, and runners/srt-slurm/patches backports NVIDIA/srt-slurm#629 so the readiness wait asks /health?instances=true when /health is status-only." - "Compared with the retired dspark-disagg recipe, both roles set SGLANG_DSV4_KV_LAYOUT=v41_new, prefill also sets SGLANG_OPT_DSV41_BF16_INDEX_LOGITS=1, and decode sets enable-fp32-lm-head and reads Engram rows from peer shards (SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS=1) instead of a host table." - "Retire dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg (nightly-dev-cu13-20261025-65633129, Mooncake external linker) and its disagg-variants.yaml recipe; the hicache key replaces its c2884 and c4096 points and drops c1228." - - "Move dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg to nightly-dev-cu13-20261028-586355a8 (Dynamo 1.7.0.dev20261027), pinned by digest, with the Dynamo 1.7 backend admission limits raised to 10000. Remove the TP2/EP2 c16 point; TP4 c1/c4/c16 keep their settings." + - "Move dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg to nightly-dev-cu13-20261029-c10e0e17 (Dynamo 1.7.0.dev20261028), pinned by digest, with the Dynamo 1.7 backend admission limits raised to 10000. The image includes sgl-project/sglang#46404 and bundles sgl-nvidia-mega 54b428921749 (megakernel API 2), so TP4 c1 keeps the NVIDIA megakernel; its 16 explicit megakernel runtime-optimization variables are replaced by SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION=1 (sgl-project/sglang#47105), which also enables the Qwen3.5/EAGLE runtime group. Remove the TP2/EP2 c16 point; TP4 c4/c16 keep their settings." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4595 From 4145e263b55bbb071885a6c17bb40c5463f6f70e Mon Sep 17 00:00:00 2001 From: weireweire <20922698+weireweire@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:02:30 +0800 Subject: [PATCH 9/9] Drop the srt-slurm #629 backport patch now that main pins srt-slurm v2.65.2 Main bumped inferencex-e2e/utils/srt-slurm to v2.65.2 (#4601), which includes NVIDIA/srt-slurm#629, so runners/srt-slurm/patches/629-dynamo-health-instances.patch and its README row are removed and the changelog entry no longer mentions it. --- inferencex-e2e/perf-changelog.yaml | 2 +- .../patches/629-dynamo-health-instances.patch | 48 ------------------- .../runners/srt-slurm/patches/README.md | 1 - 3 files changed, 1 insertion(+), 50 deletions(-) delete mode 100644 inferencex-e2e/runners/srt-slurm/patches/629-dynamo-health-instances.patch diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index d542ef2bc1..bb6bc829bb 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11678,7 +11678,7 @@ description: - "Add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg-hicache on nightly-dev-cu13-20261028-586355a8 (Dynamo 1.7.0.dev20261027), pinned by digest: DEP2 prefill and one DEP16 decode at c2884 with 5 prefill workers (decode CUDA graph batch 40), c4096 with 8 prefill workers (batch 64) and c7480 with 9 prefill workers (batch 160, max-running-requests 2560). The image includes sgl-project/sglang#46404 (DSpark simulated-acceptance bonus drawn at the simulated row)." - "Prefill keeps evicted prefixes in a local host-DRAM HiCache tier (ratio 1.2, write-back, direct I/O, mem-fraction-static 0.85) instead of the Mooncake external linker, whose layer-wise L3 reads block the forward pass and stall prefill after the client's turn burst. Decode runs at mem-fraction-static 0.75 and admits transferred requests every step (SGLang's default disaggregation-decode-polling-interval of 1)." - - "Dynamo 1.7 backend admission limits are raised to 10000 so waiting turns queue in the prefill scheduler instead of failing with 529, and runners/srt-slurm/patches backports NVIDIA/srt-slurm#629 so the readiness wait asks /health?instances=true when /health is status-only." + - "Dynamo 1.7 backend admission limits are raised to 10000 so waiting turns queue in the prefill scheduler instead of failing with 529." - "Compared with the retired dspark-disagg recipe, both roles set SGLANG_DSV4_KV_LAYOUT=v41_new, prefill also sets SGLANG_OPT_DSV41_BF16_INDEX_LOGITS=1, and decode sets enable-fp32-lm-head and reads Engram rows from peer shards (SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS=1) instead of a host table." - "Retire dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg (nightly-dev-cu13-20261025-65633129, Mooncake external linker) and its disagg-variants.yaml recipe; the hicache key replaces its c2884 and c4096 points and drops c1228." - "Move dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg to nightly-dev-cu13-20261029-c10e0e17 (Dynamo 1.7.0.dev20261028), pinned by digest, with the Dynamo 1.7 backend admission limits raised to 10000. The image includes sgl-project/sglang#46404 and bundles sgl-nvidia-mega 54b428921749 (megakernel API 2), so TP4 c1 keeps the NVIDIA megakernel; its 16 explicit megakernel runtime-optimization variables are replaced by SGLANG_ENABLE_NV_MEGA_RUNTIME_OPTIMIZATION=1 (sgl-project/sglang#47105), which also enables the Qwen3.5/EAGLE runtime group. Remove the TP2/EP2 c16 point; TP4 c4/c16 keep their settings." diff --git a/inferencex-e2e/runners/srt-slurm/patches/629-dynamo-health-instances.patch b/inferencex-e2e/runners/srt-slurm/patches/629-dynamo-health-instances.patch deleted file mode 100644 index e0092720d4..0000000000 --- a/inferencex-e2e/runners/srt-slurm/patches/629-dynamo-health-instances.patch +++ /dev/null @@ -1,48 +0,0 @@ -diff --git a/src/srtctl/frontends/dynamo.py b/src/srtctl/frontends/dynamo.py ---- a/src/srtctl/frontends/dynamo.py -+++ b/src/srtctl/frontends/dynamo.py -@@ -12,6 +12,7 @@ import shlex - import threading - from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast - -+import requests - import yaml - - from srtctl.backends.vllm import VLLMBackend -@@ -42,6 +43,8 @@ logger = logging.getLogger(__name__) - - ROUTER_POLICY_CONFIG_FILENAME = "router_policy_config.yaml" - ROUTER_POLICY_CONFIG_CONTAINER_PATH = f"/logs/{ROUTER_POLICY_CONFIG_FILENAME}" -+# Since ai-dynamo/dynamo#16454 (a789babbe3) /health is status-only unless asked for its instance list. -+HEALTH_INSTANCES_ENDPOINT = "/health?instances=true" - - - def _vllm_mode_config(backend: VLLMBackend, mode: str) -> dict[str, Any]: -@@ -176,9 +179,26 @@ class DynamoFrontend(DynamicFrontend): - expected_prefill: int, - expected_decode: int, - ) -> WorkerHealthResult: -- """Parse dynamo /health endpoint response.""" -+ """Count the generate instances in a Dynamo /health instance listing (see ``probe_ready``).""" - return check_dynamo_health(response_json, expected_prefill, expected_decode) - -+ def probe_ready( -+ self, host: str, port: int, expected_prefill: int, expected_decode: int, config: Any -+ ) -> WorkerHealthResult: -+ """Count instances from ``/health``, asking ``/health?instances=true`` when it answers status-only. -+ -+ Frontends before ai-dynamo/dynamo#16454 always list instances in ``/health`` and get the one -+ request they always got; newer ones answer ``{"status": "healthy", "stage": "ready"}`` there. -+ """ -+ for endpoint in (self.health_endpoint, HEALTH_INSTANCES_ENDPOINT): -+ response = requests.get(f"http://{host}:{port}{endpoint}", timeout=5.0) -+ if response.status_code != 200: -+ return WorkerHealthResult(ready=False, message=f"{endpoint} returned HTTP {response.status_code}") -+ body = response.json() -+ if "instances" in body: -+ break -+ return self.parse_health(body, expected_prefill, expected_decode) -+ - def health_expectations(self, config: Any, processes: list["Process"] | None) -> tuple[int, int, str]: - """Dynamo's /health counts registered generate instances. - diff --git a/inferencex-e2e/runners/srt-slurm/patches/README.md b/inferencex-e2e/runners/srt-slurm/patches/README.md index 7881645376..d9052d1aa8 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/README.md +++ b/inferencex-e2e/runners/srt-slurm/patches/README.md @@ -9,4 +9,3 @@ Each patch backports an upstream fix that the shared submodule pin does not incl | Patch | Upstream PR | Fix | |-------|-------------|-----| | `573-participating-gpus.patch` | [NVIDIA/srt-slurm#573](https://github.com/NVIDIA/srt-slurm/pull/573) | Record only the GPUs workers occupy on a worker node; an exclusive node's exporter (DCGM or AMD) reports every GPU, which the collector otherwise flags as `unexpected_device`. The pin includes #573, but upstream merged a different design: it keeps every row and excuses idle worker-node GPUs only after a DCGM worker GPU UUID probe. [`infx/results/power/multinode.py`](../../../infx/results/power/multinode.py) does not apply that excusal yet, so keep this filter until it does. | -| `629-dynamo-health-instances.patch` | [NVIDIA/srt-slurm#629](https://github.com/NVIDIA/srt-slurm/pull/629) | Dynamo frontends since ai-dynamo/dynamo#16454 (1.7 dev builds) answer `/health` status-only and list instances only for `/health?instances=true`, so the readiness wait never counted any workers. `DynamoFrontend.probe_ready` reads `/health` and asks `/health?instances=true` only when the answer has no `instances` key; older frontends get the same single request. |