diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index ac4347b3fe..de76e89d9e 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -283,7 +283,7 @@ jobs: - name: Prepare launcher Python env: *uv-setup-env run: | - launch_venv="$RUNNER_TEMP/infx-launch-venv" + launch_venv="$(mktemp -d "$RUNNER_TEMP/infx-launch-venv.XXXXXX")" uv venv --quiet --python 3.12 "$launch_venv" uv pip install --quiet --exclude-newer PT12H --python "$launch_venv/bin/python" \ -r "$GITHUB_WORKSPACE/.result-tooling/inferencex-e2e/pyproject.toml" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4/agentx/disagg-variants.yaml index fe8a0cbe8c..8969c6fb1c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/vllm/gb300-fp4/agentx/disagg-variants.yaml @@ -217,3 +217,27 @@ override_8p1d_dep2_dep16_c4096: max-num-seqs: 72 max-num-batched-tokens: 720 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","max_cudagraph_capture_size":432,"cudagraph_capture_sizes":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192,198,204,210,216,222,228,234,240,246,252,258,264,270,276,282,288,294,300,306,312,318,324,330,336,342,348,354,360,366,372,378,384,390,396,402,408,414,420,426,432]}' + +override_10p1d_dep2_dep16_c6656: + name: "dsv41flash-fp4-gb300-dynamo-vllm-agentx-disagg-10p1d-dep2-dep16-c6656" + roles: + prefill: + nodes: 5 + workers: 10 + decode: + args: + max-num-seqs: 132 + max-num-batched-tokens: 1320 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","max_cudagraph_capture_size":792,"cudagraph_capture_sizes":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192,198,204,210,216,222,228,234,240,246,252,258,264,270,276,282,288,294,300,306,312,318,324,330,336,342,348,354,360,366,372,378,384,390,396,402,408,414,420,426,432,438,444,450,456,462,468,474,480,486,492,498,504,510,516,522,528,534,540,546,552,558,564,570,576,582,588,594,600,606,612,618,624,630,636,642,648,654,660,666,672,678,684,690,696,702,708,714,720,726,732,738,744,750,756,762,768,774,780,786,792]}' + +override_8p1d_dep2_dep16_c5120: + name: "dsv41flash-fp4-gb300-dynamo-vllm-agentx-disagg-8p1d-dep2-dep16-c5120" + roles: + prefill: + nodes: 4 + workers: 8 + decode: + args: + max-num-seqs: 70 + max-num-batched-tokens: 700 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","max_cudagraph_capture_size":420,"cudagraph_capture_sizes":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192,198,204,210,216,222,228,234,240,246,252,258,264,270,276,282,288,294,300,306,312,318,324,330,336,342,348,354,360,366,372,378,384,390,396,402,408,414,420]}' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 1fc173c6a4..69dc0904bd 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9187,6 +9187,36 @@ dsv41flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg: tp: 16 ep: 16 dp-attn: true + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + srt-recipe: disagg-variants.yaml:override_8p1d_dep2_dep16_c5120 + conc-list: [5120] + prefill: + num-worker: 8 + tp: 2 + ep: 2 + dp-attn: true + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + srt-recipe: disagg-variants.yaml:override_10p1d_dep2_dep16_c6656 + conc-list: [6656] + prefill: + num-worker: 10 + tp: 2 + ep: 2 + dp-attn: true + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true # One serving GPU, stock ARM64 release, native AgentX client and proportional host cache. qwen3.8next-fp4-gb300-sglang-agentic-mtp: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 8ec368892e..fefd7900d2 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10013,3 +10013,12 @@ - "Add a HiCache host-DRAM KV offload arm at concurrency 2, 4 and 8 with 160 GB per rank, alongside the GPU-resident sweep; the few-GB GPU KV pool left by 67 GB of weights per H100 evicts agentic prefixes between turns." - "Keep the batch-8 cap on running requests and decode graphs from the GPU-resident sweep." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4181 + +- config-keys: + - dsv41flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg + scenario-type: + - agentic-coding + description: + - "Append GB300 DeepSeek-V4.1-Flash Dynamo + vLLM AgentX c5120 (8P1D) and c6656 (10P1D) points using the agentx-harness 1.0.10 release already pinned on main." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4126 + append-only: true