diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml new file mode 100644 index 0000000000..4d5b07b59e --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/agg-variants.yaml @@ -0,0 +1,164 @@ +schema: 2 +base: + name: minimaxm31flash-gb300-dynamo-agg + identity: + frameworks: + dynamo: 1.6.0.dev20261023 + dynamo: + install: true + source: + wheel: 1.6.0.dev20261023 + resources: + gpu_type: gb300 + gpus_per_node: 4 + # Cold loads of the 250 GB checkpoint from the hub cache run past half an hour. + observability: + tachometer: + enabled: false + health_check: + max_attempts: 1440 + interval_seconds: 5 + environment: + PYTHONHASHSEED: '0' + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: kv + router-session-affinity-ttl-secs: 14400 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + DYN_TCP_CONNECT_TIMEOUT: '120' + engine: + type: vllm + connector: null + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + kv_events: true + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + VLLM_FLOAT32_MATMUL_PRECISION: high + PYTHONUNBUFFERED: '1' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: '600' + DYN_TCP_CONNECT_TIMEOUT: '120' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: '1024' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: '1024' + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: '2048' + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + language-model-only: true + trust-remote-code: true + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + # MSA sparse/index cache needs 128-token blocks. + block-size: 128 + # The checkpoint is QAT-trained for an NVFP4 KV cache. + kv-cache-dtype: nvfp4 + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + enable-prefix-caching: true + tensor-parallel-size: 4 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + max-num-batched-tokens: 32768 + # Prefill CUDA graphs up to 8192 tokens; the default capture stops at 512. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,320,384,448,512,576,640,704,768,832,896,960,1024,1536,2048,3072,4096,6144,8192]}' + # Golden AL 4.04. + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASH_ATTN","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + sbatch_directives: + cpus-per-task: '144' + mem: '0' + srun_options: + container-remap-root: '' + benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + +override_tp4_c1: + name: minimaxm31flash-gb300-dynamo-agg-tp4-c1 + roles: + agg: + args: + max-num-seqs: 2 + +override_tp4_c4: + name: minimaxm31flash-gb300-dynamo-agg-tp4-c4 + roles: + agg: + args: + max-num-seqs: 8 + +override_tp4_c8: + name: minimaxm31flash-gb300-dynamo-agg-tp4-c8 + roles: + agg: + args: + max-num-seqs: 16 + +override_tp4_c16: + name: minimaxm31flash-gb300-dynamo-agg-tp4-c16 + roles: + agg: + args: + max-num-seqs: 32 + +override_tp4_c32: + name: minimaxm31flash-gb300-dynamo-agg-tp4-c32 + roles: + agg: + args: + max-num-seqs: 64 + +override_tp4_c40: + name: minimaxm31flash-gb300-dynamo-agg-tp4-dram-c40 + roles: + agg: + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + args: + max-num-seqs: 64 + gpu-memory-utilization: 0.86 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' + +override_tp4_c48: + name: minimaxm31flash-gb300-dynamo-agg-tp4-dram-c48 + roles: + agg: + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + args: + max-num-seqs: 64 + gpu-memory-utilization: 0.86 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' + +override_tp4_c56: + name: minimaxm31flash-gb300-dynamo-agg-tp4-dram-c56 + roles: + agg: + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + args: + max-num-seqs: 96 + gpu-memory-utilization: 0.86 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":860000000000,"lazy_offload":true}}' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml new file mode 100644 index 0000000000..6bf4f89e09 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm31flash/vllm/gb300-fp4/agentx/disagg-mooncake.yaml @@ -0,0 +1,221 @@ +schema: 2 +base: + identity: + frameworks: + dynamo: 1.6.0.dev20261023 + dynamo: + install: true + source: + wheel: 1.6.0.dev20261023 + resources: + gpu_type: gb300 + gpus_per_node: 4 + observability: + tachometer: + enabled: false + health_check: + max_attempts: 1440 + interval_seconds: 5 + environment: + PYTHONHASHSEED: '0' + NCCL_NET_GDR_C2C: '1' + NVSHMEM_REMOTE_TRANSPORT: none + NVSHMEM_ENABLE_NIC_PE_MAPPING: '1' + NVSHMEM_HCA_PE_MAPPING: mlx5_0:1:1,mlx5_1:1:1,mlx5_2:1:1,mlx5_3:1:1 + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + preamble: bash /configs/vllm-mooncake.sh + args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + options: + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 160GB + local_buffer_size: 4GB + protocol: rdma + device_name: '@fabric.rdma-devices' + enable_offload: false + frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: kv + router-session-affinity-ttl-secs: 14400 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + DYN_TCP_CONNECT_TIMEOUT: '120' + placement: + node: head + engine: + type: vllm + connector: null + dp_launch_mode: per_node + roles: + prefill: + kv_events: true + nodes: 3 + workers: 6 + gpus: 2 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + DYN_TCP_CONNECT_TIMEOUT: '120' + UCX_CUDA_IPC_ENABLE_MNNVL: 'n' + UCX_MEMTYPE_CACHE: 'n' + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_REG_WHOLE: 'n' + UCX_RNDV_PIPELINE_ERROR_HANDLING: 'y' + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + NIXL_UCX_SGL_ENABLE: '1' + NCCL_IB_HCA: '@fabric.rdma-devices' + VLLM_USE_V2_MODEL_RUNNER: '1' + NIXL_TELEMETRY_ENABLE: 'y' + NIXL_TELEMETRY_PROMETHEUS_PORT: '0' + UCX_NET_DEVICES: '@fabric.rdma-device-ports' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: '1024' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: '1024' + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: '2048' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '8' + MC_SLICE_SIZE: '1048576' + MC_MAX_MR_SIZE: '4294967296' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + language-model-only: true + trust-remote-code: true + enable-expert-parallel: true + kv-cache-dtype: nvfp4 + block-size: 128 + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + max-model-len: 1048576 + kernel-config: '{"moe_backend":"deep_gemm_mega_moe"}' + numa-bind: true + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + tensor-parallel-size: 1 + gpu-memory-utilization: 0.85 + max-num-seqs: 64 + max-num-batched-tokens: 32768 + long-prefill-token-threshold: 4096 + prefill-schedule-interval: 1 + enable-prefix-caching: true + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + max-cudagraph-capture-size: 2048 + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"kv_lease_duration":120}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}]}}' + data-parallel-size: 2 + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_ENGINE_READY_TIMEOUT_S: '7200' + PYTHONUNBUFFERED: '1' + DYN_TCP_CONNECT_TIMEOUT: '120' + UCX_CUDA_IPC_ENABLE_MNNVL: 'n' + UCX_MEMTYPE_CACHE: 'n' + UCX_TLS: rc,cuda_copy + UCX_MEMTYPE_REG_WHOLE: 'n' + UCX_RNDV_PIPELINE_ERROR_HANDLING: 'y' + UCX_MODULE_DIR: /usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx + NIXL_UCX_SGL_ENABLE: '1' + NCCL_IB_HCA: '@fabric.rdma-devices' + VLLM_USE_V2_MODEL_RUNNER: '1' + NIXL_TELEMETRY_ENABLE: 'y' + NIXL_TELEMETRY_PROMETHEUS_PORT: '0' + UCX_NET_DEVICES: '@fabric.rdma-device-ports' + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: '600' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_LIMIT: '1024' + DYN_BACKEND_ADMISSION_ENGINE_REQUEST_WAITING_LIMIT: '1024' + DYN_BACKEND_ADMISSION_REQUEST_QUEUE_LIMIT: '2048' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + MC_MAX_MR_SIZE: '4294967296' + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + VLLM_MAX_ASYNC_KV_ADMITS_PER_STEP: '4' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + VLLM_ALLREDUCE_USE_SYMM_MEM: '0' + args: + served-model-name: MiniMaxAI/MiniMax-M3.1-Flash + language-model-only: true + trust-remote-code: true + enable-expert-parallel: true + kv-cache-dtype: nvfp4 + block-size: 128 + attention-config: '{"indexer_kv_dtype":"nvfp4","use_prefill_query_quantization":true}' + disable-hybrid-kv-cache-manager: true + max-model-len: 1048576 + kernel-config: '{"moe_backend":"deep_gemm_mega_moe"}' + numa-bind: true + dyn-tool-call-parser: minimax_m3 + dyn-reasoning-parser: minimax_m3 + speculative-config: '{"method":"dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_TS","kv_cache_dtype":"fp8","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + tensor-parallel-size: 1 + gpu-memory-utilization: 0.95 + max-num-seqs: 48 + max-num-batched-tokens: 8192 + watermark: 0.05 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + max-cudagraph-capture-size: 384 + numa-bind-nodes: + - 0 + - 0 + - 1 + - 1 + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"kv_lease_duration":120}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false,"save_decode_cache":true}}]}}' + data-parallel-size: 8 + enable-prefix-caching: true + data-parallel-rpc-port: 13345 + sbatch_directives: + cpus-per-task: '144' + mem: '0' + srun_options: + container-remap-root: '' + benchmark: + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: '1.0' + placement: + node: dedicated + name: minimaxm31flash-gb300-disagg-mooncake + setup_script: vllm-mooncake.sh +override_3p2d8: + name: minimaxm31flash-gb300-disagg-mooncake-3p2d8 +override_3p4d16: + name: minimaxm31flash-gb300-disagg-mooncake-3p4d16 + roles: + decode: + nodes: 4 + gpus: 16 + args: + data-parallel-size: 16 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 7b4482e29a..ba48d18e77 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -10761,3 +10761,106 @@ minimaxm31flash-fp4-gb200-dynamo-vllm-agentic-dspark-disagg: srt-recipe: disagg-mooncake.yaml:override_3p1d_c256 prefill: { num-worker: 3, tp: 4, ep: 4, dp-attn: true } decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } +minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg: + image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: + name: dynamo-router + version: 1.6.0.dev20261023 + multinode: true + disagg: false + srt-recipe-dir: minimaxm31flash/vllm/gb300-fp4/agentx + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: [1] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c1 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [4] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c4 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [8] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c8 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [16] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c16 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [32] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c32 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - dram-utilization: 1.0 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [40] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c40 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [48] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c48 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [56] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c56 + worker: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } + +minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg: + image: vllm/vllm-openai:minimax-m31-flash-arm64-cu134@sha256:5d76347628973fb80519a919a2f2740b6645b62cba3fe2bd87b25f111bd0e948 + model: MiniMaxAI/MiniMax-M3.1-Flash + model-prefix: minimaxm31flash + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: + name: dynamo-router + version: 1.6.0.dev20261023 + kv-p2p-transfer: nixl + multinode: true + disagg: true + srt-recipe-dir: minimaxm31flash/vllm/gb300-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 1.0 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } + conc-list: [280, 320, 360] + srt-recipe: disagg-mooncake.yaml:override_3p2d8 + prefill: { num-worker: 6, tp: 2, ep: 2, dp-attn: true } + decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: 0.3.13.post1 } + conc-list: [300] + srt-recipe: disagg-mooncake.yaml:override_3p4d16 + prefill: { num-worker: 6, tp: 2, ep: 2, dp-attn: true } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: true } + diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 476c6af6d4..18e8e72c5c 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11545,3 +11545,14 @@ - "Disagg: 3 DEP4 prefill workers with a DEP8 decode at conc 256 or a DEP16 decode at conc 280, and 4 DEP4 prefill workers with a DEP16 decode at conc 352; DeepGEMM MegaMoE, NIXL transfer plus a shared MooncakeStore DRAM pool (150 GB per rank), Dynamo KV router." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4464 scenario-type: [agentic-coding] + +- config-keys: + - minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg + - minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg + description: + - "Add MiniMax-M3.1-Flash (MiniMaxAI/MiniMax-M3.1-Flash, NVFP4 experts, NVFP4 KV and indexer caches) AgentX on GB300 with Dynamo and the arm64 vLLM image (minimax-m31-flash-arm64-cu134)." + - "DSpark with the built-in drafter, 7 draft tokens (golden AL 4.04) and an fp8 drafter KV cache on every point." + - "Agg: TP4 at conc 1-32, and TP4 with SimpleCPUOffloadConnector (lazy offload, 860 GB) at conc 40-56." + - "Disagg: 6 DEP2 prefill workers with a DEP8 decode at conc 280-360 or a DEP16 decode at conc 300; DeepGEMM MegaMoE, NIXL transfer plus a shared MooncakeStore DRAM pool (160 GB per rank), Dynamo KV router." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4468 + scenario-type: [agentic-coding]