diff --git a/collectivex/configs/platform_config.json b/collectivex/configs/platform_config.json index 535cd8fe2f..3a0cd96e84 100644 --- a/collectivex/configs/platform_config.json +++ b/collectivex/configs/platform_config.json @@ -133,17 +133,23 @@ "mi300x": { "arch": "gfx942", "product": "mi300x", - "image": "rocm/sgl-dev:sglang-0.5.14-rocm720-mi35x-mori-0701", + "image": "vllm/vllm-openai-rocm:v0.32.0", "image_platform": "linux/amd64", "gpus_per_node": 8, "scale_up_domain": 8, "scale_up_transport": "xgmi", "launcher": "mi-amds", - "backends": {"mori": [8], "uccl-ep": [8]}, - "ll_backends": {}, + "backends": {"mori": [8, 16], "uccl-ep": [8, 16]}, + "ll_backends": {"mori": [16], "uccl-ep": [16]}, "fabric": { - "nic": "n/a (single-node scale-up)", - "switch": "n/a (single-node scale-up)" + "nic": "Broadcom 400GbE RoCE v2 (bnxt_re), 8 rails", + "switch": "routed L3 RoCE fabric" + }, + "network": { + "socket_ifname": "ens50f1np1", + "rdma_devices": "bnxt_re_bond0,bnxt_re_bond1,bnxt_re_bond2,bnxt_re_bond3,bnxt_re_bond4,bnxt_re_bond5,bnxt_re_bond6,bnxt_re_bond7", + "ib_gid_index": 3, + "host_verbs_provider": "/usr/local/lib/x86_64-linux-gnu/libbnxt_re-rdmav34.so" }, "operator": { "partition": "MI300X-UBUNTU", @@ -153,18 +159,24 @@ "mi325x": { "arch": "gfx942", "product": "mi325x", - "image": "rocm/sgl-dev:sglang-0.5.14-rocm720-mi35x-mori-0701", + "image": "vllm/vllm-openai-rocm:v0.32.0", "image_platform": "linux/amd64", "gpus_per_node": 8, "scale_up_domain": 8, "scale_up_transport": "xgmi", "launcher": "mi-amds", "runner_label": "cluster:mi325x-amds", - "backends": {"mori": [8], "uccl-ep": [8]}, - "ll_backends": {}, + "backends": {"mori": [8, 16], "uccl-ep": [8, 16]}, + "ll_backends": {"mori": [16], "uccl-ep": [16]}, "fabric": { - "nic": "n/a (single-node scale-up)", - "switch": "n/a (single-node scale-up)" + "nic": "Broadcom 400GbE RoCE v2 (bnxt_re), 8 rails", + "switch": "routed L3 RoCE fabric" + }, + "network": { + "socket_ifname": "enp193s0f0np0", + "rdma_devices": "bnxt_re4,bnxt_re5,bnxt_re1,bnxt_re0,bnxt_re10,bnxt_re11,bnxt_re7,bnxt_re6", + "ib_gid_index": 3, + "host_verbs_provider": "/usr/local/lib/x86_64-linux-gnu/libbnxt_re-rdmav34.so" }, "operator": { "partition": "MI325X-UBUNTU", diff --git a/collectivex/launchers/launch_mi-amds.sh b/collectivex/launchers/launch_mi-amds.sh index c82900ccf9..86ad978b19 100644 --- a/collectivex/launchers/launch_mi-amds.sh +++ b/collectivex/launchers/launch_mi-amds.sh @@ -103,6 +103,7 @@ for allocation_attempt in 1 2 3; do done unset COLLX_SALLOC_ATTEMPT COLLX_NETWORK_VALIDATION_ATTEMPT CONTAINER_MOUNTS="$MOUNT_SRC:$MOUNT_DIR$DEVICE_MOUNTS" +collx_add_host_verbs_provider_mount # uccl-ep builds from source, so it gets the cross-allocation backend cache single-slurm uses. # Its parent is COLLX_STAGE_DIR, not SQUASH_DIR: this SKU's squash dir is node-local and # root-owned (/var/lib/squash), so the submit-side mkdir fails and the path is not what compute diff --git a/collectivex/launchers/launch_single-slurm.sh b/collectivex/launchers/launch_single-slurm.sh index 2da866f5d9..3188fc6aa0 100644 --- a/collectivex/launchers/launch_single-slurm.sh +++ b/collectivex/launchers/launch_single-slurm.sh @@ -67,6 +67,7 @@ collx_select_image "$IMAGE" collx_stage_with_backend_cache # Multi-node shards on the EFA pools (b300, h100-dgxc) carry the host EFA user space themselves. collx_add_efa_host_mounts +collx_add_host_verbs_provider_mount command -v salloc >/dev/null || collx_die "salloc not found on this runner" allocation=(--partition="$COLLX_PARTITION" --nodes="$NODES" --gres=gpu:"$GPN" @@ -81,7 +82,13 @@ for allocation_attempt in 1 2 3; do [ -z "$excluded_nodes" ] || attempt_allocation+=(--exclude="$excluded_nodes") export COLLX_SALLOC_ATTEMPT="$allocation_attempt" export COLLX_NETWORK_VALIDATION_ATTEMPT="$allocation_attempt" - collx_salloc_jobid "${attempt_allocation[@]}" + JOB_ID="" + if ! collx_salloc_jobid "${attempt_allocation[@]}"; then + # A job that dies before its grant (a failed node prolog) is worth another allocation. + [ "$allocation_attempt" != 3 ] || collx_die "scheduler allocation failed on every attempt" + collx_log "allocation attempt $allocation_attempt failed; retrying" + continue + fi [ -n "$JOB_ID" ] || collx_die "could not resolve allocated JOB_ID from salloc" if ! collx_validate_network_profile_on_job "$JOB_ID" "$NODES" "$COLLX_TRANSPORT"; then validation_failure=network diff --git a/collectivex/results/cases.json b/collectivex/results/cases.json index d3f2d0fbcb..b8efe5a46c 100644 --- a/collectivex/results/cases.json +++ b/collectivex/results/cases.json @@ -5544,28 +5544,636 @@ "workload": "deepseek-v4-pro" }, "detail": null, - "disposition": "unsupported", - "reason": "backend-platform-unsupported", + "disposition": "runnable", + "reason": null, + "sku": "mi300x" + }, + { + "case": { + "backend": "uccl-ep", + "case_id": "mi300x-uccl-ep-deepseek-v4-pro-normal-decode-ep16-uniform-fp8", + "ep": 16, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1 2 4 8 16 32 64 128 256 512", + "mode": "normal", + "nodes": 2, + "phase": "decode", + "precision": "fp8", + "routing": "uniform", + "scale_out_transport": "rdma", + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-out", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi300x-xgmi-rdma", + "transport": "xgmi-rdma", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi300x" + }, + { + "case": { + "backend": "uccl-ep", + "case_id": "mi300x-uccl-ep-deepseek-v4-pro-low-latency-decode-ep16-uniform-bf16", + "ep": 16, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1 2 4 8 16 32 64 128 256 512", + "mode": "low-latency", + "nodes": 2, + "phase": "decode", + "precision": "bf16", + "routing": "uniform", + "scale_out_transport": "rdma", + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-out", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi300x-xgmi-rdma", + "transport": "xgmi-rdma", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi300x" + }, + { + "case": { + "backend": "uccl-ep", + "case_id": "mi300x-uccl-ep-deepseek-v4-pro-low-latency-decode-ep16-uniform-fp8", + "ep": 16, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1 2 4 8 16 32 64 128 256 512", + "mode": "low-latency", + "nodes": 2, + "phase": "decode", + "precision": "fp8", + "routing": "uniform", + "scale_out_transport": "rdma", + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-out", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi300x-xgmi-rdma", + "transport": "xgmi-rdma", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi300x" + }, + { + "case": { + "backend": "mori", + "case_id": "mi300x-mori-deepseek-v4-pro-normal-decode-ep16-uniform-bf16", + "ep": 16, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1 2 4 8 16 32 64 128 256 512", + "mode": "normal", + "nodes": 2, + "phase": "decode", + "precision": "bf16", + "routing": "uniform", + "scale_out_transport": "rdma", + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-out", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi300x-xgmi-rdma", + "transport": "xgmi-rdma", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi300x" + }, + { + "case": { + "backend": "mori", + "case_id": "mi300x-mori-deepseek-v4-pro-normal-decode-ep16-uniform-fp8", + "ep": 16, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1 2 4 8 16 32 64 128 256 512", + "mode": "normal", + "nodes": 2, + "phase": "decode", + "precision": "fp8", + "routing": "uniform", + "scale_out_transport": "rdma", + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-out", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi300x-xgmi-rdma", + "transport": "xgmi-rdma", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi300x" + }, + { + "case": { + "backend": "mori", + "case_id": "mi300x-mori-deepseek-v4-pro-low-latency-decode-ep16-uniform-bf16", + "ep": 16, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1 2 4 8 16 32 64 128 256 512", + "mode": "low-latency", + "nodes": 2, + "phase": "decode", + "precision": "bf16", + "routing": "uniform", + "scale_out_transport": "rdma", + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-out", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi300x-xgmi-rdma", + "transport": "xgmi-rdma", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi300x" + }, + { + "case": { + "backend": "mori", + "case_id": "mi300x-mori-deepseek-v4-pro-low-latency-decode-ep16-uniform-fp8", + "ep": 16, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1 2 4 8 16 32 64 128 256 512", + "mode": "low-latency", + "nodes": 2, + "phase": "decode", + "precision": "fp8", + "routing": "uniform", + "scale_out_transport": "rdma", + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-out", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi300x-xgmi-rdma", + "transport": "xgmi-rdma", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi300x" + }, + { + "case": { + "backend": "uccl-ep", + "case_id": "mi300x-uccl-ep-deepseek-v4-pro-normal-prefill-ep16-uniform-bf16", + "ep": 16, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1024 2048 4096 8192", + "mode": "normal", + "nodes": 2, + "phase": "prefill", + "precision": "bf16", + "routing": "uniform", + "scale_out_transport": "rdma", + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-out", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi300x-xgmi-rdma", + "transport": "xgmi-rdma", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi300x" + }, + { + "case": { + "backend": "uccl-ep", + "case_id": "mi300x-uccl-ep-deepseek-v4-pro-normal-prefill-ep16-uniform-fp8", + "ep": 16, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1024 2048 4096 8192", + "mode": "normal", + "nodes": 2, + "phase": "prefill", + "precision": "fp8", + "routing": "uniform", + "scale_out_transport": "rdma", + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-out", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi300x-xgmi-rdma", + "transport": "xgmi-rdma", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi300x" + }, + { + "case": { + "backend": "mori", + "case_id": "mi300x-mori-deepseek-v4-pro-normal-prefill-ep16-uniform-bf16", + "ep": 16, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1024 2048 4096 8192", + "mode": "normal", + "nodes": 2, + "phase": "prefill", + "precision": "bf16", + "routing": "uniform", + "scale_out_transport": "rdma", + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-out", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi300x-xgmi-rdma", + "transport": "xgmi-rdma", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi300x" + }, + { + "case": { + "backend": "mori", + "case_id": "mi300x-mori-deepseek-v4-pro-normal-prefill-ep16-uniform-fp8", + "ep": 16, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1024 2048 4096 8192", + "mode": "normal", + "nodes": 2, + "phase": "prefill", + "precision": "fp8", + "routing": "uniform", + "scale_out_transport": "rdma", + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-out", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi300x-xgmi-rdma", + "transport": "xgmi-rdma", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, "sku": "mi300x" }, { "case": { - "backend": "mori", - "case_id": "mi300x-mori-deepseek-v4-pro-normal-decode-ep16-uniform-bf16", - "ep": 16, + "backend": "uccl-ep", + "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-decode-ep8-uniform-bf16", + "ep": 8, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1 2 4 8 16 32 64 128 256 512", + "mode": "normal", + "nodes": 1, + "phase": "decode", + "precision": "bf16", + "routing": "uniform", + "scale_out_transport": null, + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-up", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi325x-xgmi", + "transport": "xgmi", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi325x" + }, + { + "case": { + "backend": "uccl-ep", + "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-decode-ep8-uniform-fp8", + "ep": 8, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1 2 4 8 16 32 64 128 256 512", + "mode": "normal", + "nodes": 1, + "phase": "decode", + "precision": "fp8", + "routing": "uniform", + "scale_out_transport": null, + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-up", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi325x-xgmi", + "transport": "xgmi", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi325x" + }, + { + "case": { + "backend": "mori", + "case_id": "mi325x-mori-deepseek-v4-pro-normal-decode-ep8-uniform-bf16", + "ep": 8, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1 2 4 8 16 32 64 128 256 512", + "mode": "normal", + "nodes": 1, + "phase": "decode", + "precision": "bf16", + "routing": "uniform", + "scale_out_transport": null, + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-up", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi325x-xgmi", + "transport": "xgmi", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi325x" + }, + { + "case": { + "backend": "mori", + "case_id": "mi325x-mori-deepseek-v4-pro-normal-decode-ep8-uniform-fp8", + "ep": 8, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1 2 4 8 16 32 64 128 256 512", + "mode": "normal", + "nodes": 1, + "phase": "decode", + "precision": "fp8", + "routing": "uniform", + "scale_out_transport": null, + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-up", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi325x-xgmi", + "transport": "xgmi", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi325x" + }, + { + "case": { + "backend": "uccl-ep", + "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-prefill-ep8-uniform-bf16", + "ep": 8, + "experts": 384, + "gpus_per_node": 8, + "hidden": 7168, + "ladder": "1024 2048 4096 8192", + "mode": "normal", + "nodes": 1, + "phase": "prefill", + "precision": "bf16", + "routing": "uniform", + "scale_out_transport": null, + "scale_up_domain": 8, + "scale_up_transport": "xgmi", + "scope": "scale-up", + "seed": 67, + "suite": "ep-core", + "timing": { + "chain_drop": 16, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "iters_per_trial": 8, + "trials_per_point": 256, + "warmup_iters_per_trial": 32 + }, + "topk": 6, + "topology_class": "mi325x-xgmi", + "transport": "xgmi", + "workload": "deepseek-v4-pro" + }, + "detail": null, + "disposition": "runnable", + "reason": null, + "sku": "mi325x" + }, + { + "case": { + "backend": "uccl-ep", + "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-prefill-ep8-uniform-fp8", + "ep": 8, "experts": 384, "gpus_per_node": 8, "hidden": 7168, - "ladder": "1 2 4 8 16 32 64 128 256 512", + "ladder": "1024 2048 4096 8192", "mode": "normal", - "nodes": 2, - "phase": "decode", - "precision": "bf16", + "nodes": 1, + "phase": "prefill", + "precision": "fp8", "routing": "uniform", - "scale_out_transport": "rdma", + "scale_out_transport": null, "scale_up_domain": 8, "scale_up_transport": "xgmi", - "scope": "scale-out", + "scope": "scale-up", "seed": 67, "suite": "ep-core", "timing": { @@ -5577,33 +6185,33 @@ "warmup_iters_per_trial": 32 }, "topk": 6, - "topology_class": "mi300x-xgmi-rdma", - "transport": "xgmi-rdma", + "topology_class": "mi325x-xgmi", + "transport": "xgmi", "workload": "deepseek-v4-pro" }, "detail": null, - "disposition": "unsupported", - "reason": "backend-platform-unsupported", - "sku": "mi300x" + "disposition": "runnable", + "reason": null, + "sku": "mi325x" }, { "case": { - "backend": "uccl-ep", - "case_id": "mi300x-uccl-ep-deepseek-v4-pro-normal-prefill-ep16-uniform-bf16", - "ep": 16, + "backend": "mori", + "case_id": "mi325x-mori-deepseek-v4-pro-normal-prefill-ep8-uniform-bf16", + "ep": 8, "experts": 384, "gpus_per_node": 8, "hidden": 7168, "ladder": "1024 2048 4096 8192", "mode": "normal", - "nodes": 2, + "nodes": 1, "phase": "prefill", "precision": "bf16", "routing": "uniform", - "scale_out_transport": "rdma", + "scale_out_transport": null, "scale_up_domain": 8, "scale_up_transport": "xgmi", - "scope": "scale-out", + "scope": "scale-up", "seed": 67, "suite": "ep-core", "timing": { @@ -5615,33 +6223,33 @@ "warmup_iters_per_trial": 32 }, "topk": 6, - "topology_class": "mi300x-xgmi-rdma", - "transport": "xgmi-rdma", + "topology_class": "mi325x-xgmi", + "transport": "xgmi", "workload": "deepseek-v4-pro" }, "detail": null, - "disposition": "unsupported", - "reason": "backend-platform-unsupported", - "sku": "mi300x" + "disposition": "runnable", + "reason": null, + "sku": "mi325x" }, { "case": { "backend": "mori", - "case_id": "mi300x-mori-deepseek-v4-pro-normal-prefill-ep16-uniform-bf16", - "ep": 16, + "case_id": "mi325x-mori-deepseek-v4-pro-normal-prefill-ep8-uniform-fp8", + "ep": 8, "experts": 384, "gpus_per_node": 8, "hidden": 7168, "ladder": "1024 2048 4096 8192", "mode": "normal", - "nodes": 2, + "nodes": 1, "phase": "prefill", - "precision": "bf16", + "precision": "fp8", "routing": "uniform", - "scale_out_transport": "rdma", + "scale_out_transport": null, "scale_up_domain": 8, "scale_up_transport": "xgmi", - "scope": "scale-out", + "scope": "scale-up", "seed": 67, "suite": "ep-core", "timing": { @@ -5653,33 +6261,33 @@ "warmup_iters_per_trial": 32 }, "topk": 6, - "topology_class": "mi300x-xgmi-rdma", - "transport": "xgmi-rdma", + "topology_class": "mi325x-xgmi", + "transport": "xgmi", "workload": "deepseek-v4-pro" }, "detail": null, - "disposition": "unsupported", - "reason": "backend-platform-unsupported", - "sku": "mi300x" + "disposition": "runnable", + "reason": null, + "sku": "mi325x" }, { "case": { "backend": "uccl-ep", - "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-decode-ep8-uniform-bf16", - "ep": 8, + "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-decode-ep16-uniform-bf16", + "ep": 16, "experts": 384, "gpus_per_node": 8, "hidden": 7168, "ladder": "1 2 4 8 16 32 64 128 256 512", "mode": "normal", - "nodes": 1, + "nodes": 2, "phase": "decode", "precision": "bf16", "routing": "uniform", - "scale_out_transport": null, + "scale_out_transport": "rdma", "scale_up_domain": 8, "scale_up_transport": "xgmi", - "scope": "scale-up", + "scope": "scale-out", "seed": 67, "suite": "ep-core", "timing": { @@ -5691,8 +6299,8 @@ "warmup_iters_per_trial": 32 }, "topk": 6, - "topology_class": "mi325x-xgmi", - "transport": "xgmi", + "topology_class": "mi325x-xgmi-rdma", + "transport": "xgmi-rdma", "workload": "deepseek-v4-pro" }, "detail": null, @@ -5703,21 +6311,21 @@ { "case": { "backend": "uccl-ep", - "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-decode-ep8-uniform-fp8", - "ep": 8, + "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-decode-ep16-uniform-fp8", + "ep": 16, "experts": 384, "gpus_per_node": 8, "hidden": 7168, "ladder": "1 2 4 8 16 32 64 128 256 512", "mode": "normal", - "nodes": 1, + "nodes": 2, "phase": "decode", "precision": "fp8", "routing": "uniform", - "scale_out_transport": null, + "scale_out_transport": "rdma", "scale_up_domain": 8, "scale_up_transport": "xgmi", - "scope": "scale-up", + "scope": "scale-out", "seed": 67, "suite": "ep-core", "timing": { @@ -5729,8 +6337,8 @@ "warmup_iters_per_trial": 32 }, "topk": 6, - "topology_class": "mi325x-xgmi", - "transport": "xgmi", + "topology_class": "mi325x-xgmi-rdma", + "transport": "xgmi-rdma", "workload": "deepseek-v4-pro" }, "detail": null, @@ -5740,22 +6348,22 @@ }, { "case": { - "backend": "mori", - "case_id": "mi325x-mori-deepseek-v4-pro-normal-decode-ep8-uniform-bf16", - "ep": 8, + "backend": "uccl-ep", + "case_id": "mi325x-uccl-ep-deepseek-v4-pro-low-latency-decode-ep16-uniform-bf16", + "ep": 16, "experts": 384, "gpus_per_node": 8, "hidden": 7168, "ladder": "1 2 4 8 16 32 64 128 256 512", - "mode": "normal", - "nodes": 1, + "mode": "low-latency", + "nodes": 2, "phase": "decode", "precision": "bf16", "routing": "uniform", - "scale_out_transport": null, + "scale_out_transport": "rdma", "scale_up_domain": 8, "scale_up_transport": "xgmi", - "scope": "scale-up", + "scope": "scale-out", "seed": 67, "suite": "ep-core", "timing": { @@ -5767,8 +6375,8 @@ "warmup_iters_per_trial": 32 }, "topk": 6, - "topology_class": "mi325x-xgmi", - "transport": "xgmi", + "topology_class": "mi325x-xgmi-rdma", + "transport": "xgmi-rdma", "workload": "deepseek-v4-pro" }, "detail": null, @@ -5778,22 +6386,22 @@ }, { "case": { - "backend": "mori", - "case_id": "mi325x-mori-deepseek-v4-pro-normal-decode-ep8-uniform-fp8", - "ep": 8, + "backend": "uccl-ep", + "case_id": "mi325x-uccl-ep-deepseek-v4-pro-low-latency-decode-ep16-uniform-fp8", + "ep": 16, "experts": 384, "gpus_per_node": 8, "hidden": 7168, "ladder": "1 2 4 8 16 32 64 128 256 512", - "mode": "normal", - "nodes": 1, + "mode": "low-latency", + "nodes": 2, "phase": "decode", "precision": "fp8", "routing": "uniform", - "scale_out_transport": null, + "scale_out_transport": "rdma", "scale_up_domain": 8, "scale_up_transport": "xgmi", - "scope": "scale-up", + "scope": "scale-out", "seed": 67, "suite": "ep-core", "timing": { @@ -5805,8 +6413,8 @@ "warmup_iters_per_trial": 32 }, "topk": 6, - "topology_class": "mi325x-xgmi", - "transport": "xgmi", + "topology_class": "mi325x-xgmi-rdma", + "transport": "xgmi-rdma", "workload": "deepseek-v4-pro" }, "detail": null, @@ -5816,22 +6424,22 @@ }, { "case": { - "backend": "uccl-ep", - "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-prefill-ep8-uniform-bf16", - "ep": 8, + "backend": "mori", + "case_id": "mi325x-mori-deepseek-v4-pro-normal-decode-ep16-uniform-bf16", + "ep": 16, "experts": 384, "gpus_per_node": 8, "hidden": 7168, - "ladder": "1024 2048 4096 8192", + "ladder": "1 2 4 8 16 32 64 128 256 512", "mode": "normal", - "nodes": 1, - "phase": "prefill", + "nodes": 2, + "phase": "decode", "precision": "bf16", "routing": "uniform", - "scale_out_transport": null, + "scale_out_transport": "rdma", "scale_up_domain": 8, "scale_up_transport": "xgmi", - "scope": "scale-up", + "scope": "scale-out", "seed": 67, "suite": "ep-core", "timing": { @@ -5843,8 +6451,8 @@ "warmup_iters_per_trial": 32 }, "topk": 6, - "topology_class": "mi325x-xgmi", - "transport": "xgmi", + "topology_class": "mi325x-xgmi-rdma", + "transport": "xgmi-rdma", "workload": "deepseek-v4-pro" }, "detail": null, @@ -5854,22 +6462,22 @@ }, { "case": { - "backend": "uccl-ep", - "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-prefill-ep8-uniform-fp8", - "ep": 8, + "backend": "mori", + "case_id": "mi325x-mori-deepseek-v4-pro-normal-decode-ep16-uniform-fp8", + "ep": 16, "experts": 384, "gpus_per_node": 8, "hidden": 7168, - "ladder": "1024 2048 4096 8192", + "ladder": "1 2 4 8 16 32 64 128 256 512", "mode": "normal", - "nodes": 1, - "phase": "prefill", + "nodes": 2, + "phase": "decode", "precision": "fp8", "routing": "uniform", - "scale_out_transport": null, + "scale_out_transport": "rdma", "scale_up_domain": 8, "scale_up_transport": "xgmi", - "scope": "scale-up", + "scope": "scale-out", "seed": 67, "suite": "ep-core", "timing": { @@ -5881,8 +6489,8 @@ "warmup_iters_per_trial": 32 }, "topk": 6, - "topology_class": "mi325x-xgmi", - "transport": "xgmi", + "topology_class": "mi325x-xgmi-rdma", + "transport": "xgmi-rdma", "workload": "deepseek-v4-pro" }, "detail": null, @@ -5893,21 +6501,21 @@ { "case": { "backend": "mori", - "case_id": "mi325x-mori-deepseek-v4-pro-normal-prefill-ep8-uniform-bf16", - "ep": 8, + "case_id": "mi325x-mori-deepseek-v4-pro-low-latency-decode-ep16-uniform-bf16", + "ep": 16, "experts": 384, "gpus_per_node": 8, "hidden": 7168, - "ladder": "1024 2048 4096 8192", - "mode": "normal", - "nodes": 1, - "phase": "prefill", + "ladder": "1 2 4 8 16 32 64 128 256 512", + "mode": "low-latency", + "nodes": 2, + "phase": "decode", "precision": "bf16", "routing": "uniform", - "scale_out_transport": null, + "scale_out_transport": "rdma", "scale_up_domain": 8, "scale_up_transport": "xgmi", - "scope": "scale-up", + "scope": "scale-out", "seed": 67, "suite": "ep-core", "timing": { @@ -5919,8 +6527,8 @@ "warmup_iters_per_trial": 32 }, "topk": 6, - "topology_class": "mi325x-xgmi", - "transport": "xgmi", + "topology_class": "mi325x-xgmi-rdma", + "transport": "xgmi-rdma", "workload": "deepseek-v4-pro" }, "detail": null, @@ -5931,21 +6539,21 @@ { "case": { "backend": "mori", - "case_id": "mi325x-mori-deepseek-v4-pro-normal-prefill-ep8-uniform-fp8", - "ep": 8, + "case_id": "mi325x-mori-deepseek-v4-pro-low-latency-decode-ep16-uniform-fp8", + "ep": 16, "experts": 384, "gpus_per_node": 8, "hidden": 7168, - "ladder": "1024 2048 4096 8192", - "mode": "normal", - "nodes": 1, - "phase": "prefill", + "ladder": "1 2 4 8 16 32 64 128 256 512", + "mode": "low-latency", + "nodes": 2, + "phase": "decode", "precision": "fp8", "routing": "uniform", - "scale_out_transport": null, + "scale_out_transport": "rdma", "scale_up_domain": 8, "scale_up_transport": "xgmi", - "scope": "scale-up", + "scope": "scale-out", "seed": 67, "suite": "ep-core", "timing": { @@ -5957,8 +6565,8 @@ "warmup_iters_per_trial": 32 }, "topk": 6, - "topology_class": "mi325x-xgmi", - "transport": "xgmi", + "topology_class": "mi325x-xgmi-rdma", + "transport": "xgmi-rdma", "workload": "deepseek-v4-pro" }, "detail": null, @@ -5969,15 +6577,15 @@ { "case": { "backend": "uccl-ep", - "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-decode-ep16-uniform-bf16", + "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-prefill-ep16-uniform-bf16", "ep": 16, "experts": 384, "gpus_per_node": 8, "hidden": 7168, - "ladder": "1 2 4 8 16 32 64 128 256 512", + "ladder": "1024 2048 4096 8192", "mode": "normal", "nodes": 2, - "phase": "decode", + "phase": "prefill", "precision": "bf16", "routing": "uniform", "scale_out_transport": "rdma", @@ -6000,23 +6608,23 @@ "workload": "deepseek-v4-pro" }, "detail": null, - "disposition": "unsupported", - "reason": "backend-platform-unsupported", + "disposition": "runnable", + "reason": null, "sku": "mi325x" }, { "case": { - "backend": "mori", - "case_id": "mi325x-mori-deepseek-v4-pro-normal-decode-ep16-uniform-bf16", + "backend": "uccl-ep", + "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-prefill-ep16-uniform-fp8", "ep": 16, "experts": 384, "gpus_per_node": 8, "hidden": 7168, - "ladder": "1 2 4 8 16 32 64 128 256 512", + "ladder": "1024 2048 4096 8192", "mode": "normal", "nodes": 2, - "phase": "decode", - "precision": "bf16", + "phase": "prefill", + "precision": "fp8", "routing": "uniform", "scale_out_transport": "rdma", "scale_up_domain": 8, @@ -6038,14 +6646,14 @@ "workload": "deepseek-v4-pro" }, "detail": null, - "disposition": "unsupported", - "reason": "backend-platform-unsupported", + "disposition": "runnable", + "reason": null, "sku": "mi325x" }, { "case": { - "backend": "uccl-ep", - "case_id": "mi325x-uccl-ep-deepseek-v4-pro-normal-prefill-ep16-uniform-bf16", + "backend": "mori", + "case_id": "mi325x-mori-deepseek-v4-pro-normal-prefill-ep16-uniform-bf16", "ep": 16, "experts": 384, "gpus_per_node": 8, @@ -6076,14 +6684,14 @@ "workload": "deepseek-v4-pro" }, "detail": null, - "disposition": "unsupported", - "reason": "backend-platform-unsupported", + "disposition": "runnable", + "reason": null, "sku": "mi325x" }, { "case": { "backend": "mori", - "case_id": "mi325x-mori-deepseek-v4-pro-normal-prefill-ep16-uniform-bf16", + "case_id": "mi325x-mori-deepseek-v4-pro-normal-prefill-ep16-uniform-fp8", "ep": 16, "experts": 384, "gpus_per_node": 8, @@ -6092,7 +6700,7 @@ "mode": "normal", "nodes": 2, "phase": "prefill", - "precision": "bf16", + "precision": "fp8", "routing": "uniform", "scale_out_transport": "rdma", "scale_up_domain": 8, @@ -6114,8 +6722,8 @@ "workload": "deepseek-v4-pro" }, "detail": null, - "disposition": "unsupported", - "reason": "backend-platform-unsupported", + "disposition": "runnable", + "reason": null, "sku": "mi325x" }, { diff --git a/collectivex/run.sh b/collectivex/run.sh index 5a4f61ba58..498f8506e7 100755 --- a/collectivex/run.sh +++ b/collectivex/run.sh @@ -52,9 +52,12 @@ done # Results record the commit they were measured at; uncommitted edits to the benchmark (anything # under collectivex/ except results/) would make that a lie, so they block the run by default. +checkout_state() { + git -C "$REPO_ROOT" status --porcelain --untracked-files=no -- \ + collectivex ':(exclude)collectivex/results' +} source_sha="$(git -C "$REPO_ROOT" rev-parse HEAD)" -dirty="$(git -C "$REPO_ROOT" status --porcelain --untracked-files=no -- \ - collectivex ':(exclude)collectivex/results')" +dirty="$(checkout_state)" source_dirty=0 if [ -n "$dirty" ]; then source_dirty=1 @@ -65,8 +68,13 @@ if [ -n "$dirty" ]; then fi fi -mkdir -p "$HERE/.shards" -matrix="$HERE/.shards/matrix-$SKU.json" +# Each invocation stages its matrix and shard controls in its own directory: two runs from one +# checkout (one per backend, say) would otherwise overwrite each other's files and run the wrong +# shards. +shard_dir="$HERE/.shards/run-$$" +mkdir -p "$shard_dir" +trap 'rm -rf "$shard_dir"' EXIT +matrix="$shard_dir/matrix-$SKU.json" python3 "$HERE/sweep_matrix.py" "${matrix_args[@]}" --out "$matrix" >/dev/null # One tab-separated line per selected shard: @@ -104,7 +112,15 @@ stamp="$(date -u +%Y%m%dT%H%M%SZ)" rcs=() for row in "${rows[@]}"; do IFS=$'\t' read -r id launcher backend nodes gpn domain ncases image staged minutes <<< "$row" - control="$HERE/.shards/$id.json" + # Every shard stages the checkout as it is now, but its results carry the source recorded at + # start: stop rather than stamp a later shard with a commit it did not run. + if [ "$dry" = 0 ] && { [ "$(git -C "$REPO_ROOT" rev-parse HEAD)" != "$source_sha" ] || + [ "$(checkout_state)" != "$dirty" ]; }; then + echo "collectivex/ changed since this run started at $source_sha; stopping before $id" >&2 + rcs+=(drift) + break + fi + control="$shard_dir/$id.json" python3 "$HERE/sweep_matrix.py" --extract-from "$matrix" --shard-id "$id" --out "$control" \ >/dev/null 2>&1 # The pid keeps two runs of one shard started in the same second (graphed and eager, say) out @@ -143,8 +159,8 @@ done echo "=== summary for $SKU (source $source_sha, dirty=$source_dirty)" failed=0 for i in "${!rows[@]}"; do - printf '%-34s rc=%s\n' "${rows[$i]%%$'\t'*}" "${rcs[$i]}" - [ "${rcs[$i]}" = 0 ] || failed=$((failed + 1)) + printf '%-34s rc=%s\n' "${rows[$i]%%$'\t'*}" "${rcs[$i]:-not run}" + [ "${rcs[$i]:-}" = 0 ] || failed=$((failed + 1)) done python3 "$HERE/summarize.py" --results-dir "$HERE/results/$SKU" || true git -C "$REPO_ROOT" status --short -- "collectivex/results/$SKU" diff --git a/collectivex/runtime/common.sh b/collectivex/runtime/common.sh index c39c99e17f..2385081f18 100644 --- a/collectivex/runtime/common.sh +++ b/collectivex/runtime/common.sh @@ -125,7 +125,7 @@ collx_load_operator_config() { unset COLLX_EXCLUDE_NODES COLLX_NODELIST COLLX_LOCK_DIR COLLX_MASTER_PORT unset COLLX_SOCKET_IFNAME COLLX_RDMA_DEVICES COLLX_IB_GID_INDEX COLLX_RDMA_SERVICE_LEVEL unset COLLX_RDMA_TRAFFIC_CLASS COLLX_RAIL_ISOLATED COLLX_SINGLE_NODE_RDMA_DEVICES COLLX_RDMA_FABRIC - unset COLLX_RDMA_RELAXED_ORDERING COLLX_RDMA_GBS + unset COLLX_RDMA_RELAXED_ORDERING COLLX_RDMA_GBS COLLX_HOST_VERBS_PROVIDER unset MASTER_ADDR MASTER_PORT RANK WORLD_SIZE LOCAL_RANK LOCAL_WORLD_SIZE config_path="${COLLECTIVEX_OPERATOR_CONFIG:-${XDG_CONFIG_HOME:-${HOME}/.config}/inferencex/collectivex.json}" if [ ! -e "$config_path" ]; then @@ -243,6 +243,22 @@ collx_add_efa_host_mounts() { CONTAINER_MOUNTS="$CONTAINER_MOUNTS,$mounts" } +# Where collx_add_host_verbs_provider_mount binds a host verbs provider inside the container. +COLLX_HOST_VERBS_ROOT=/opt/collectivex/host-verbs + +# Some NIC vendors replace the host's rdma-core provider with their own build for an out-of-tree +# kernel driver (the MI300X/MI325X pools' Broadcom bnxt_re 237.x speaks kernel ABI 8; rdma-core's stock bnxt_re +# provider, the image's, supports only ABI 1, so libibverbs lists no device). Multi-node shards +# bind the registry-named host provider read-only; backend preparation installs it into the +# container over the image's file (verbs_provider_prepare in prepare_backend.sh). +collx_add_host_verbs_provider_mount() { + local provider="${COLLX_HOST_VERBS_PROVIDER:-}" + [ -n "$provider" ] && [ "${NODES:-1}" -gt 1 ] && [ "${COLLX_TRANSPORT:-}" != mnnvl ] || return 0 + [[ "$provider" =~ ^/[A-Za-z0-9_./-]+/lib[a-z0-9_]+-rdmav[0-9]+\.so$ ]] && [[ "$provider" != *..* ]] \ + || collx_die "invalid host verbs provider path" + CONTAINER_MOUNTS="$CONTAINER_MOUNTS,$provider:$COLLX_HOST_VERBS_ROOT/${provider##*/}:ro" +} + # AWS EFA scale-out (b300: p6-b300.48xlarge, 16 EFA devices per node; h100-dgxc: p5.48xlarge, 32). # EFA is not a verbs HCA: its ports report link_layer Unspecified, it has no GID table, service # level or traffic class, and NCCL reaches it only through the aws-ofi-nccl libfabric plugin from @@ -294,7 +310,7 @@ collx_apply_network_profile() { unset NVSHMEM_HCA_LIST NVSHMEM_IB_GID_INDEX NVSHMEM_IB_SL unset NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_IBGDA_NIC_HANDLER unset EP_NIC_NAME EP_OVERRIDE_RDMA_SL - unset MORI_RDMA_DEVICES + unset MORI_RDMA_DEVICES MORI_SOCKET_IFNAME unset MORI_RDMA_TC MORI_IO_TC MORI_RDMA_SL MORI_IO_SL unset UCCL_SOCKET_IFNAME UCCL_IB_HCA UCCL_IB_GID_INDEX UCCL_IB_SL UCCL_IB_TC unset UCCL_IB_MAX_INFLIGHT_BYTES UCCL_IB_MAX_INFLIGHT_NORMAL UCCL_EP_ENABLE_AGGRESSIVE_ATOMIC @@ -317,7 +333,10 @@ collx_apply_network_profile() { if [ -n "${COLLX_SOCKET_IFNAME:-}" ]; then [[ "$COLLX_SOCKET_IFNAME" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}$ ]] \ || collx_die "invalid private socket interface selector" - export NCCL_SOCKET_IFNAME="$COLLX_SOCKET_IFNAME" GLOO_SOCKET_IFNAME="$COLLX_SOCKET_IFNAME" + # MoRI's shmem bootstrap otherwise picks its own interface, which can be a rail without a + # kernel route to the root rank (per-rail /31 links on the Broadcom pools). + export NCCL_SOCKET_IFNAME="$COLLX_SOCKET_IFNAME" GLOO_SOCKET_IFNAME="$COLLX_SOCKET_IFNAME" \ + MORI_SOCKET_IFNAME="$COLLX_SOCKET_IFNAME" fi [[ "$COLLX_RDMA_DEVICES" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(:[1-9][0-9]*)?(,[A-Za-z][A-Za-z0-9_.-]{0,31}(:[1-9][0-9]*)?)*$ ]] \ || collx_die "invalid private RDMA device selector" @@ -553,6 +572,17 @@ collx_slurm_job_name() { } # The job name is prepended so a caller-supplied --job-name wins (salloc takes the last). +# True only when Slurm says the job is gone: listed as nothing, or purged ("Invalid job id"). A +# failed query (slurmctld busy or restarting) is not an answer, so it reads as still there. +collx_job_gone() { + local out + if out="$(squeue -h -j "$1" -o %A 2>&1)"; then + [ -z "$out" ] + else + [[ "$out" == *"Invalid job id"* ]] + fi +} + collx_salloc_jobid() { local log_label log job_id set -- --job-name="$(collx_slurm_job_name)" "$@" @@ -562,7 +592,25 @@ collx_salloc_jobid() { return 1 fi collx_log "scheduler-request=submit" - if ! (salloc "$@" --no-shell) > "$log" 2>&1; then + # salloc --no-shell does not notice a job that ends between the grant and its own poll (a node + # prolog that fails the job ten seconds in leaves it waiting forever), so the job is watched + # here: once Slurm no longer lists the queued job and no grant was logged, salloc is stopped. + (salloc "$@" --no-shell) > "$log" 2>&1 & + local salloc_pid=$! queued="" + while kill -0 "$salloc_pid" 2>/dev/null; do + sleep 10 + grep -q "Granted job allocation" "$log" && break + [ -n "$queued" ] || queued="$(sed -nE 's/.*Pending job allocation ([1-9][0-9]*).*/\1/p' "$log" | head -n1)" + if [ -n "$queued" ] && collx_job_gone "$queued" \ + && ! grep -q "Granted job allocation" "$log"; then + collx_log "ERROR: scheduled job $queued ended before its allocation was granted" + kill -TERM "$salloc_pid" 2>/dev/null + wait "$salloc_pid" 2>/dev/null + collx_log_tail "$log" + return 1 + fi + done + if ! wait "$salloc_pid"; then collx_log "ERROR: scheduler allocation failed" collx_log_tail "$log" return 1 diff --git a/collectivex/runtime/config.py b/collectivex/runtime/config.py index cdccfc88a3..5aecacbfd7 100644 --- a/collectivex/runtime/config.py +++ b/collectivex/runtime/config.py @@ -16,7 +16,7 @@ NETWORK_FIELDS = { "socket_ifname", "rdma_devices", "ib_gid_index", "rdma_service_level", "rdma_traffic_class", "rail_isolated", "single_node_rdma_devices", "rdma_fabric", - "rdma_relaxed_ordering", "rdma_gbs", + "rdma_relaxed_ordering", "rdma_gbs", "host_verbs_provider", } # Timing knobs, in the order the legacy colon-string encoded them, paired with the run_ep flag # each one drives. The names match configs/sweep.json so a case's timing block is readable diff --git a/collectivex/runtime/elf_vernaux.py b/collectivex/runtime/elf_vernaux.py new file mode 100644 index 0000000000..08b3368013 --- /dev/null +++ b/collectivex/runtime/elf_vernaux.py @@ -0,0 +1,76 @@ +"""Retarget one version requirement of a shared library to another version of the same library. + +Broadcom's host libbnxt_re is linked on Ubuntu 24.04 and needs GLIBC_2.38 for one symbol, +__isoc23_strtol; the ROCm images ship glibc 2.35. After that symbol is unversioned and supplied by a +shim, the GLIBC_2.38 entry in .gnu.version_r still makes the loader refuse the library. This rewrites +that entry's name and hash to a version libc does provide, in place; nothing else in the file moves. + +usage: elf_vernaux.py LIBRARY FROM_VERSION TO_VERSION +""" +import struct +import sys + +SHT_GNU_VERNEED = 0x6FFFFFFE + + +def elf_hash(name: bytes) -> int: + h = 0 + for byte in name: + h = ((h << 4) + byte) & 0xFFFFFFFF + g = h & 0xF0000000 + if g: + h ^= g >> 24 + h &= ~g & 0xFFFFFFFF + return h + + +def retarget(path: str, old: str, new: str) -> None: + with open(path, "r+b") as stream: + data = bytearray(stream.read()) + if data[:4] != b"\x7fELF" or data[4] != 2 or data[5] != 1: + raise SystemExit("not a little-endian ELF64 file") + shoff, = struct.unpack_from(" bytes: + start = strtab + index + return bytes(data[start:data.index(0, start)]) + + targets, replacement = [], None + entry = offset + for _ in range(info): + _, _, _, aux, following = struct.unpack_from("/dev/null; then + collx_log "verbs provider: $name needs a newer glibc; shimming __isoc23_strtol" + printf '%s\n' '#include ' \ + 'long __isoc23_strtol(const char *s, char **end, int base) { return strtol(s, end, base); }' \ + > "$work/cx_c23.c" + cc -shared -fPIC -O2 -o "$work/libcx_c23.so" "$work/cx_c23.c" \ + && python3 -m pip install -q --disable-pip-version-check --no-input \ + --target "$work/patchelf" "$VERBS_PATCHELF_SPEC" >&2 \ + && "$work/patchelf/bin/patchelf" --clear-symbol-version __isoc23_strtol \ + --add-needed libcx_c23.so --set-rpath '$ORIGIN' "$work/$name" \ + && python3 "$COLLX_RUNTIME_DIR/elf_vernaux.py" "$work/$name" GLIBC_2.38 GLIBC_2.2.5 \ + || { collx_log "ERROR: verbs provider patch failed"; return 1; } + python3 -c 'import ctypes, sys; ctypes.CDLL(sys.argv[1])' "$work/$name" \ + || { collx_log "ERROR: patched verbs provider does not load"; return 1; } + cp "$work/libcx_c23.so" "$providers/libcx_c23.so" || return 1 + fi + cp "$work/$name" "$target.collectivex" && mv -f "$target.collectivex" "$target" \ + || { collx_log "ERROR: verbs provider install failed"; return 1; } + rm -rf "$work" + verbs_lists_registry_devices \ + || { collx_log "ERROR: verbs provider $name does not list every registry device"; return 1; } + collx_log "verbs provider: installed host $name" +} + main() { collx_apply_network_profile "${COLLX_NODES:-1}" "${COLLX_TRANSPORT:-}" || return 1 + verbs_provider_prepare || return 1 validate_container_network || return 1 ofi_nccl_prepare || return 1 case "$COLLX_BENCH" in diff --git a/collectivex/tests/test_elf_vernaux.py b/collectivex/tests/test_elf_vernaux.py new file mode 100644 index 0000000000..62be70c746 --- /dev/null +++ b/collectivex/tests/test_elf_vernaux.py @@ -0,0 +1,68 @@ +"""elf_vernaux.py retargets one version requirement in place and refuses anything else.""" +import struct +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "runtime")) +import elf_vernaux # noqa: E402 + + +def _library(versions) -> bytes: + """A minimal ELF64 with a string table and one .gnu.version_r entry (libc.so.6) whose aux + records require `versions`, in order.""" + strings = b"\0libc.so.6\0" + b"".join(v.encode() + b"\0" for v in versions) + offset_of = lambda text: strings.index(text.encode() + b"\0") + verneed = struct.pack(" None: + before, after = self._run(["GLIBC_2.2.5", "GLIBC_2.38"], "GLIBC_2.38", "GLIBC_2.2.5") + self.assertEqual(len(before), len(after)) + strtab = 64 + aux = before.index(struct.pack("= aux and max(changed) < aux + 12) + + def test_a_missing_requirement_or_replacement_fails_closed(self) -> None: + with self.assertRaises(SystemExit): + self._run(["GLIBC_2.2.5"], "GLIBC_2.38", "GLIBC_2.2.5") + with self.assertRaises(SystemExit): + self._run(["GLIBC_2.38"], "GLIBC_2.38", "GLIBC_2.2.5") + + def test_the_hash_is_the_sysv_elf_hash(self) -> None: + # Values glibc's own verneed entries carry. + self.assertEqual(elf_vernaux.elf_hash(b"GLIBC_2.2.5"), 0x09691A75) + self.assertEqual(elf_vernaux.elf_hash(b"GLIBC_2.38"), 0x069691B8) + + +if __name__ == "__main__": + unittest.main() diff --git a/collectivex/tests/test_runtime.py b/collectivex/tests/test_runtime.py index 310d5b4e27..ae6abf2312 100644 --- a/collectivex/tests/test_runtime.py +++ b/collectivex/tests/test_runtime.py @@ -228,6 +228,47 @@ def test_nvshmem_hca_list_per_placement(self) -> None: self.assertEqual(stdout.strip().splitlines()[-1], expected) +class HostVerbsProviderMountTests(unittest.TestCase): + PROVIDER = "/usr/local/lib/x86_64-linux-gnu/libbnxt_re-rdmav34.so" + + @staticmethod + def _mounts(provider: str, nodes: int): + return run_common( + f"COLLX_HOST_VERBS_PROVIDER='{provider}' NODES={nodes} COLLX_TRANSPORT=xgmi-rdma" + " CONTAINER_MOUNTS=/stage:/ix;" + ' collx_add_host_verbs_provider_mount && echo "$CONTAINER_MOUNTS"') + + def test_multi_node_shards_bind_the_host_provider_read_only(self) -> None: + # Backend preparation installs it over the image's provider; the bind only delivers it. + result = self._mounts(self.PROVIDER, 2) + self.assertEqual(result.stdout.strip(), "/stage:/ix," + f"{self.PROVIDER}:/opt/collectivex/host-verbs/libbnxt_re-rdmav34.so:ro") + + def test_single_node_and_unset_keep_their_mounts(self) -> None: + for provider, nodes in ((self.PROVIDER, 1), ("", 2)): + with self.subTest(provider=provider, nodes=nodes): + self.assertEqual(self._mounts(provider, nodes).stdout.strip(), "/stage:/ix") + + def test_a_path_that_is_not_a_verbs_provider_fails_closed(self) -> None: + for provider in ("/usr/lib/libc.so.6", "relative/libbnxt_re-rdmav34.so", + "/usr/../etc/libbnxt_re-rdmav34.so"): + with self.subTest(provider=provider): + self.assertNotEqual(self._mounts(provider, 2).returncode, 0) + + +class SocketInterfaceProfileTests(unittest.TestCase): + def test_every_bootstrap_uses_the_registry_interface(self) -> None: + # MoRI's shmem bootstrap picks its own interface otherwise; on per-rail /31 links that can + # be one without a kernel route to the root rank. + stdout = run_common( + "export COLLX_RDMA_DEVICES=bnxt_re0 COLLX_SOCKET_IFNAME=mgmt0 MORI_SOCKET_IFNAME=stale;" + " collx_apply_network_profile 2 xgmi-rdma;" + ' echo "$NCCL_SOCKET_IFNAME $GLOO_SOCKET_IFNAME $MORI_SOCKET_IFNAME $UCCL_SOCKET_IFNAME"', + check=True, + ).stdout + self.assertEqual(stdout.strip().splitlines()[-1], "mgmt0 mgmt0 mgmt0 mgmt0") + + class RelaxedOrderingProfileTests(unittest.TestCase): def test_only_the_registry_flag_relaxes_the_deepep_window(self) -> None: for flag, expected in (("1", "1"), ("0", "unset"), ("", "unset")):