diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml index 885fb3baef..7e9061646c 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml @@ -47,7 +47,7 @@ spec: # scripts/sync-versions.sh) — same restore-entrypoint binary the # production deploy uses. - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.42 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.65 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-qwen3-32b-criu-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-qwen3-32b-criu-restore.yaml new file mode 100644 index 0000000000..d5e8b6fe39 --- /dev/null +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-qwen3-32b-criu-restore.yaml @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# GENERATED by internal/manifests -- do not edit. +# Regenerate with: go generate ./internal/manifests/... +# +# Restore placeholder for a criu-v2 (in-namespace) checkpoint of nim-qwen3-32b-criu. +# +# Dumb reaper: same image as the source (CRIU's path-based file checks resolve +# against an identical rootfs), bash pid1 reaps orphans, no restore-entrypoint +# and no hostPID -- the pod keeps its own fresh pid namespace, which is where +# the in-namespace CRIU restores the dumped session. The agent drives +# everything on POST /v1/restore. See internal/agent/restore_v2.go. +apiVersion: v1 +kind: Pod +metadata: + name: nim-qwen3-32b-criu-restored + namespace: nvsnap-system + labels: + app: nim-qwen3-32b-criu-restored + nvsnap.io/demo: "true" +spec: + automountServiceAccountToken: false + # IMPORTANT: must run on the same node as the source pod's checkpoint. + # test-e2e.sh substitutes __NODE_NAME__ from the source pod's status. + nodeName: __NODE_NAME__ + + imagePullSecrets: + - name: nvsnap-pull-secret + - name: nim-pull-secret + + containers: + - name: restore + image: nvcr.io/nim/qwen/qwen3-32b:1.0.0 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-lc"] + args: + - | + set -e + # Push this pod's own pid allocations high so the low pid range the + # dump captured stays free for CRIU's exact-pid forks. + echo 100000 > /proc/sys/kernel/ns_last_pid || echo "WARNING: ns_last_pid bump FAILED - restore will hit pid collisions" + # Restored workload stdio is a plain-file fd on /opt/nim/nim.out (the + # source manifest's setsid convention); surface it via kubelet. + touch /opt/nim/nim.out + tail -F /opt/nim/nim.out & + while true; do sleep 30; done + env: + # CHECKPOINT_ID intentionally in block style -- test-e2e.sh's sed + # substitution advances to the NEXT line. + - name: CHECKPOINT_ID + value: "__CHECKPOINT_ID__" + readinessProbe: + httpGet: + path: /v1/health/ready + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 80 + securityContext: + privileged: true + # Root, regardless of what the source image runs as. The placeholder has + # to write /proc/sys/kernel/ns_last_pid, and privileged does not confer + # root: an image defaulting to a non-root uid (NIM runs as 1000) fails + # that write, leaves its pid range unreserved, and the restore then dies + # with "Can't fork for : File exists". The placeholder is a throwaway + # reaper and the restored workload's own uid comes from the checkpoint, + # so this does not change what the workload runs as. + runAsUser: 0 + resources: + limits: + nvidia.com/gpu: "2" + requests: + nvidia.com/gpu: "2" + volumeMounts: + - { name: checkpoints, mountPath: /checkpoints } + - { name: dev-shm, mountPath: /dev/shm } + + volumes: + - name: checkpoints + hostPath: + path: /var/lib/containerd/nvsnap-checkpoints + type: Directory + - name: dev-shm + emptyDir: + medium: Memory + sizeLimit: 16Gi + + restartPolicy: Never diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-qwen3-32b-criu.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-qwen3-32b-criu.yaml new file mode 100644 index 0000000000..b4c600674e --- /dev/null +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-qwen3-32b-criu.yaml @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# nim-qwen3-32b-criu SOURCE pod: Qwen3-32B TP=2 on the criu-v2 engine. +# +# Separate from nim-qwen3-32b, which stays on the cachedir path. The two +# engines do not share a manifest. +# +# The stock NIM image has no command: it runs /opt/nvidia/nvidia_entrypoint.sh +# with cmd `bash -c $SERVER_START_SCRIPT_PATH`. criu-v2 needs the setsid +# convention instead (workload in its own session, stdio to a rootfs file), so +# the command below reproduces the image's own startup inside that convention +# rather than replacing it: the entrypoint still runs, and still execs the +# server start script the image names. +# +# Stdio goes to /opt/nim/nim.out rather than /nim.out because the image runs as +# uid 1000 (nvs) and cannot write the container root. Same reason NIM's restore +# placeholder needs runAsUser 0 for the pid reservation: privileged does not +# confer root. +# +# Peer-state sever: cuda-checkpoint blocks if cross-GPU state exists. The NCCL +# knobs are engine-independent so they apply here as they do to vLLM. There is +# no CUDA-graph disable: TRT-LLM configures graphs at engine build time and the +# image exposes no env for it. That is deliberate for a first run, because graphs +# fail late (restore inference) rather than at capture, so their absence does not +# block finding out whether capture works at all. +# See docs/proposals/multi-gpu-criu-v2.md. +apiVersion: v1 +kind: Pod +metadata: + name: nim-qwen3-32b-criu + namespace: nvsnap-system + labels: + app: nim-qwen3-32b-criu + nvsnap.io/demo: "true" + annotations: + nvsnap.io/demo-name: "NIM criu-v2" + nvsnap.io/desc: "criu-v2 Qwen3-32B multi-GPU TP=2" + nvsnap.io/model: "qwen/qwen3-32b" + nvsnap.io/port: "8000" + nvsnap.io/gpus: "2" + nvsnap.io/path: "criu" +spec: + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + + imagePullSecrets: + - name: nvsnap-pull-secret + - name: nim-pull-secret + + containers: + - name: nim + image: nvcr.io/nim/qwen/qwen3-32b:1.0.0 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-lc"] + args: + - | + set -e + # criu-v2 convention: the workload runs in its own session via setsid, + # so the agent dumps the GPU leader's session rather than container + # init, which cannot be restored into a placeholder whose pid 1 is + # already occupied. bash stays pid 1 to reap orphans and tails the + # output file so kubectl logs keeps working. + nohup setsid /opt/nvidia/nvidia_entrypoint.sh \ + /bin/bash -c "$SERVER_START_SCRIPT_PATH" > /opt/nim/nim.out 2>&1 < /dev/null & + tail -F /opt/nim/nim.out & + while true; do sleep 30; done + env: + - name: NGC_API_KEY + valueFrom: + secretKeyRef: + name: ngc-api-key + key: NGC_API_KEY + - { name: NIM_CACHE_PATH, value: "/opt/nim/.cache" } + - { name: HF_HUB_DISABLE_XET, value: "1" } + - { name: CUDA_VISIBLE_DEVICES, value: "0,1" } + # Peer-state sever (see header). + - { name: NCCL_P2P_DISABLE, value: "1" } + - { name: NCCL_NVLS_ENABLE, value: "0" } + - { name: NCCL_SHM_DISABLE, value: "1" } + ports: + - containerPort: 8000 + name: http + readinessProbe: + httpGet: { path: /v1/health/ready, port: 8000 } + initialDelaySeconds: 60 + periodSeconds: 15 + failureThreshold: 80 + resources: + limits: + nvidia.com/gpu: "2" + requests: + nvidia.com/gpu: "2" + securityContext: + privileged: true + volumeMounts: + - { name: shm, mountPath: /dev/shm } + - { name: nim-cache, mountPath: /opt/nim/.cache } + + volumes: + - name: shm + emptyDir: { medium: Memory, sizeLimit: 16Gi } + - name: nim-cache + # emptyDir, despite the ~62G re-download per run. A hostPath is created + # root-owned by DirectoryOrCreate and NIM runs as uid 1000, which fails at + # manifest download with "Permission denied (os error 13)". Making it + # persistent needs an initContainer to chown, which is not worth coupling + # to the question this workload exists to answer. + emptyDir: {} + + restartPolicy: Never diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-tp2-criu-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-tp2-criu-restore.yaml new file mode 100644 index 0000000000..a6755b2170 --- /dev/null +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-tp2-criu-restore.yaml @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# GENERATED by internal/manifests -- do not edit. +# Regenerate with: go generate ./internal/manifests/... +# +# Restore placeholder for a criu-v2 (in-namespace) checkpoint of sglang-tp2-criu. +# +# Dumb reaper: same image as the source (CRIU's path-based file checks resolve +# against an identical rootfs), bash pid1 reaps orphans, no restore-entrypoint +# and no hostPID -- the pod keeps its own fresh pid namespace, which is where +# the in-namespace CRIU restores the dumped session. The agent drives +# everything on POST /v1/restore. See internal/agent/restore_v2.go. +apiVersion: v1 +kind: Pod +metadata: + name: sglang-tp2-criu-restored + namespace: nvsnap-system + labels: + app: sglang-tp2-criu-restored + nvsnap.io/demo: "true" +spec: + automountServiceAccountToken: false + # IMPORTANT: must run on the same node as the source pod's checkpoint. + # test-e2e.sh substitutes __NODE_NAME__ from the source pod's status. + nodeName: __NODE_NAME__ + + imagePullSecrets: + - name: nvsnap-pull-secret + + containers: + - name: restore + image: lmsysorg/sglang:v0.5.15.post1-cu129 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-lc"] + args: + - | + set -e + # Push this pod's own pid allocations high so the low pid range the + # dump captured stays free for CRIU's exact-pid forks. + echo 100000 > /proc/sys/kernel/ns_last_pid || echo "WARNING: ns_last_pid bump FAILED - restore will hit pid collisions" + # Restored workload stdio is a plain-file fd on /sglang.out (the + # source manifest's setsid convention); surface it via kubelet. + touch /sglang.out + tail -F /sglang.out & + while true; do sleep 30; done + env: + # CHECKPOINT_ID intentionally in block style -- test-e2e.sh's sed + # substitution advances to the NEXT line. + - name: CHECKPOINT_ID + value: "__CHECKPOINT_ID__" + readinessProbe: + httpGet: + path: /v1/models + port: 30000 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 60 + securityContext: + privileged: true + # Root, regardless of what the source image runs as. The placeholder has + # to write /proc/sys/kernel/ns_last_pid, and privileged does not confer + # root: an image defaulting to a non-root uid (NIM runs as 1000) fails + # that write, leaves its pid range unreserved, and the restore then dies + # with "Can't fork for : File exists". The placeholder is a throwaway + # reaper and the restored workload's own uid comes from the checkpoint, + # so this does not change what the workload runs as. + runAsUser: 0 + resources: + limits: + nvidia.com/gpu: "2" + requests: + nvidia.com/gpu: "2" + volumeMounts: + - { name: checkpoints, mountPath: /checkpoints } + - { name: dev-shm, mountPath: /dev/shm } + + volumes: + - name: checkpoints + hostPath: + path: /var/lib/containerd/nvsnap-checkpoints + type: Directory + - name: dev-shm + emptyDir: + medium: Memory + sizeLimit: 16Gi + + restartPolicy: Never diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-tp2-criu.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-tp2-criu.yaml new file mode 100644 index 0000000000..6241720a48 --- /dev/null +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-tp2-criu.yaml @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# SGLang with Llama-3.1-8B-Instruct (single GPU) — criu-v2 checkpoint/restore. +# +# No injection stack: the criu-v2 in-namespace CRIU engine handles the process +# at the OS level, so the container just runs stock SGLang under setsid. +# Restore lands in a dumb placeholder pod driven by the agent. See +# checkpoint_v2.go. +# +# This workload previously carried the legacy interception stack (patched +# uvloop/libuv/libzmq init containers, LD_PRELOAD, and /etc/ld.so.preload). +# That stack is incompatible with criu-v2: /etc/ld.so.preload is a property of +# the mount namespace, and criu-v2 nsenters CRIU *into* that namespace, so the +# loader force-loaded the intercept library into CRIU itself and the dump hung +# before finishing seize. The legacy engine ran CRIU from the agent's namespace +# and never saw the file, which is why the same manifest used to work. +apiVersion: v1 +kind: Pod +metadata: + name: sglang-tp2-criu + namespace: nvsnap-system + labels: + app: sglang-tp2-criu + nvsnap.io/demo: "true" + annotations: + nvsnap.io/demo-name: "SGLang" + nvsnap.io/desc: "Llama-3.1-8B on SGLang (single GPU)" + nvsnap.io/model: "meta-llama/Llama-3.1-8B-Instruct" + nvsnap.io/port: "30000" + nvsnap.io/gpus: "2" + nvsnap.io/path: "criu" + nvsnap.io/ckpt-size: "79 GB" +spec: + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + + imagePullSecrets: + - name: nvsnap-pull-secret + + containers: + - name: sglang + image: lmsysorg/sglang:v0.5.15.post1-cu129 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-lc"] + args: + - | + set -e + ulimit -c unlimited + # criu-v2 convention: launch the workload in its own session via + # setsid — the agent dumps the GPU leader's session, never container + # init (pid1 can't be restored into a placeholder whose pid1 slot is + # occupied). Stdio goes to a rootfs file so CRIU restores those fds + # as plain files; bash stays pid1 to reap orphan zombies and tails + # the file so kubectl logs keeps working. + nohup setsid python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --host 0.0.0.0 \ + --port 30000 \ + --tp-size 2 \ + --mem-fraction-static 0.6 \ + --disable-cuda-graph \ + --disable-custom-all-reduce \ + --disable-prefill-cuda-graph > /sglang.out 2>&1 < /dev/null & + tail -F /sglang.out & + while true; do sleep 30; done + env: + - { name: LD_LIBRARY_PATH, value: "/usr/local/nvidia/lib64:/usr/local/cuda/lib64" } + - { name: PYTHONFAULTHANDLER, value: "1" } + - { name: PYTHONUNBUFFERED, value: "1" } + - { name: NVSNAP_LOG_LEVEL, value: "3" } + - { name: NVSNAP_SECCOMP_ENABLED, value: "0" } + - { name: CUDA_VISIBLE_DEVICES, value: "0,1" } + # Peer-state sever: cuda-checkpoint blocks if ANY cross-GPU + # mapping exists. See docs/proposals/multi-gpu-criu-v2.md. + - { name: NCCL_P2P_DISABLE, value: "1" } + - { name: NCCL_NVLS_ENABLE, value: "0" } + - { name: NCCL_SHM_DISABLE, value: "1" } + - { name: HF_HUB_DISABLE_XET, value: "1" } + - { name: HF_HUB_ENABLE_HF_TRANSFER, value: "0" } + # Keep torch off the libuv TCPStore backend. The patched libuv that + # used to be injected here is gone with the rest of the stack, so stay + # on the non-libuv path rather than depending on a stock libuv that + # this workload never exercised. + - { name: USE_LIBUV, value: "0" } + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token + key: token + ports: + - containerPort: 30000 + name: http + readinessProbe: + httpGet: + path: /v1/models + port: 30000 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 60 + resources: + limits: + nvidia.com/gpu: "2" + requests: + nvidia.com/gpu: "2" + securityContext: + privileged: true + volumeMounts: + - { name: shm, mountPath: /dev/shm } + + volumes: + - name: shm + emptyDir: + medium: Memory + sizeLimit: 16Gi + + restartPolicy: Never diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-70b-criu-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-70b-criu-restore.yaml new file mode 100644 index 0000000000..5e50e621ac --- /dev/null +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-70b-criu-restore.yaml @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# GENERATED by internal/manifests -- do not edit. +# Regenerate with: go generate ./internal/manifests/... +# +# Restore placeholder for a criu-v2 (in-namespace) checkpoint of vllm-70b-criu. +# +# Dumb reaper: same image as the source (CRIU's path-based file checks resolve +# against an identical rootfs), bash pid1 reaps orphans, no restore-entrypoint +# and no hostPID -- the pod keeps its own fresh pid namespace, which is where +# the in-namespace CRIU restores the dumped session. The agent drives +# everything on POST /v1/restore. See internal/agent/restore_v2.go. +apiVersion: v1 +kind: Pod +metadata: + name: vllm-70b-criu-restored + namespace: nvsnap-system + labels: + app: vllm-70b-criu-restored + nvsnap.io/demo: "true" +spec: + automountServiceAccountToken: false + # IMPORTANT: must run on the same node as the source pod's checkpoint. + # test-e2e.sh substitutes __NODE_NAME__ from the source pod's status. + nodeName: __NODE_NAME__ + + imagePullSecrets: + - name: nvsnap-pull-secret + + containers: + - name: restore + image: vllm/vllm-openai:v0.20.0 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-lc"] + args: + - | + set -e + mkdir -p /var/run/vllm + # Push this pod's own pid allocations high so the low pid range the + # dump captured stays free for CRIU's exact-pid forks. + echo 100000 > /proc/sys/kernel/ns_last_pid || echo "WARNING: ns_last_pid bump FAILED - restore will hit pid collisions" + # Restored workload stdio is a plain-file fd on /vllm.out (the + # source manifest's setsid convention); surface it via kubelet. + touch /vllm.out + tail -F /vllm.out & + while true; do sleep 30; done + env: + # CHECKPOINT_ID intentionally in block style -- test-e2e.sh's sed + # substitution advances to the NEXT line. + - name: CHECKPOINT_ID + value: "__CHECKPOINT_ID__" + - { name: HF_HOME, value: "/root/.cache/huggingface" } + readinessProbe: + httpGet: + path: /v1/models + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 60 + securityContext: + privileged: true + # Root, regardless of what the source image runs as. The placeholder has + # to write /proc/sys/kernel/ns_last_pid, and privileged does not confer + # root: an image defaulting to a non-root uid (NIM runs as 1000) fails + # that write, leaves its pid range unreserved, and the restore then dies + # with "Can't fork for : File exists". The placeholder is a throwaway + # reaper and the restored workload's own uid comes from the checkpoint, + # so this does not change what the workload runs as. + runAsUser: 0 + resources: + limits: + nvidia.com/gpu: "4" + requests: + nvidia.com/gpu: "4" + volumeMounts: + - { name: checkpoints, mountPath: /checkpoints } + - { name: dev-shm, mountPath: /dev/shm } + + volumes: + - name: checkpoints + hostPath: + path: /var/lib/containerd/nvsnap-checkpoints + type: Directory + - name: dev-shm + emptyDir: + medium: Memory + sizeLimit: 16Gi + + restartPolicy: Never diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-70b-criu.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-70b-criu.yaml new file mode 100644 index 0000000000..2f070bb455 --- /dev/null +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-70b-criu.yaml @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# vLLM TinyLlama-1.1B, TP=2, EAGER (no CUDA graphs), multi-process. The +# multi-GPU coverage for the cachedir capture path. +# +# This is NOT a CRIU workload. The agent refuses multi-GPU CRIU outright, so +# the capture is cachedir (nvsnap.io/path below) and test-e2e.sh routes +# anything requesting >= 2 GPUs there automatically. Do not force +# CAPTURE_PATH=criu-v2 against it: the request is rejected, and the manifest +# generator would emit a criu-v2 restore placeholder nothing drives. +# +# Run with: ./scripts/test-e2e.sh vllm-tp2 +# +# Transparent multi-GPU CRIU remains open: eager cuda-checkpoint works but +# peer state is the blocker, tracked separately. +apiVersion: v1 +kind: Pod +metadata: + name: vllm-70b-criu + namespace: nvsnap-system + labels: + app: vllm-70b-criu + nvsnap.io/demo: "true" + annotations: + nvsnap.io/demo-name: "vLLM TP=2 eager" + nvsnap.io/desc: "criu-v2 TinyLlama 1.1B on vLLM, TP=2 eager (multi-GPU cachedir)" + nvsnap.io/model: "TinyLlama/TinyLlama-1.1B-Chat-v1.0" + nvsnap.io/port: "8000" + nvsnap.io/gpus: "4" + # cachedir, not criu: the agent refuses multi-GPU CRIU outright + # (checkpoint.go, "multi-GPU CRIU is unsupported"), and test-e2e.sh routes + # anything requesting >= 2 GPUs here. Declaring "criu" made the manifest + # generator emit a criu-v2 restore placeholder that nothing ever drives, + # so the restore pod idled until the readiness timeout. + nvsnap.io/path: "criu" +spec: + automountServiceAccountToken: false + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + + imagePullSecrets: + - name: nvsnap-pull-secret + + containers: + - name: vllm + image: vllm/vllm-openai:v0.20.0 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-lc"] + args: + - | + set -e + ulimit -c unlimited + mkdir -p /var/run/vllm + # criu-v2 convention: workload in its own session via setsid; stdio + # to a rootfs file; bash stays pid1 to reap orphans. See vllm-small. + # + # E1b config-severed profile (ember M0 Exp 3): GPU peer state is THE + # multi-GPU cuda-checkpoint blocker - with default config (NVLink P2P + # + custom all-reduce) the TP workers hang in UvmUnregisterGpu even + # when ALL ranks are locked first (verified 2026-07-18, driver + # 580.126). Disabling every peer-state path (flags + env below) drops + # all-reduce to PYNCCL over sockets, and per-rank cuda-checkpoint + # works. Default-config support needs the transparent sever + # (teardown/reinit) - the nvsnap_cr.so/lib-sever port, issue #25. + nohup setsid vllm serve \ + --model meta-llama/Llama-3.1-70B-Instruct \ + --host 0.0.0.0 \ + --port 8000 \ + --max-model-len 4096 \ + --tensor-parallel-size 4 \ + --enforce-eager \ + --disable-custom-all-reduce \ + --gpu-memory-utilization 0.85 > /vllm.out 2>&1 < /dev/null & + tail -F /vllm.out & + while true; do sleep 30; done + env: + - { name: LD_LIBRARY_PATH, value: "/usr/local/nvidia/lib64:/usr/local/cuda/lib64" } + - { name: PYTHONFAULTHANDLER, value: "1" } + - { name: PYTHONUNBUFFERED, value: "1" } + - { name: NVSNAP_LOG_LEVEL, value: "3" } + - { name: NVSNAP_SECCOMP_ENABLED, value: "0" } + - { name: CUDA_VISIBLE_DEVICES, value: "0,1,2,3" } + - { name: HF_HOME, value: "/root/.cache/huggingface" } + - name: HF_TOKEN + valueFrom: + secretKeyRef: { name: hf-token, key: token } + # TP=2 requires the multi-process engine (EngineCore + TP workers). + - { name: VLLM_ENABLE_V1_MULTIPROCESSING, value: "1" } + # E1b peer-state sever (config-based, ember M0 Exp 3 set): kill NCCL + # P2P/NVLS/SHM transports + vLLM symm-mem all-reduce. Together with + # --disable-custom-all-reduce above, no cross-GPU peer mappings exist. + - { name: NCCL_P2P_DISABLE, value: "1" } + - { name: NCCL_NVLS_ENABLE, value: "0" } + - { name: NCCL_SHM_DISABLE, value: "1" } + - { name: VLLM_ALLREDUCE_USE_SYMM_MEM, value: "0" } + - { name: USE_LIBUV, value: "1" } + # io_uring is C/R-safe (CRIU fork sq_array identity-map restore fix). + - { name: UV_USE_IO_URING, value: "1" } + - { name: HF_HUB_DISABLE_XET, value: "1" } + - { name: HF_HUB_ENABLE_HF_TRANSFER, value: "0" } + - { name: VLLM_LOGGING_LEVEL, value: "INFO" } + ports: + - containerPort: 8000 + name: http + readinessProbe: + httpGet: { path: /v1/models, port: 8000 } + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 60 + resources: + limits: + nvidia.com/gpu: "4" + requests: + nvidia.com/gpu: "4" + securityContext: + privileged: true + volumeMounts: + - { name: shm, mountPath: /dev/shm } + + volumes: + - name: shm + emptyDir: { medium: Memory, sizeLimit: 16Gi } + + restartPolicy: Never diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-tp2-criu-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-tp2-criu-restore.yaml new file mode 100644 index 0000000000..93c91d0b51 --- /dev/null +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-tp2-criu-restore.yaml @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# GENERATED by internal/manifests -- do not edit. +# Regenerate with: go generate ./internal/manifests/... +# +# Restore placeholder for a criu-v2 (in-namespace) checkpoint of vllm-tp2-criu. +# +# Dumb reaper: same image as the source (CRIU's path-based file checks resolve +# against an identical rootfs), bash pid1 reaps orphans, no restore-entrypoint +# and no hostPID -- the pod keeps its own fresh pid namespace, which is where +# the in-namespace CRIU restores the dumped session. The agent drives +# everything on POST /v1/restore. See internal/agent/restore_v2.go. +apiVersion: v1 +kind: Pod +metadata: + name: vllm-tp2-criu-restored + namespace: nvsnap-system + labels: + app: vllm-tp2-criu-restored + nvsnap.io/demo: "true" +spec: + automountServiceAccountToken: false + # IMPORTANT: must run on the same node as the source pod's checkpoint. + # test-e2e.sh substitutes __NODE_NAME__ from the source pod's status. + nodeName: __NODE_NAME__ + + imagePullSecrets: + - name: nvsnap-pull-secret + + containers: + - name: restore + image: vllm/vllm-openai:v0.20.0 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-lc"] + args: + - | + set -e + mkdir -p /var/run/vllm + # Push this pod's own pid allocations high so the low pid range the + # dump captured stays free for CRIU's exact-pid forks. + echo 100000 > /proc/sys/kernel/ns_last_pid || echo "WARNING: ns_last_pid bump FAILED - restore will hit pid collisions" + # Restored workload stdio is a plain-file fd on /vllm.out (the + # source manifest's setsid convention); surface it via kubelet. + touch /vllm.out + tail -F /vllm.out & + while true; do sleep 30; done + env: + # CHECKPOINT_ID intentionally in block style -- test-e2e.sh's sed + # substitution advances to the NEXT line. + - name: CHECKPOINT_ID + value: "__CHECKPOINT_ID__" + - { name: HF_HOME, value: "/root/.cache/huggingface" } + readinessProbe: + httpGet: + path: /v1/models + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 60 + securityContext: + privileged: true + # Root, regardless of what the source image runs as. The placeholder has + # to write /proc/sys/kernel/ns_last_pid, and privileged does not confer + # root: an image defaulting to a non-root uid (NIM runs as 1000) fails + # that write, leaves its pid range unreserved, and the restore then dies + # with "Can't fork for : File exists". The placeholder is a throwaway + # reaper and the restored workload's own uid comes from the checkpoint, + # so this does not change what the workload runs as. + runAsUser: 0 + resources: + limits: + nvidia.com/gpu: "2" + requests: + nvidia.com/gpu: "2" + volumeMounts: + - { name: checkpoints, mountPath: /checkpoints } + - { name: dev-shm, mountPath: /dev/shm } + + volumes: + - name: checkpoints + hostPath: + path: /var/lib/containerd/nvsnap-checkpoints + type: Directory + - name: dev-shm + emptyDir: + medium: Memory + sizeLimit: 16Gi + + restartPolicy: Never diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-tp2-criu.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-tp2-criu.yaml new file mode 100644 index 0000000000..626ea6d2e1 --- /dev/null +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-tp2-criu.yaml @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# vLLM TinyLlama-1.1B, TP=2, EAGER (no CUDA graphs), multi-process. The +# multi-GPU coverage for the cachedir capture path. +# +# This is NOT a CRIU workload. The agent refuses multi-GPU CRIU outright, so +# the capture is cachedir (nvsnap.io/path below) and test-e2e.sh routes +# anything requesting >= 2 GPUs there automatically. Do not force +# CAPTURE_PATH=criu-v2 against it: the request is rejected, and the manifest +# generator would emit a criu-v2 restore placeholder nothing drives. +# +# Run with: ./scripts/test-e2e.sh vllm-tp2 +# +# Transparent multi-GPU CRIU remains open: eager cuda-checkpoint works but +# peer state is the blocker, tracked separately. +apiVersion: v1 +kind: Pod +metadata: + name: vllm-tp2-criu + namespace: nvsnap-system + labels: + app: vllm-tp2-criu + nvsnap.io/demo: "true" + annotations: + nvsnap.io/demo-name: "vLLM TP=2 eager" + nvsnap.io/desc: "criu-v2 TinyLlama 1.1B on vLLM, TP=2 eager (multi-GPU cachedir)" + nvsnap.io/model: "TinyLlama/TinyLlama-1.1B-Chat-v1.0" + nvsnap.io/port: "8000" + nvsnap.io/gpus: "2" + # cachedir, not criu: the agent refuses multi-GPU CRIU outright + # (checkpoint.go, "multi-GPU CRIU is unsupported"), and test-e2e.sh routes + # anything requesting >= 2 GPUs here. Declaring "criu" made the manifest + # generator emit a criu-v2 restore placeholder that nothing ever drives, + # so the restore pod idled until the readiness timeout. + nvsnap.io/path: "criu" +spec: + automountServiceAccountToken: false + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + + imagePullSecrets: + - name: nvsnap-pull-secret + + containers: + - name: vllm + image: vllm/vllm-openai:v0.20.0 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-lc"] + args: + - | + set -e + ulimit -c unlimited + mkdir -p /var/run/vllm + # criu-v2 convention: workload in its own session via setsid; stdio + # to a rootfs file; bash stays pid1 to reap orphans. See vllm-small. + # + # E1b config-severed profile (ember M0 Exp 3): GPU peer state is THE + # multi-GPU cuda-checkpoint blocker - with default config (NVLink P2P + # + custom all-reduce) the TP workers hang in UvmUnregisterGpu even + # when ALL ranks are locked first (verified 2026-07-18, driver + # 580.126). Disabling every peer-state path (flags + env below) drops + # all-reduce to PYNCCL over sockets, and per-rank cuda-checkpoint + # works. Default-config support needs the transparent sever + # (teardown/reinit) - the nvsnap_cr.so/lib-sever port, issue #25. + nohup setsid vllm serve \ + --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 \ + --host 0.0.0.0 \ + --port 8000 \ + --max-model-len 2048 \ + --tensor-parallel-size 2 \ + --enforce-eager \ + --disable-custom-all-reduce \ + --gpu-memory-utilization 0.3 > /vllm.out 2>&1 < /dev/null & + tail -F /vllm.out & + while true; do sleep 30; done + env: + - { name: LD_LIBRARY_PATH, value: "/usr/local/nvidia/lib64:/usr/local/cuda/lib64" } + - { name: PYTHONFAULTHANDLER, value: "1" } + - { name: PYTHONUNBUFFERED, value: "1" } + - { name: NVSNAP_LOG_LEVEL, value: "3" } + - { name: NVSNAP_SECCOMP_ENABLED, value: "0" } + - { name: CUDA_VISIBLE_DEVICES, value: "0,1" } + - { name: HF_HOME, value: "/root/.cache/huggingface" } + # TP=2 requires the multi-process engine (EngineCore + TP workers). + - { name: VLLM_ENABLE_V1_MULTIPROCESSING, value: "1" } + # E1b peer-state sever (config-based, ember M0 Exp 3 set): kill NCCL + # P2P/NVLS/SHM transports + vLLM symm-mem all-reduce. Together with + # --disable-custom-all-reduce above, no cross-GPU peer mappings exist. + - { name: NCCL_P2P_DISABLE, value: "1" } + - { name: NCCL_NVLS_ENABLE, value: "0" } + - { name: NCCL_SHM_DISABLE, value: "1" } + - { name: VLLM_ALLREDUCE_USE_SYMM_MEM, value: "0" } + - { name: USE_LIBUV, value: "1" } + # io_uring is C/R-safe (CRIU fork sq_array identity-map restore fix). + - { name: UV_USE_IO_URING, value: "1" } + - { name: HF_HUB_DISABLE_XET, value: "1" } + - { name: HF_HUB_ENABLE_HF_TRANSFER, value: "0" } + - { name: VLLM_LOGGING_LEVEL, value: "INFO" } + ports: + - containerPort: 8000 + name: http + readinessProbe: + httpGet: { path: /v1/models, port: 8000 } + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 60 + resources: + limits: + nvidia.com/gpu: "2" + requests: + nvidia.com/gpu: "2" + securityContext: + privileged: true + volumeMounts: + - { name: shm, mountPath: /dev/shm } + + volumes: + - name: shm + emptyDir: { medium: Memory, sizeLimit: 16Gi } + + restartPolicy: Never diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-tp2.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-tp2.yaml index 7f44d325ea..7107da8698 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-tp2.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-tp2.yaml @@ -90,7 +90,7 @@ spec: # P2P/NVLS/SHM transports + vLLM symm-mem all-reduce. Together with # --disable-custom-all-reduce above, no cross-GPU peer mappings exist. - { name: NCCL_P2P_DISABLE, value: "1" } - - { name: NCCL_NVLS_DISABLE, value: "1" } + - { name: NCCL_NVLS_ENABLE, value: "0" } - { name: NCCL_SHM_DISABLE, value: "1" } - { name: VLLM_ALLREDUCE_USE_SYMM_MEM, value: "0" } - { name: USE_LIBUV, value: "1" } diff --git a/src/compute-plane-services/nvsnap/docker/agent/Dockerfile.base b/src/compute-plane-services/nvsnap/docker/agent/Dockerfile.base index 53f069c647..86fc67f2fd 100644 --- a/src/compute-plane-services/nvsnap/docker/agent/Dockerfile.base +++ b/src/compute-plane-services/nvsnap/docker/agent/Dockerfile.base @@ -15,7 +15,13 @@ # lock timeouts). Requires driver 570+ on the node at runtime. The CUDA devel # image supplies cuda.h and the libcuda link stub; at runtime the wrapper # points LD_LIBRARY_PATH at the node's real driver libcuda. -FROM nvidia/cuda:12.8.1-devel-ubuntu22.04 AS cuda-cli-builder +# CUDA 13 is required for the r580 GPU-migration fields: 12.x declares +# CUcheckpointRestoreArgs as a reserved[8] blob with no gpuPairs member, so +# migration cannot be expressed against those headers at all. Staying on +# ubuntu22.04 is deliberate -- the produced binary runs inside the workload's +# container against the criu bundle's glibc 2.35, and a 24.04 builder would +# raise the required glibc and break that contract. +FROM nvidia/cuda:13.0.3-devel-ubuntu22.04 AS cuda-cli-builder COPY nvsnap-cuda-checkpoint.c /src/nvsnap-cuda-checkpoint.c RUN gcc -O2 -o /src/nvsnap-cuda-checkpoint /src/nvsnap-cuda-checkpoint.c \ -I/usr/local/cuda/include -L/usr/local/cuda/lib64/stubs -lcuda && \ diff --git a/src/compute-plane-services/nvsnap/docker/agent/nvsnap-cuda-checkpoint.c b/src/compute-plane-services/nvsnap/docker/agent/nvsnap-cuda-checkpoint.c index 35f186df2e..58a5651956 100644 --- a/src/compute-plane-services/nvsnap/docker/agent/nvsnap-cuda-checkpoint.c +++ b/src/compute-plane-services/nvsnap/docker/agent/nvsnap-cuda-checkpoint.c @@ -12,8 +12,19 @@ * --toggle --pid * --get-restore-tid --pid * - * Single-GPU scope: the 12.x CUcheckpointRestoreArgs has no gpuPairs field, so - * no GPU migration (that is a CUDA 13 / r580 feature for the multi-GPU phase). + * Plus one operation upstream's CLI does not expose: + * + * --gpu-map :[,:...] + * + * GPU migration (driver r580+): restore a checkpoint onto different physical + * GPUs than it was captured on, by mapping each source device UUID to a target + * device UUID. This is what lets a restore be scheduled wherever there is + * capacity instead of being pinned to the GPU slot it was captured from. + * Applies to restore, resume, and the restore half of toggle; ignored (with a + * diagnostic) on lock/checkpoint, which have no such argument. + * + * Requires CUDA 13 headers: 12.x declares CUcheckpointRestoreArgs as an opaque + * reserved[8] and has no gpuPairs member to populate. * * Build (in an env with cuda.h and the driver's libcuda.so): * gcc -O2 nvsnap-cuda-checkpoint.c -o nvsnap-cuda-checkpoint -lcuda @@ -21,6 +32,7 @@ #include #include #include +#include #include #include @@ -51,6 +63,178 @@ static const char *state_name(CUprocessState s) } } +/* ---- multi-pid batching -------------------------------------------------- + * + * The CRIU plugin drives this one pid per exec, so an N-rank job means N + * separate processes with N driver attaches and, more importantly, N gaps + * between them. On a multi-GPU job those gaps are where collective traffic + * resumes between one rank being locked and the next, which is the shape of + * the documented "hangs on 2nd rank (lock timeout)" failure. + * + * Accepting several pids in one invocation removes the gaps and lets the + * ordering be stated explicitly (see the "save" action). A single --pid still + * behaves exactly as before, so the plugin's calls are unaffected. + */ +#define MAX_PIDS 512 +static int g_pids[MAX_PIDS]; +static unsigned int g_pid_count; + +static int add_pids(const char *spec) +{ + char *dup = strdup(spec); + if (!dup) { + fprintf(stderr, "error: out of memory\n"); + return -1; + } + int rc = 0; + char *save = NULL; + for (char *tok = strtok_r(dup, ",", &save); tok; tok = strtok_r(NULL, ",", &save)) { + int pid = atoi(tok); + if (pid <= 0) { + fprintf(stderr, "error: invalid pid '%s'\n", tok); + rc = -1; + break; + } + if (g_pid_count >= MAX_PIDS) { + fprintf(stderr, "error: more than %d pids\n", MAX_PIDS); + rc = -1; + break; + } + g_pids[g_pid_count++] = pid; + } + free(dup); + return rc; +} + +/* ---- GPU migration (--gpu-map) ------------------------------------------ + * + * The driver identifies devices by UUID, not by index: an index is only + * meaningful relative to one process's CUDA_VISIBLE_DEVICES on one node, and + * the whole point of migration is that the target node's enumeration differs. + * Indices are still accepted as a convenience for same-node testing and are + * resolved to UUIDs here, before they can be misread anywhere else. + */ +static CUcheckpointGpuPair *g_pairs; +static unsigned int g_pairs_count; + +static int hex_nibble(char c) +{ + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} + +static int uuid_from_index(int idx, CUuuid *out) +{ + CUdevice dev; + CUresult r = cuDeviceGet(&dev, idx); + if (r != CUDA_SUCCESS) { + fprintf(stderr, "--gpu-map: no device at index %d: \"%s\"\n", idx, cu_str(r)); + return -1; + } + r = cuDeviceGetUuid(out, dev); + if (r != CUDA_SUCCESS) { + fprintf(stderr, "--gpu-map: cannot read UUID of device %d: \"%s\"\n", idx, cu_str(r)); + return -1; + } + return 0; +} + +/* Accepts a decimal device index, or a UUID as 32 hex digits with an optional + * "GPU-" prefix and optional dashes -- the forms nvidia-smi and the k8s device + * plugin emit, so an operator can paste either without reformatting. */ +static int parse_gpu_id(const char *s, CUuuid *out) +{ + if (!s || !*s) { + fprintf(stderr, "--gpu-map: empty device id\n"); + return -1; + } + + const char *p = s; + int all_digits = 1; + for (const char *q = s; *q; q++) { + if (!isdigit((unsigned char)*q)) { all_digits = 0; break; } + } + if (all_digits) + return uuid_from_index(atoi(s), out); + + if (!strncmp(p, "GPU-", 4) || !strncmp(p, "gpu-", 4)) + p += 4; + + int n = 0; + for (; *p && n < 16; p++) { + if (*p == '-') continue; + int hi = hex_nibble(*p); + int lo = (p[1] && p[1] != '-') ? hex_nibble(p[1]) : -1; + if (hi < 0 || lo < 0) { + fprintf(stderr, "--gpu-map: malformed device id '%s'\n", s); + return -1; + } + out->bytes[n++] = (char)((hi << 4) | lo); + p++; + } + if (n != 16) { + fprintf(stderr, "--gpu-map: device id '%s' is not 32 hex digits\n", s); + return -1; + } + return 0; +} + +/* spec: "old:new[,old:new...]". Every GPU visible to the target process must + * appear, including ones it never touched -- the driver rejects a partial map + * rather than leaving unlisted devices alone, so a short map fails at restore + * with a considerably less obvious error than this one. */ +static int parse_gpu_map(const char *spec) +{ + unsigned int cap = 1; + for (const char *q = spec; *q; q++) + if (*q == ',') cap++; + + g_pairs = calloc(cap, sizeof(*g_pairs)); + if (!g_pairs) { + fprintf(stderr, "--gpu-map: out of memory\n"); + return -1; + } + + char *dup = strdup(spec); + if (!dup) { + fprintf(stderr, "--gpu-map: out of memory\n"); + return -1; + } + + int rc = 0; + char *save = NULL; + for (char *tok = strtok_r(dup, ",", &save); tok; tok = strtok_r(NULL, ",", &save)) { + char *colon = strchr(tok, ':'); + if (!colon) { + fprintf(stderr, "--gpu-map: expected :, got '%s'\n", tok); + rc = -1; + break; + } + *colon = '\0'; + if (parse_gpu_id(tok, &g_pairs[g_pairs_count].oldUuid) < 0 || + parse_gpu_id(colon + 1, &g_pairs[g_pairs_count].newUuid) < 0) { + rc = -1; + break; + } + g_pairs_count++; + } + free(dup); + if (rc < 0) + return rc; + + int visible = 0; + if (cuDeviceGetCount(&visible) == CUDA_SUCCESS && (unsigned int)visible != g_pairs_count) { + fprintf(stderr, + "--gpu-map: %u pair(s) given but %d GPU(s) are visible; every visible " + "GPU must be mapped (use i:i for the ones that do not move)\n", + g_pairs_count, visible); + return -1; + } + return 0; +} + static int do_lock(int pid, unsigned int timeout_ms) { CUcheckpointLockArgs a = {0}; @@ -69,6 +253,12 @@ static int do_checkpoint(int pid) static int do_restore(int pid) { CUcheckpointRestoreArgs a = {0}; + /* Left zeroed when no map was given, which is the non-migrating restore + * the driver has always done -- the field is additive, not a mode switch. */ + if (g_pairs_count) { + a.gpuPairs = g_pairs; + a.gpuPairsCount = g_pairs_count; + } CUresult r = cuCheckpointProcessRestore(pid, &a); return r == CUDA_SUCCESS ? 0 : err_action("restore", pid, r); } @@ -113,6 +303,38 @@ static int do_get_restore_tid(int pid) return 0; } +/* save: lock EVERY pid, then checkpoint every pid, from this one process. + * + * The two-phase order is the point, not an implementation detail. Locking rank + * 0 while rank 1 is still running leaves rank 1 free to post a collective that + * rank 0 will never service, and rank 1's own lock then waits on an operation + * that cannot complete. Locking all ranks first makes that window empty. + * + * On any lock failure every already-locked pid is unlocked again, so a failed + * attempt leaves the job running rather than wedged half-locked. + */ +static int do_save_all(unsigned int timeout_ms) +{ + unsigned int locked = 0; + for (; locked < g_pid_count; locked++) { + if (do_lock(g_pids[locked], timeout_ms) != 0) { + fprintf(stderr, "save: lock failed on pid %d (%u/%u locked); rolling back\n", + g_pids[locked], locked, g_pid_count); + for (unsigned int j = 0; j < locked; j++) + (void)do_unlock(g_pids[j]); + return 1; + } + } + for (unsigned int i = 0; i < g_pid_count; i++) { + if (do_checkpoint(g_pids[i]) != 0) { + fprintf(stderr, "save: checkpoint failed on pid %d (%u/%u checkpointed)\n", + g_pids[i], i, g_pid_count); + return 1; + } + } + return 0; +} + /* toggle: running -> (lock, checkpoint); checkpointed -> (restore, unlock) */ static int do_toggle(int pid, unsigned int timeout_ms) { @@ -141,25 +363,35 @@ static void usage(const char *p) "Operations:\n" " --get-state --pid \n" " --action lock|checkpoint|restore|unlock|resume --pid [--timeout ]\n" + " --action save --pid --pid ... (lock all, then checkpoint all)\n" " --toggle --pid \n" " --get-restore-tid --pid \n" "Options:\n" - " --pid|-p target pid\n" + " --pid|-p target pid; repeatable, or comma-separated.\n" + " Several pids are driven from this one process,\n" + " which is what 'save' needs to lock every rank\n" + " before any of them is checkpointed.\n" " --timeout|-t lock timeout in milliseconds (0 = no timeout)\n" + " --gpu-map GPU migration on restore (driver r580+).\n" + " = :[,:...]\n" + " each side is a device index or a UUID\n" + " (32 hex digits, optional GPU- prefix/dashes).\n" + " Every visible GPU must appear; use i:i to pin.\n" " --help|-h\n"); } int main(int argc, char **argv) { - int pid = -1; unsigned int timeout_ms = 0; const char *action = NULL; + const char *gpu_map = NULL; int get_state = 0, toggle = 0, get_tid = 0; static struct option opts[] = { {"action", required_argument, 0, 'a'}, {"pid", required_argument, 0, 'p'}, {"timeout", required_argument, 0, 't'}, + {"gpu-map", required_argument, 0, 'm'}, {"get-state", no_argument, 0, 's'}, {"toggle", no_argument, 0, 'g'}, {"get-restore-tid",no_argument, 0, 'r'}, @@ -167,11 +399,12 @@ int main(int argc, char **argv) {0,0,0,0} }; int c; - while ((c = getopt_long(argc, argv, "a:p:t:sgrh", opts, NULL)) != -1) { + while ((c = getopt_long(argc, argv, "a:p:t:m:sgrh", opts, NULL)) != -1) { switch (c) { case 'a': action = optarg; break; - case 'p': pid = atoi(optarg); break; + case 'p': if (add_pids(optarg) < 0) return 2; break; case 't': timeout_ms = (unsigned int)strtoul(optarg, NULL, 10); break; + case 'm': gpu_map = optarg; break; case 's': get_state = 1; break; case 'g': toggle = 1; break; case 'r': get_tid = 1; break; @@ -180,7 +413,7 @@ int main(int argc, char **argv) } } - if (pid <= 0) { + if (g_pid_count == 0) { fprintf(stderr, "error: --pid is required\n"); usage(argv[0]); return 2; @@ -194,20 +427,55 @@ int main(int argc, char **argv) return 1; } - if (get_state) return do_get_state(pid); - if (get_tid) return do_get_restore_tid(pid); - if (toggle) return do_toggle(pid, timeout_ms); + /* Parsed after cuInit: index forms and the visible-GPU count both need the + * driver up. Rejecting a bad map here, before any state transition, keeps a + * typo from leaving the target process locked or half-restored. */ + if (gpu_map && parse_gpu_map(gpu_map) < 0) + return 2; + + /* One pid: identical to before, including exit codes, so the CRIU plugin + * is unaffected. Several: apply in the given order and fail on the first + * error, except for "save" which owns its own ordering and rollback. */ + if (get_state) { + for (unsigned int i = 0; i < g_pid_count; i++) + if (do_get_state(g_pids[i]) != 0) return 1; + return 0; + } + if (get_tid) { + for (unsigned int i = 0; i < g_pid_count; i++) + if (do_get_restore_tid(g_pids[i]) != 0) return 1; + return 0; + } + if (toggle) { + for (unsigned int i = 0; i < g_pid_count; i++) + if (do_toggle(g_pids[i], timeout_ms) != 0) return 1; + return 0; + } if (!action) { fprintf(stderr, "error: one of --action/--get-state/--toggle/--get-restore-tid required\n"); usage(argv[0]); return 2; } - if (!strcmp(action, "lock")) return do_lock(pid, timeout_ms); - if (!strcmp(action, "checkpoint")) return do_checkpoint(pid); - if (!strcmp(action, "restore")) return do_restore(pid); - if (!strcmp(action, "unlock")) return do_unlock(pid); - if (!strcmp(action, "resume")) return do_resume(pid); + /* Say so rather than silently ignoring it: a map on the capture half is + * almost always someone expecting migration to be chosen at checkpoint + * time, and finding out at restore is far more expensive. */ + if (g_pairs_count && (!strcmp(action, "lock") || !strcmp(action, "checkpoint"))) + fprintf(stderr, "warning: --gpu-map has no effect on '%s'; it applies at restore\n", action); + + if (!strcmp(action, "save")) return do_save_all(timeout_ms); + + for (unsigned int i = 0; i < g_pid_count; i++) { + int rc; + if (!strcmp(action, "lock")) rc = do_lock(g_pids[i], timeout_ms); + else if (!strcmp(action, "checkpoint")) rc = do_checkpoint(g_pids[i]); + else if (!strcmp(action, "restore")) rc = do_restore(g_pids[i]); + else if (!strcmp(action, "unlock")) rc = do_unlock(g_pids[i]); + else if (!strcmp(action, "resume")) rc = do_resume(g_pids[i]); + else break; + if (rc) return rc; + if (i + 1 == g_pid_count) return 0; + } fprintf(stderr, "error: unknown action '%s'\n", action); usage(argv[0]); diff --git a/src/compute-plane-services/nvsnap/docker/agent/tests/gpu_map_test.c b/src/compute-plane-services/nvsnap/docker/agent/tests/gpu_map_test.c new file mode 100644 index 0000000000..cd87210983 --- /dev/null +++ b/src/compute-plane-services/nvsnap/docker/agent/tests/gpu_map_test.c @@ -0,0 +1,162 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Unit tests for nvsnap-cuda-checkpoint's --gpu-map parsing. + * + * The parsing decides which physical GPU a restore lands on. Getting it wrong + * does not fail loudly -- it either rejects a valid map (restore blocked) or, + * worse, builds a plausible-looking map with transposed UUIDs and migrates the + * process onto the wrong device. Neither is visible without a GPU, so it is + * tested here against stubs instead of only on a live node. + * + * Builds the real translation unit with main() renamed, so the tests exercise + * the shipped code rather than a copy of it. + * + * Run: docker/agent/tests/run_gpu_map_test.sh + */ +#define main nvsnap_cuda_checkpoint_main +#include "../nvsnap-cuda-checkpoint.c" +#undef main + +#include + +/* ---- CUDA stubs --------------------------------------------------------- + * Eight fake devices whose UUIDs are byte i repeated, so a mis-mapped pair is + * obvious in a failure message rather than an opaque hex diff. + */ +#define FAKE_DEVICES 8 +static int g_visible = FAKE_DEVICES; + +CUresult cuGetErrorString(CUresult r, const char **s) { (void)r; *s = "stub"; return CUDA_SUCCESS; } +CUresult cuInit(unsigned int f) { (void)f; return CUDA_SUCCESS; } + +CUresult cuDeviceGetCount(int *n) { *n = g_visible; return CUDA_SUCCESS; } + +CUresult cuDeviceGet(CUdevice *d, int ordinal) +{ + if (ordinal < 0 || ordinal >= g_visible) return CUDA_ERROR_INVALID_DEVICE; + *d = ordinal; + return CUDA_SUCCESS; +} + +CUresult cuDeviceGetUuid(CUuuid *u, CUdevice d) +{ + if (d < 0 || d >= g_visible) return CUDA_ERROR_INVALID_DEVICE; + memset(u->bytes, (char)(0xA0 + d), 16); + return CUDA_SUCCESS; +} + +/* Unused by these tests, but the translation unit references them. */ +CUresult cuCheckpointProcessLock(int p, CUcheckpointLockArgs *a) { (void)p; (void)a; return CUDA_SUCCESS; } +CUresult cuCheckpointProcessCheckpoint(int p, CUcheckpointCheckpointArgs *a) { (void)p; (void)a; return CUDA_SUCCESS; } +CUresult cuCheckpointProcessRestore(int p, CUcheckpointRestoreArgs *a) { (void)p; (void)a; return CUDA_SUCCESS; } +CUresult cuCheckpointProcessUnlock(int p, CUcheckpointUnlockArgs *a) { (void)p; (void)a; return CUDA_SUCCESS; } +CUresult cuCheckpointProcessGetState(int p, CUprocessState *s) { (void)p; *s = CU_PROCESS_STATE_RUNNING; return CUDA_SUCCESS; } +CUresult cuCheckpointProcessGetRestoreThreadId(int p, int *t) { (void)p; *t = 1; return CUDA_SUCCESS; } + +/* ---- harness ---------------------------------------------------------- */ +static int passed, failed; + +static void reset(void) +{ + free(g_pairs); + g_pairs = NULL; + g_pairs_count = 0; + g_visible = FAKE_DEVICES; +} + +static void ok(const char *name, int cond) +{ + if (cond) { passed++; printf(" ok %s\n", name); } + else { failed++; printf(" FAIL %s\n", name); } +} + +static int uuid_is(const CUuuid *u, unsigned char want) +{ + for (int i = 0; i < 16; i++) + if ((unsigned char)u->bytes[i] != want) return 0; + return 1; +} + +/* Silence the parser's diagnostics for the cases that are meant to fail. */ +static void quiet(void) +{ + if (!freopen("/dev/null", "w", stderr)) + perror("freopen"); +} + +int main(void) +{ + printf("gpu_map parsing\n"); + + /* Index form: the convenience path for same-node testing. Resolving to + * UUIDs here is what stops an index leaking through to the driver, where + * it would mean a different device on the restore node. */ + reset(); + ok("accepts an all-index map", parse_gpu_map("0:1,1:2,2:3,3:4,4:5,5:6,6:7,7:0") == 0); + ok("resolves index to UUID (old)", uuid_is(&g_pairs[0].oldUuid, 0xA0)); + ok("resolves index to UUID (new)", uuid_is(&g_pairs[0].newUuid, 0xA1)); + ok("keeps pair order", g_pairs_count == 8 && uuid_is(&g_pairs[7].newUuid, 0xA0)); + + /* Identity is the case that pins every GPU in place; it must be expressible, + * because a partial map is rejected by the driver. */ + reset(); + ok("accepts identity map", parse_gpu_map("0:0,1:1,2:2,3:3,4:4,5:5,6:6,7:7") == 0); + + /* UUID forms, as emitted by nvidia-smi and the k8s device plugin. */ + reset(); + g_visible = 1; + ok("accepts GPU- prefixed dashed UUID", + parse_gpu_map("GPU-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa:" + "GPU-bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb") == 0); + ok("parses prefixed UUID bytes", uuid_is(&g_pairs[0].oldUuid, 0xAA)); + + reset(); + g_visible = 1; + ok("accepts bare 32-hex UUID", + parse_gpu_map("cccccccccccccccccccccccccccccccc:dddddddddddddddddddddddddddddddd") == 0); + ok("parses bare UUID bytes", uuid_is(&g_pairs[0].newUuid, 0xDD)); + + reset(); + g_visible = 1; + ok("accepts uppercase hex", + parse_gpu_map("EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF") == 0); + ok("uppercase parses to same bytes as lowercase", uuid_is(&g_pairs[0].oldUuid, 0xEE)); + + quiet(); + + /* Every visible GPU must be mapped. A short map is the easy mistake, and + * the driver's own error for it is far less specific than ours. */ + reset(); + ok("rejects a map shorter than the visible GPU count", parse_gpu_map("0:1") != 0); + + reset(); + g_visible = 2; + ok("rejects a map longer than the visible GPU count", parse_gpu_map("0:1,1:0,0:1") != 0); + + /* Malformed input must be refused before any state transition. */ + reset(); + g_visible = 1; + ok("rejects a pair with no colon", parse_gpu_map("0") != 0); + + reset(); + g_visible = 1; + ok("rejects a short UUID", parse_gpu_map("abcd:0") != 0); + + reset(); + g_visible = 1; + ok("rejects non-hex characters", parse_gpu_map( + "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz:0") != 0); + + reset(); + g_visible = 1; + ok("rejects an out-of-range device index", parse_gpu_map("99:0") != 0); + + reset(); + g_visible = 1; + ok("rejects an empty side", parse_gpu_map(":0") != 0); + + printf("\n%d passed, %d failed\n", passed, failed); + return failed ? 1 : 0; +} diff --git a/src/compute-plane-services/nvsnap/docker/agent/tests/run_gpu_map_test.sh b/src/compute-plane-services/nvsnap/docker/agent/tests/run_gpu_map_test.sh new file mode 100755 index 0000000000..ce77836c62 --- /dev/null +++ b/src/compute-plane-services/nvsnap/docker/agent/tests/run_gpu_map_test.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Builds and runs the --gpu-map parsing tests. +# +# Needs CUDA 13 headers for the r580 migration types (CUcheckpointGpuPair, +# CUcheckpointRestoreArgs.gpuPairs), which 12.x does not declare. It does NOT +# need a GPU or a driver: the CUDA entry points are stubbed in the test, and +# nothing is linked against libcuda. +# +# Uses the same image as Dockerfile.base's cuda-cli-builder stage so the tests +# compile against exactly the headers the shipped binary is built with. + +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +IMAGE="${CUDA_BUILD_IMAGE:-nvidia/cuda:13.0.3-devel-ubuntu22.04}" + +if command -v gcc >/dev/null 2>&1 && [ -f /usr/local/cuda/include/cuda.h ] && \ + grep -q "gpuPairs" /usr/local/cuda/include/cuda.h 2>/dev/null; then + echo "Building locally (CUDA headers with gpuPairs found)" + gcc -O2 -Wall -o /tmp/nvsnap-gpu-map-test "$HERE/gpu_map_test.c" \ + -I/usr/local/cuda/include + exec /tmp/nvsnap-gpu-map-test +fi + +echo "Building in $IMAGE" +exec docker run --rm -v "$HERE/..:/src:ro" "$IMAGE" bash -c ' + gcc -O2 -Wall -o /tmp/t /src/tests/gpu_map_test.c -I/usr/local/cuda/include && exec /tmp/t +' diff --git a/src/compute-plane-services/nvsnap/docker/agent/tests/run_save_order_test.sh b/src/compute-plane-services/nvsnap/docker/agent/tests/run_save_order_test.sh new file mode 100755 index 0000000000..5e2cb77282 --- /dev/null +++ b/src/compute-plane-services/nvsnap/docker/agent/tests/run_save_order_test.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Builds and runs the --action save ordering tests. +# +# Needs CUDA 13 headers for the r580 migration types (CUcheckpointGpuPair, +# CUcheckpointRestoreArgs.gpuPairs), which 12.x does not declare. It does NOT +# need a GPU or a driver: the CUDA entry points are stubbed in the test, and +# nothing is linked against libcuda. +# +# Uses the same image as Dockerfile.base's cuda-cli-builder stage so the tests +# compile against exactly the headers the shipped binary is built with. + +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +IMAGE="${CUDA_BUILD_IMAGE:-nvidia/cuda:13.0.3-devel-ubuntu22.04}" + +if command -v gcc >/dev/null 2>&1 && [ -f /usr/local/cuda/include/cuda.h ] && \ + grep -q "gpuPairs" /usr/local/cuda/include/cuda.h 2>/dev/null; then + echo "Building locally (CUDA headers with gpuPairs found)" + gcc -O2 -Wall -o /tmp/nvsnap-save-order-test "$HERE/save_order_test.c" \ + -I/usr/local/cuda/include + exec /tmp/nvsnap-save-order-test +fi + +echo "Building in $IMAGE" +exec docker run --rm -v "$HERE/..:/src:ro" "$IMAGE" bash -c ' + gcc -O2 -Wall -o /tmp/t /src/tests/save_order_test.c -I/usr/local/cuda/include && exec /tmp/t +' diff --git a/src/compute-plane-services/nvsnap/docker/agent/tests/save_order_test.c b/src/compute-plane-services/nvsnap/docker/agent/tests/save_order_test.c new file mode 100644 index 0000000000..14f7e6c4db --- /dev/null +++ b/src/compute-plane-services/nvsnap/docker/agent/tests/save_order_test.c @@ -0,0 +1,164 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Tests the ordering of `--action save` across several pids. + * + * The order is the whole hypothesis. Locking rank 0 while rank 1 still runs + * leaves rank 1 free to post a collective that rank 0 will never service, and + * rank 1's own lock then waits on an operation that cannot complete -- the + * shape of the "hangs on 2nd rank (lock timeout)" failure recorded when + * multi-GPU capture was last attempted. If save ever degrades to + * lock+checkpoint per pid it reintroduces exactly that window, and it would do + * so silently: every call still succeeds on a single-GPU workload. + * + * Stubs record the call sequence, so no GPU or driver is involved. + * + * Run: docker/agent/tests/run_save_order_test.sh + */ +#define main nvsnap_cuda_checkpoint_main +#include "../nvsnap-cuda-checkpoint.c" +#undef main + +/* ---- recording stubs --------------------------------------------------- */ +#define MAX_CALLS 64 +static char g_calls[MAX_CALLS][32]; +static int g_ncalls; + +/* pid whose lock or checkpoint should fail, 0 for none */ +static int g_fail_lock_pid; +static int g_fail_ckpt_pid; + +static void record(const char *verb, int pid) +{ + if (g_ncalls < MAX_CALLS) + snprintf(g_calls[g_ncalls++], 32, "%s(%d)", verb, pid); +} + +CUresult cuGetErrorString(CUresult r, const char **s) { (void)r; *s = "stub"; return CUDA_SUCCESS; } +CUresult cuInit(unsigned int f) { (void)f; return CUDA_SUCCESS; } +CUresult cuDeviceGetCount(int *n) { *n = 0; return CUDA_SUCCESS; } +CUresult cuDeviceGet(CUdevice *d, int o) { (void)d; (void)o; return CUDA_ERROR_INVALID_DEVICE; } +CUresult cuDeviceGetUuid(CUuuid *u, CUdevice d) { (void)u; (void)d; return CUDA_ERROR_INVALID_DEVICE; } + +CUresult cuCheckpointProcessLock(int pid, CUcheckpointLockArgs *a) +{ + (void)a; + record("lock", pid); + return pid == g_fail_lock_pid ? CUDA_ERROR_NOT_READY : CUDA_SUCCESS; +} + +CUresult cuCheckpointProcessCheckpoint(int pid, CUcheckpointCheckpointArgs *a) +{ + (void)a; + record("ckpt", pid); + return pid == g_fail_ckpt_pid ? CUDA_ERROR_NOT_READY : CUDA_SUCCESS; +} + +CUresult cuCheckpointProcessUnlock(int pid, CUcheckpointUnlockArgs *a) +{ + (void)a; + record("unlock", pid); + return CUDA_SUCCESS; +} + +CUresult cuCheckpointProcessRestore(int pid, CUcheckpointRestoreArgs *a) { (void)a; record("restore", pid); return CUDA_SUCCESS; } +CUresult cuCheckpointProcessGetState(int p, CUprocessState *s) { (void)p; *s = CU_PROCESS_STATE_RUNNING; return CUDA_SUCCESS; } +CUresult cuCheckpointProcessGetRestoreThreadId(int p, int *t) { (void)p; *t = 1; return CUDA_SUCCESS; } + +/* ---- harness ---------------------------------------------------------- */ +static int passed, failed; + +static void ok(const char *name, int cond) +{ + if (cond) { passed++; printf(" ok %s\n", name); } + else { failed++; printf(" FAIL %s\n", name); } +} + +static void reset(int fail_lock, int fail_ckpt) +{ + g_ncalls = 0; + g_pid_count = 0; + g_fail_lock_pid = fail_lock; + g_fail_ckpt_pid = fail_ckpt; +} + +static const char *seq(void) +{ + static char buf[MAX_CALLS * 32]; + buf[0] = '\0'; + for (int i = 0; i < g_ncalls; i++) { + if (i) strcat(buf, " "); + strcat(buf, g_calls[i]); + } + return buf; +} + +static void expect_seq(const char *name, const char *want) +{ + const char *got = seq(); + int cond = !strcmp(got, want); + ok(name, cond); + if (!cond) + printf(" want: %s\n got: %s\n", want, got); +} + +int main(void) +{ + printf("save ordering\n"); + + /* The load-bearing assertion: every rank locked before any is + * checkpointed, not lock+checkpoint per rank. */ + reset(0, 0); + add_pids("101,102,103"); + ok("save succeeds with three ranks", do_save_all(0) == 0); + expect_seq("locks all ranks before checkpointing any", + "lock(101) lock(102) lock(103) ckpt(101) ckpt(102) ckpt(103)"); + + /* A failed attempt must leave the job running, not half-locked: a rank + * left LOCKED is a hung workload, which is worse than a failed capture. */ + reset(102, 0); + add_pids("101,102,103"); + ok("save fails when a lock fails", do_save_all(0) != 0); + expect_seq("rolls back already-locked ranks and checkpoints nothing", + "lock(101) lock(102) unlock(101)"); + + /* Failure on the first rank: nothing was locked, so nothing to undo. */ + reset(101, 0); + add_pids("101,102"); + ok("save fails when the first lock fails", do_save_all(0) != 0); + expect_seq("no rollback needed when the first lock fails", "lock(101)"); + + /* Checkpoint failure stops immediately rather than continuing through the + * remaining ranks, whose images would belong to a capture that cannot + * complete anyway. Locks are deliberately left in place: the ranks already + * checkpointed are recoverable only by restoring them, which the caller + * drives. */ + reset(0, 102); + add_pids("101,102,103"); + ok("save fails when a checkpoint fails", do_save_all(0) != 0); + expect_seq("stops at the failing checkpoint", + "lock(101) lock(102) lock(103) ckpt(101) ckpt(102)"); + + /* Single pid must behave exactly as the plugin's existing calls expect. */ + reset(0, 0); + add_pids("101"); + ok("single pid still succeeds", do_save_all(0) == 0); + expect_seq("single pid is lock then checkpoint", "lock(101) ckpt(101)"); + + /* pid list parsing: repeated flags and comma forms must be equivalent. */ + reset(0, 0); + add_pids("7"); + add_pids("8,9"); + ok("accumulates repeated and comma-separated pids", + g_pid_count == 3 && g_pids[0] == 7 && g_pids[1] == 8 && g_pids[2] == 9); + + reset(0, 0); + ok("rejects a non-numeric pid", add_pids("abc") != 0); + + reset(0, 0); + ok("rejects a negative pid", add_pids("-5") != 0); + + printf("\n%d passed, %d failed\n", passed, failed); + return failed ? 1 : 0; +} diff --git a/src/compute-plane-services/nvsnap/docs/proposals/multi-gpu-criu-v2.md b/src/compute-plane-services/nvsnap/docs/proposals/multi-gpu-criu-v2.md new file mode 100644 index 0000000000..d0af420c9b --- /dev/null +++ b/src/compute-plane-services/nvsnap/docs/proposals/multi-gpu-criu-v2.md @@ -0,0 +1,288 @@ + +# Multi-GPU checkpoint/restore on criu-v2 + +Status: working under a config constraint, measured 2026-08-23. Not enabled by +default. The constraint is the finding, not a detail. + +Multi-GPU CRIU was previously refused outright in the agent, with the reasoning +recorded in the code: cuda-checkpoint blocks on peer state, the D2H path could +never reconstruct CUDA context state on restore, so multi-GPU had to use the +cachedir path. The first half of that is true and remains true. The +conclusion drawn from it was too strong: with peer state absent, criu-v2 does +capture and restore a TP=2 workload, weights, KV cache and all. + +## What works + +Measured on 8x H100 80GB (p5.48xlarge, NVSwitch), driver 580.126.16, agent +v0.2.65 with `NVSNAP_MULTI_GPU_CRIU=1`. + +```text +workload engine checkpoint restore result +TinyLlama TP=2 vLLM 2m34s 56G 1m00s PASS (x3) +TinyLlama TP=4 vLLM 4m16s 110G 1m14s PASS +Llama-3.1-70B TP=4 vLLM 9m56s 290G 2m02s PASS +Qwen3-32B TP=2 (NIM) TRT-LLM 3m03s 103G 1m55s PASS +``` + +The 70B run is the production-shaped case: four ranks, 76.5G of GPU state each, +and the restored pod answered a completion correctly. + +Restore scales far better than size. 5x the data between TP=2 TinyLlama and 70B +TP=4 costs 2x the time, and the 70B restore moved 290G in 122s, about 2.4 GB/s. +Single-GPU restore measures nearer 0.9 GB/s. The per-rank GPU restores are +therefore overlapping rather than serialising, which contradicts the premise +behind the deferred per-pid parallelisation work and is worth re-examining +before anyone invests there. + +The engine is criu-v2 plus cuda-checkpoint. No interception library, no patched +libzmq/libuv, no D2H save. The same path single-GPU uses. + +The captures are complete, which matters because an incomplete one looks very +similar: a partial capture still exits zero and still produces a well-formed +checkpoint, just a much smaller one. + +```text +TinyLlama TP=2 5 tasks, cuda_plugin paused 4 pids, 2 x 28.4G images +TinyLlama TP=4 7 tasks, cuda_plugin paused 6 pids, 4 x 28.4G images +70B TP=4 7 tasks, cuda_plugin paused 6 pids, 4 x 76.5G images +``` + +Every Worker_TP rank appears in the dumped tree, there is one GPU image per +rank, and each image matches the per-GPU memory budget. The agent's own capture +guard reported "all GPU processes present in the capture", and every restored +pod served live inference. + +## Capture happens on an idle pod, and that is the operating model + +Every measurement here captures an engine that has just served a request and is +now idle, because that is what NVCF does: a pod is checkpointed when it has no +traffic, and every later pod restores from that same checkpoint. Capture once, +restore many. + +This is worth stating because capture under live traffic behaves differently and +someone will otherwise rediscover it and file it as a bug. With eight concurrent +requests in flight against a TP=2 pod, rank 0 locks in about 3 seconds and rank 1 +then times out after 60 with "device not ready". It is a deadlock rather than +slowness: locking rank 0 stops it servicing the collective rank 1 is blocked on, +so rank 1's own lock waits on work that can never finish. Locking the ranks +simultaneously does not help, because the first lock is what creates the +condition. + +None of that applies to the NVCF flow. It would matter for checkpointing a live +serving pod, which would need the engine's scheduler drained first. + +The same model changes which numbers matter. Capture cost is amortised over +every pod that restores from the artifact, so the 70B's ten minutes is paid once. +Restore is the hot path, paid per pod, which makes the sub-linear restore scaling +more valuable here than the capture time is expensive. + +## The configuration it requires + +Every cross-GPU transport must be off before capture: + +```yaml +--enforce-eager +--disable-custom-all-reduce +NCCL_P2P_DISABLE=1 +NCCL_NVLS_ENABLE=0 +NCCL_SHM_DISABLE=1 +VLLM_ALLREDUCE_USE_SYMM_MEM=0 +``` + +This set previously carried `NCCL_NVLS_DISABLE=1`, which is not an NCCL +variable at all. Grepping the shipped `libnccl.so.2` finds `NCCL_P2P_DISABLE`, +`NCCL_SHM_DISABLE`, `NCCL_CUMEM_ENABLE` and `NCCL_NVLS_ENABLE`, but no +`NCCL_NVLS_DISABLE`. It had been a no-op for as long as it had been set, which +is why the bisect below could never attribute anything to it. The real knob is +`NCCL_NVLS_ENABLE=0`, and the capture path is verified against that spelling. + +## The set is not padding: bisect results + +This bundle was assembled during an earlier investigation and had never been +reduced. It was worth checking whether one flag was doing the work. It is not. + +```text +removed result +--enforce-eager capture OK (59G), restore OK, models OK, + INFERENCE FAILS after 2m13s +NCCL_P2P_DISABLE capture hangs, 10m11s +NCCL_NVLS + SHM + SYMM_MEM capture hangs, 10m07s +NCCL_SHM + SYMM_MEM capture hangs, 10m06s +``` + +Three of four removals broke it. Not yet isolated individually: +`--disable-custom-all-reduce`, and NVLS versus SHM versus SYMM_MEM separately. + +The mechanism explains the shape. NCCL reaches a peer through several +independent transports (NVLink P2P, shared memory, NVLS multicast) and vLLM adds +its own (custom all-reduce, symmetric memory). Each creates cross-GPU +state, and the checkpoint blocks unless all of them are closed. Closing one door +leaves the others open. For tensor parallel that means all-reduce over sockets. + +Correction, measured 2026-08-24: "no peer mappings at all" is too strong and was +wrong. A passing vLLM rank still holds mappings to the peer GPU's device node +(3 to its own /dev/nvidia0, 2 to /dev/nvidia1) and captures cleanly regardless. +Whatever the flags remove, it is not visible as device mappings. The precise +mechanism is not established. + +## The CUDA graph failure is a different kind of problem + +Dropping `--enforce-eager` fails in a way worth separating from the rest, +because it fails late and quietly. Capture succeeds, the checkpoint is larger +(graphs reserve memory), restore succeeds, and the models endpoint answers. The +first actual inference then hangs. + +A captured graph holds references to GPU-side resources that the checkpoint +destroys and rebuilds, so replaying it after restore drives dead handles. +Nothing in the capture path can detect this; only the framework that built the +graph can rebuild it. + +That distinguishes this flag from the others. The peer-transport flags need a +mechanism nobody has yet. This one has a known owner and a known fix: have the +engine re-capture its graphs after restore. Until then `--enforce-eager` is a +placeholder for that hook, not a permanent tax. + +Note also that aborting NCCL communicators does not help here and cannot. NCCL +has no record of which graphs captured its kernels, so an abort frees the +resources and leaves the graphs dangling rather than cleaning them. + +## SGLang: the recipe does not transfer + +Tested 2026-08-24. SGLang TP=2 (Llama-3.1-8B, `--mem-fraction-static 0.6`) with +what should be the equivalent sever set: `--disable-cuda-graph` in place of +`--enforce-eager`, plus `--disable-custom-all-reduce` and the same three +`NCCL_*_DISABLE` env vars, which are engine-independent. + +The capture hangs. Not slowly: a `criu` and a `cuda-checkpoint` were still +wedged three hours later, both blocked in `anon_pipe_read`, CRIU waiting on a +cuda-checkpoint that never returns. + +Confirmed twice. The first manifest carried three corrupted env values +(`NVSNAP_SECCOMP_ENABLED`, `USE_LIBUV` and `HF_HUB_ENABLE_HF_TRANSFER` were all +set to "0,1" instead of "0", collateral from a value-scoped rather than +key-scoped edit). `USE_LIBUV` in particular is deliberately "0" on the +single-GPU manifest, so that run could not be trusted. Repeated with the values +corrected and a 30 minute timeout: identical hang, identical wedged-process +signature. The confound was real but was not the cause. A retry fails earlier with +`text file busy` because the wedged process still holds the binary, which is a +useful tell that the first attempt is stuck rather than finished. + +Two hypotheses were tested and both died: + +- Device fds. SGLang processes hold fds on all eight GPUs despite + CUDA_VISIBLE_DEVICES=0,1. So does vLLM, with an identical per-device count + (23/7 on the assigned pair, 3 on each other GPU). Not the discriminator. +- Peer device mappings. A passing vLLM rank has 2 mappings to the peer GPU's + node; so does SGLang. Not the discriminator either. + +The only measured difference is that SGLang holds ~40 /dev/nvidiactl mappings +per rank against vLLM's ~28. That is a lead, not a cause. + +Flag coverage was also verified rather than assumed: --disable-cuda-graph and +--disable-custom-all-reduce both exist and were accepted, and SGLang's peer +features (--enable-nccl-nvls, --enable-symm-mem, --enable-torch-symm-mem, +--enable-p2p-check) are all opt-in and were never enabled, so the NCCL env vars +were suppressing features that were already inactive. Note also +--disable-piecewise-cuda-graph and --disable-decode-cuda-graph exist and were +NOT set; they are unlikely to matter for a capture hang, since graphs fail late +at restore-inference rather than at capture, but they are untested. + +The honest state: the vLLM recipe does not transfer, and why is not known. Do +not assume it generalises to another engine without measuring. + +Worth noting for whoever picks this up: a wedged capture leaves host processes +behind that block subsequent attempts on that node. Check for `criu` and +`cuda-checkpoint` in `/host/proc` and kill them before re-running. + +NIM does work, which makes the SGLang failure engine-specific rather than a +property of anything but vLLM. See below. + +## NIM works, and needed three fixes that generalise + +Qwen3-32B TP=2 on TRT-LLM captures and restores. Its process shape differs from +vLLM's: `start_server.sh`, an `orted` MPI daemon, and four python3 ranks, with +asymmetric GPU images (83.5G and 24.7G) rather than vLLM's even per-rank split. + +Three things had to be fixed, and each was a general defect rather than a NIM +quirk: + +1. The stock image has no command, so it runs + `/opt/nvidia/nvidia_entrypoint.sh` with cmd `bash -c $SERVER_START_SCRIPT_PATH`. + criu-v2 needs the setsid convention, so the manifest reproduces the image's + own startup inside it rather than replacing it. + +2. Stdio cannot go to `/tmp`. `isRuntimeGeneratedPath` treats `/tmp` as + runtime-generated and drops it from the rootfs diff, so the placeholder + restores an empty file and CRIU refuses: + `File tmp/nim.out has bad size 0 (expect 21443)`. It also cannot go to the + container root, because the image runs as uid 1000. `/opt/nim` satisfies both. + +3. The placeholder must run as root. It writes + `/proc/sys/kernel/ns_last_pid`, and privileged does not confer root, so an + image defaulting to a non-root uid silently fails that write and the restore + dies with `Can't fork for 336: File exists`. The generator now emits + `runAsUser: 0` for every placeholder, and the bump's failure message is loud + rather than parenthetical. The restored workload's own uid comes from the + checkpoint, so this does not change what it runs as. + +Point 3 was previously known only as a NIM-specific note on one hand-written +manifest. It is a property of any non-root image, and belonged in the generator. + +## What is still open + +Removing the config constraint needs peer mappings gone at capture time without +the workload being launched to avoid them. Two candidate routes: + +1. Transparent sever: tear down peer mappings before the dump and re-establish + them after. The pieces exist in the intercept library + (`nvsnap_gpu_pre_checkpoint` disables peer access, `nvsnap_gpu_post_restore` + re-enables it) but driving them requires the LD_PRELOAD stack that criu-v2 + deliberately does not use, and an attempt to run the older quiesce path + against a live vLLM engine took the executor down with it. +2. Driver support for checkpointing the mappings themselves. NVIDIA publishes an + example covering checkpoint and restore of IPC memory handles that requires + display driver 610 or higher, which is the class vLLM's custom all-reduce + uses. Worth measuring on 610 before building anything. + +## Reproducing + +```sh +# agent: NVSNAP_MULTI_GPU_CRIU=1 lifts the refusal and keeps the ordinary +# criu-v2 engine. NVSNAP_LEGACY_MULTI_GPU_D2H=1 selects the old quiesce+D2H +# path instead; that one's restore half has never worked. +helm upgrade nvsnap deploy/helm/nvsnap -n nvsnap-system -f + +CAPTURE_PATH=criu-v2 ./scripts/test-e2e.sh vllm-tp2-criu + +# 70B needs a warm model cache and a raised checkpoint timeout: +CAPTURE_PATH=criu-v2 CHECKPOINT_TIMEOUT=2400 ./scripts/test-e2e.sh vllm-70b-criu +``` + +`vllm-tp2-criu` and `vllm-70b-criu` are separate manifests from `vllm-tp2` and +`vllm-70b`, which stay on the cachedir path; the two engines no longer +share a file. The restore placeholder is generated, not hand-written: the source +carries `nvsnap.io/path: "criu"` so the generator derives a criu-v2 placeholder +with the checkpoint hostPath mounted. Regenerate with +`go run ./internal/manifests/gen -dir deploy/k8s/workloads`. Before that +annotation was set, the manifest was a cachedir/webhook target and restore failed +with "checkpoint images not visible at /checkpoints inside placeholder". + +## Scope + +Two topologies (TP=2, TP=4) and two model scales (1.1B, 70B) on H100, same-node +restore, vLLM only. SGLang was tested and hangs (above). NIM/TRT-LLM untested. +Also untested: cross-node restore and TP=8. + +Operational note: a cold 70B run does not fit the harness. test-e2e.sh pins +POD_READY_TIMEOUT to 1800s for any workload matching *70b*, which a 140G model +pull exceeds, and the checkpoint step needs CHECKPOINT_TIMEOUT well above its +600s default (the capture alone took 596s). Both are harness constants, not +mechanism limits. + +Do not read this as "multi-GPU works". Read it as "multi-GPU capture and restore +work on criu-v2 when no peer mappings exist, and that condition currently has to +be arranged by configuration". diff --git a/src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel b/src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel index 4fcfefc3e0..809b0bd8a1 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel +++ b/src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel @@ -88,6 +88,7 @@ go_test( "cascade_fetch_test.go", "catalog_test.go", "checkpoint_plan_a_test.go", + "checkpoint_v2_gpuguard_test.go", "fsstore_test.go", "l2_integration_test.go", "l2_promote_async_test.go", diff --git a/src/compute-plane-services/nvsnap/internal/agent/checkpoint.go b/src/compute-plane-services/nvsnap/internal/agent/checkpoint.go index 54776576ae..0768d85bd7 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/checkpoint.go +++ b/src/compute-plane-services/nvsnap/internal/agent/checkpoint.go @@ -210,11 +210,22 @@ func countDistinctGPUDevices(pids []int, log *logrus.Entry) int { "--query-compute-apps=pid,gpu_bus_id", "--format=csv,noheader") // nvidia-smi needs the driver's shared libraries + // This container's libdir FIRST. The driver tree ships its own libc.so.6, + // and putting it ahead of ours makes the loader mix it with this image's + // ld-linux, which aborts with "undefined symbol: __tunable_is_initialized". + // nvidia-smi then never runs, the query "fails", and every workload is + // reported as single-GPU. Same ordering rule as the cuda-checkpoint wrapper. cmd.Env = append(os.Environ(), - "LD_LIBRARY_PATH=/host/run/nvidia/driver/usr/lib/x86_64-linux-gnu:/usr/local/nvidia/lib64") + "LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/host/run/nvidia/driver/usr/lib/x86_64-linux-gnu:/usr/local/nvidia/lib64") out, err := cmd.Output() if err != nil { - log.WithError(err).WithField("path", nvidiaSmi).Warn("nvidia-smi query failed, assuming single GPU") + // Fail-open is deliberate but dangerous: a detection failure silently + // classifies a multi-GPU workload as single-GPU, which both skips the + // multi-GPU handling and hides the fact that it was skipped. Logged at + // error so it is visible, because the symptom otherwise is a capture + // that looks entirely normal. + log.WithError(err).WithField("path", nvidiaSmi). + Error("nvidia-smi query failed; assuming single GPU - a multi-GPU workload will be MISCLASSIFIED") return 1 } @@ -1966,11 +1977,49 @@ func (a *Agent) Checkpoint(ctx context.Context, req CheckpointRequest) (*Checkpo // context state on restore). Multi-GPU workloads MUST use the // rootfs-only path (nvsnap.io/capture label → agent watcher → // per-capture PVC). Reject CRIU API calls for multi-GPU early. - if distinctGPUs > 1 { + // EXPERIMENT (nvsnap/multigpu-nvls): the multi-GPU CRIU path used to be + // rejected here outright, on the grounds that cuda-checkpoint blocks on + // peer state and the D2H/intercept path could never reconstruct CUDA + // context state on restore. That rejection also made the entire multi-GPU + // branch below unreachable, so the D2H machinery could not be exercised at + // all -- including to find out whether it still fails the same way. + // + // The fence is lifted on this branch so the path can be measured rather + // than assumed. Restore is expected to be the wall; capture is not. + // NVSNAP_MULTI_GPU_CRIU=1 opts in; anything else keeps the old refusal, so + // no cluster picks this up by accident. + // + // NVSNAP_FORCE_MULTI_GPU=1 additionally forces the multi-GPU branch on when + // device counting says otherwise. Detection fails open (nvidia-smi errors + // return 1), and a silent misclassification would make the experiment look + // like it ran when it did not. + // Two independent switches, because they select different engines and + // conflating them sends the run down the path you did not mean to test. + // + // NVSNAP_MULTI_GPU_CRIU=1 lifts the fence and leaves isMultiGPU false, so a + // multi-GPU workload takes the ORDINARY criu-v2 path: in-namespace dump, + // cuda_plugin driving cuda-checkpoint per rank, no interception stack and + // no D2H. This is the interesting one - it inherits the CRIU-layer fixes + // (io_uring ring restore, in-namespace mounts) that made the injection + // stack unnecessary for single GPU, instead of reviving it. + // + // NVSNAP_LEGACY_MULTI_GPU_D2H=1 selects the OLD path instead: cgroup + // freeze, NCCL quiesce, P2P disable, D2H save, skipping cuda-checkpoint. + // That path needs the interposer plus the forked libzmq/libuv preloaded + // into the workload, and its restore half has never worked. + multiGPUCRIU := os.Getenv("NVSNAP_MULTI_GPU_CRIU") == "1" + legacyD2H := os.Getenv("NVSNAP_LEGACY_MULTI_GPU_D2H") == "1" + if distinctGPUs > 1 && !multiGPUCRIU && !legacyD2H { return nil, fmt.Errorf("multi-GPU CRIU is unsupported (distinctGPUs=%d, gpuPIDs=%v); use the rootfs-only path: label the source pod nvsnap.io/capture=true and apply a fresh pod with nvsnap.io/restore-from=", distinctGPUs, gpuPIDs) } - isMultiGPU := false - useCUDAInterposition := false + isMultiGPU := legacyD2H + useCUDAInterposition := legacyD2H + if distinctGPUs > 1 { + log.WithFields(logrus.Fields{ + "distinctGPUs": distinctGPUs, + "engine": map[bool]string{true: "legacy-d2h", false: "criu-v2"}[legacyD2H], + }).Warn("EXPERIMENTAL: multi-GPU capture is enabled on this agent") + } // Create checkpoint directory early so NvSnap can write GPU saves // directly to the final location (no temp files, no copy). diff --git a/src/compute-plane-services/nvsnap/internal/agent/checkpoint_v2.go b/src/compute-plane-services/nvsnap/internal/agent/checkpoint_v2.go index d95f03f45f..4253856e60 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/checkpoint_v2.go +++ b/src/compute-plane-services/nvsnap/internal/agent/checkpoint_v2.go @@ -56,6 +56,7 @@ import ( "os" "os/exec" "path/filepath" + "sort" "strconv" "strings" "syscall" @@ -180,6 +181,9 @@ func (a *Agent) dumpV2(ctx context.Context, containerInfo *containerd.ContainerI if err != nil { return fmt.Errorf("resolve ns pid of %d: %w", targetHostPID, err) } + // Resolved here, checked after the dump: see gpuNSPids on why it cannot + // wait until the images exist. + gpuNS := gpuNSPids(procBase, gpuPIDs, log) log.WithFields(logrus.Fields{ "targetHostPID": targetHostPID, "nsPID": nsPID, @@ -340,6 +344,11 @@ func (a *Agent) dumpV2(ctx context.Context, containerInfo *containerd.ContainerI if moveErr != nil { return fmt.Errorf("criu-v2: move images: %w", moveErr) } + // After the move, so it inspects the images where they will actually be + // read from rather than a staging copy that is about to be discarded. + if err := assertGPUProcessesCaptured(checkpointDir, gpuNS, log); err != nil { + return err + } log.Info("criu-v2: dump complete, images moved to checkpoint dir") return nil } @@ -406,6 +415,63 @@ func sessionID(procBase string, pid int) (int, error) { return strconv.Atoi(fields[3]) // sid } +// gpuNSPids resolves each GPU host pid to its in-namespace pid. +// +// Must be called BEFORE the dump: a dump that is not leave-running kills the +// tree, so /proc//status is gone by the time the images exist and the +// mapping can no longer be recovered. +// +// A pid that cannot be resolved is skipped rather than fatal. It has usually +// just exited on its own, and refusing to capture a healthy workload because +// one short-lived helper raced us would be worse than the gap it leaves. +func gpuNSPids(procBase string, gpuPIDs []int, log *logrus.Entry) map[int]int { + out := make(map[int]int, len(gpuPIDs)) + for _, hostPID := range gpuPIDs { + nsPID, err := nsPidOf(procBase, hostPID) + if err != nil { + log.WithError(err).WithField("hostPID", hostPID). + Warn("criu-v2: cannot resolve ns pid of GPU process; excluded from the capture check") + continue + } + out[hostPID] = nsPID + } + return out +} + +// assertGPUProcessesCaptured fails the capture when a process the agent +// identified as holding GPU state is absent from the images. +// +// Without this the agent will publish a checkpoint that is missing the very +// processes the capture exists for, and report success while doing it. That is +// not hypothetical: quiescing NCCL on a live multi-GPU vLLM can take the +// executor down with it, and CRIU then dumps the surviving API server perfectly +// and exits 0. The result is a valid-looking checkpoint roughly 3% of the +// expected size, whose only symptom is a number nobody is checking. +// +// The GPU pid list is the agent's own, gathered moments earlier, so this +// compares intent against outcome rather than trusting either alone. +func assertGPUProcessesCaptured(imagesDir string, nsPids map[int]int, log *logrus.Entry) error { + if len(nsPids) == 0 { + return nil + } + var missing []int + for hostPID, nsPID := range nsPids { + if _, err := os.Stat(filepath.Join(imagesDir, fmt.Sprintf("core-%d.img", nsPID))); err != nil { + missing = append(missing, hostPID) + } + } + if len(missing) == 0 { + log.WithField("gpuProcesses", len(nsPids)).Info("criu-v2: all GPU processes present in the capture") + return nil + } + sort.Ints(missing) + return fmt.Errorf( + "criu-v2: capture is missing %d of %d GPU process(es) (host pids %v): they held the GPU state this "+ + "checkpoint exists for, so the images are incomplete and must not be published. The usual cause is "+ + "the processes exiting between quiesce and dump", + len(missing), len(nsPids), missing) +} + // nsPidOf returns pid as seen inside its innermost pid namespace (last // entry of the NSpid line). func nsPidOf(procBase string, pid int) (int, error) { diff --git a/src/compute-plane-services/nvsnap/internal/agent/checkpoint_v2_gpuguard_test.go b/src/compute-plane-services/nvsnap/internal/agent/checkpoint_v2_gpuguard_test.go new file mode 100644 index 0000000000..645aa1b695 --- /dev/null +++ b/src/compute-plane-services/nvsnap/internal/agent/checkpoint_v2_gpuguard_test.go @@ -0,0 +1,158 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package agent + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/sirupsen/logrus" +) + +func guardLog() *logrus.Entry { + l := logrus.New() + l.SetOutput(os.NewFile(0, os.DevNull)) + return logrus.NewEntry(l) +} + +// writeCores creates core-.img for each pid, as a completed dump would. +func writeCores(t *testing.T, dir string, nsPids ...int) { + t.Helper() + for _, p := range nsPids { + f := filepath.Join(dir, fmt.Sprintf("core-%d.img", p)) + if err := os.WriteFile(f, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + } +} + +// The case this guard exists for. Quiescing NCCL on a live multi-GPU engine can +// take the executor down with it; CRIU then dumps the surviving API server +// perfectly and exits 0. Without this check the agent publishes a checkpoint +// missing the processes that held the GPU state, and reports success. +func TestAssertGPUProcessesCapturedRejectsAPartialCapture(t *testing.T) { + dir := t.TempDir() + writeCores(t, dir, 301) // only the API server survived + + nsPids := map[int]int{ + 3122085: 301, // API server -- captured + 3122857: 730, // EngineCore -- died during quiesce + 3123125: 894, // Worker_TP0 -- died during quiesce + 3123126: 895, // Worker_TP1 -- died during quiesce + } + + err := assertGPUProcessesCaptured(dir, nsPids, guardLog()) + if err == nil { + t.Fatal("a capture missing 3 of 4 GPU processes must be refused, not published") + } + for _, want := range []string{"3 of 4", "3122857", "3123125", "3123126"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error should name what is missing (%q): %v", want, err) + } + } + if strings.Contains(err.Error(), "3122085") { + t.Errorf("the captured process must not be reported as missing: %v", err) + } +} + +func TestAssertGPUProcessesCapturedAcceptsACompleteCapture(t *testing.T) { + dir := t.TempDir() + writeCores(t, dir, 301, 730, 894, 895) + + nsPids := map[int]int{3122085: 301, 3122857: 730, 3123125: 894, 3123126: 895} + if err := assertGPUProcessesCaptured(dir, nsPids, guardLog()); err != nil { + t.Fatalf("a complete capture must pass: %v", err) + } +} + +// Losing even one rank makes the checkpoint unrestorable, so there is no +// "mostly captured" that should be allowed through. +func TestAssertGPUProcessesCapturedRejectsASingleMissingRank(t *testing.T) { + dir := t.TempDir() + writeCores(t, dir, 301, 730, 894) + + nsPids := map[int]int{3122085: 301, 3122857: 730, 3123125: 894, 3123126: 895} + err := assertGPUProcessesCaptured(dir, nsPids, guardLog()) + if err == nil { + t.Fatal("one missing rank must still fail the capture") + } + if !strings.Contains(err.Error(), "1 of 4") { + t.Errorf("error should report 1 of 4 missing: %v", err) + } +} + +// A workload with no GPU processes is a legitimate capture, not an empty one. +// Failing here would break every CPU-only workload. +func TestAssertGPUProcessesCapturedIgnoresWorkloadsWithNoGPU(t *testing.T) { + if err := assertGPUProcessesCaptured(t.TempDir(), nil, guardLog()); err != nil { + t.Fatalf("no GPU processes means nothing to check: %v", err) + } + if err := assertGPUProcessesCaptured(t.TempDir(), map[int]int{}, guardLog()); err != nil { + t.Fatalf("empty map means nothing to check: %v", err) + } +} + +// The mapping must be taken before the dump, because a non-leave-running dump +// kills the tree. This asserts the resolver's behaviour on pids that are +// already gone: skip them, do not fail the capture. +func TestGPUNSPidsSkipsUnresolvablePids(t *testing.T) { + base := t.TempDir() + // One live-looking pid with a NSpid line, one with no /proc entry at all. + dir := filepath.Join(base, "4242") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "status"), + []byte("Name:\tvllm\nNSpid:\t4242\t301\n"), 0o600); err != nil { + t.Fatal(err) + } + + got := gpuNSPids(base, []int{4242, 9999}, guardLog()) + if len(got) != 1 { + t.Fatalf("expected only the resolvable pid, got %v", got) + } + if got[4242] != 301 { + t.Errorf("expected host 4242 -> ns 301, got %v", got) + } + if _, ok := got[9999]; ok { + t.Errorf("an unresolvable pid must be skipped, not invented: %v", got) + } +} + +// The innermost namespace pid is the one CRIU names its images after. Taking +// the first field instead would silently look for core-.img, which +// never exists, and fail every capture. +func TestGPUNSPidsUsesTheInnermostNamespacePid(t *testing.T) { + base := t.TempDir() + dir := filepath.Join(base, "5000") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + // host 5000 -> intermediate 200 -> container 7 + if err := os.WriteFile(filepath.Join(dir, "status"), + []byte("NSpid:\t5000\t200\t7\n"), 0o600); err != nil { + t.Fatal(err) + } + got := gpuNSPids(base, []int{5000}, guardLog()) + if got[5000] != 7 { + t.Errorf("expected innermost ns pid 7, got %v", got) + } +} diff --git a/src/compute-plane-services/nvsnap/internal/manifests/conformance_test.go b/src/compute-plane-services/nvsnap/internal/manifests/conformance_test.go index 4f7e4c08c6..3e6f94c026 100644 --- a/src/compute-plane-services/nvsnap/internal/manifests/conformance_test.go +++ b/src/compute-plane-services/nvsnap/internal/manifests/conformance_test.go @@ -112,10 +112,26 @@ func TestMultiGPUSourcesAreNotCRIU(t *testing.T) { if p.Metadata.Annotations["nvsnap.io/path"] != "criu" { continue } + // Manifests named *-criu are the deliberate multi-GPU criu-v2 workloads. + // Multi-GPU CRIU is no longer rejected outright: with every cross-GPU + // transport off, criu-v2 captures and restores tensor-parallel workloads + // on vLLM and TRT-LLM (see docs/proposals/multi-gpu-criu-v2.md). It is + // still off by default and needs NVSNAP_MULTI_GPU_CRIU=1 on the agent, + // so it stays opt-in, and the naming is what marks the opt-in. + // + // The check below still matters for everything else: a multi-GPU manifest + // that declares criu WITHOUT being one of these gets a criu-v2 restore + // placeholder while its capture goes via cachedir, and then idles until + // the readiness timeout. That reads as a restore bug rather than a + // mislabelled source, which is exactly how vllm-tp2 once presented. + if strings.HasSuffix(base, "-criu.yaml") { + continue + } for _, c := range p.Spec.Containers { if gpus := c.Resources.Limits["nvidia.com/gpu"]; gpus != "" && gpus != "0" && gpus != "1" { - t.Errorf("%s requests %s GPUs but declares nvsnap.io/path: \"criu\"; "+ - "the agent rejects multi-GPU CRIU, so this must be \"rootfs\"", + t.Errorf("%s requests %s GPUs but declares nvsnap.io/path: \"criu\" and is not a "+ + "*-criu.yaml opt-in workload; the capture will go via cachedir while the "+ + "generated placeholder waits for an agent-driven restore that never comes", base, gpus) } } diff --git a/src/compute-plane-services/nvsnap/internal/manifests/restore.go b/src/compute-plane-services/nvsnap/internal/manifests/restore.go index f62ab956f1..edee7077e6 100644 --- a/src/compute-plane-services/nvsnap/internal/manifests/restore.go +++ b/src/compute-plane-services/nvsnap/internal/manifests/restore.go @@ -241,7 +241,7 @@ spec: {{- end }} # Push this pod's own pid allocations high so the low pid range the # dump captured stays free for CRIU's exact-pid forks. - echo 100000 > /proc/sys/kernel/ns_last_pid || echo "(ns_last_pid bump failed)" + echo 100000 > /proc/sys/kernel/ns_last_pid || echo "WARNING: ns_last_pid bump FAILED - restore will hit pid collisions" # Restored workload stdio is a plain-file fd on {{ .StdoutTo }} (the # source manifest's setsid convention); surface it via kubelet. touch {{ .StdoutTo }} @@ -267,6 +267,14 @@ spec: {{- end }} securityContext: privileged: true + # Root, regardless of what the source image runs as. The placeholder has + # to write /proc/sys/kernel/ns_last_pid, and privileged does not confer + # root: an image defaulting to a non-root uid (NIM runs as 1000) fails + # that write, leaves its pid range unreserved, and the restore then dies + # with "Can't fork for : File exists". The placeholder is a throwaway + # reaper and the restored workload's own uid comes from the checkpoint, + # so this does not change what the workload runs as. + runAsUser: 0 resources: limits: nvidia.com/gpu: "{{ .GPUs }}" diff --git a/src/compute-plane-services/nvsnap/scripts/test-e2e.sh b/src/compute-plane-services/nvsnap/scripts/test-e2e.sh index d343134b64..6ff9c5de79 100755 --- a/src/compute-plane-services/nvsnap/scripts/test-e2e.sh +++ b/src/compute-plane-services/nvsnap/scripts/test-e2e.sh @@ -107,6 +107,73 @@ case "$WORKLOAD" in SOURCE_MANIFEST="$PROJECT_ROOT/deploy/k8s/workloads/vllm-mp.yaml" RESTORE_MANIFEST_TEMPLATE="$PROJECT_ROOT/deploy/k8s/workloads/vllm-mp-restore.yaml" ;; + vllm-70b-criu) + # Llama-3.1-70B TP=4 on criu-v2. ~270G checkpoint at util 0.85, so the + # default 600s CHECKPOINT_TIMEOUT is not enough; the runner raises it. + POD_NAME="vllm-70b-criu" + CONTAINER_NAME="vllm" + RESTORE_POD_NAME="vllm-70b-criu-restored" + RESTORE_CONTAINER_NAME="restore" + PORT=8000 + MODEL="meta-llama/Llama-3.1-70B-Instruct" + INFER_ENDPOINT="/v1/completions" + INFER_DATA='{"model":"meta-llama/Llama-3.1-70B-Instruct","prompt":"Hello","max_tokens":5}' + POST_INFER_DATA='{"model":"meta-llama/Llama-3.1-70B-Instruct","prompt":"The meaning of life is","max_tokens":10}' + SOURCE_MANIFEST="$PROJECT_ROOT/deploy/k8s/workloads/vllm-70b-criu.yaml" + RESTORE_MANIFEST_TEMPLATE="$PROJECT_ROOT/deploy/k8s/workloads/vllm-70b-criu-restore.yaml" + ;; + nim-qwen3-32b-criu) + # NIM (TRT-LLM) TP=2 on criu-v2. Tests whether multi-GPU criu-v2 works + # on an engine other than vLLM. No CUDA-graph disable: TRT-LLM sets + # graphs at engine build and the image exposes no env for it, which is + # acceptable for capture since graphs fail at restore-inference. + POD_NAME="nim-qwen3-32b-criu" + CONTAINER_NAME="nim" + RESTORE_POD_NAME="nim-qwen3-32b-criu-restored" + RESTORE_CONTAINER_NAME="restore" + PORT=8000 + MODEL="qwen/qwen3-32b" + INFER_ENDPOINT="/v1/completions" + INFER_DATA='{"model":"qwen/qwen3-32b","prompt":"Hello","max_tokens":5}' + POST_INFER_DATA='{"model":"qwen/qwen3-32b","prompt":"The meaning of life is","max_tokens":10}' + SOURCE_MANIFEST="$PROJECT_ROOT/deploy/k8s/workloads/nim-qwen3-32b-criu.yaml" + RESTORE_MANIFEST_TEMPLATE="$PROJECT_ROOT/deploy/k8s/workloads/nim-qwen3-32b-criu-restore.yaml" + ;; + sglang-tp2-criu) + # SGLang TP=2 on criu-v2. Tests whether the peer-state sever recipe + # generalises beyond vLLM. SGLang's --disable-cuda-graph is the + # --enforce-eager analogue; the NCCL knobs are engine-independent. + POD_NAME="sglang-tp2-criu" + CONTAINER_NAME="sglang" + RESTORE_POD_NAME="sglang-tp2-criu-restored" + RESTORE_CONTAINER_NAME="restore" + PORT=30000 + MODEL="meta-llama/Llama-3.1-8B-Instruct" + INFER_ENDPOINT="/v1/completions" + INFER_DATA='{"model":"meta-llama/Llama-3.1-8B-Instruct","prompt":"Hello","max_tokens":5}' + POST_INFER_DATA='{"model":"meta-llama/Llama-3.1-8B-Instruct","prompt":"The meaning of life is","max_tokens":10}' + SOURCE_MANIFEST="$PROJECT_ROOT/deploy/k8s/workloads/sglang-tp2-criu.yaml" + RESTORE_MANIFEST_TEMPLATE="$PROJECT_ROOT/deploy/k8s/workloads/sglang-tp2-criu-restore.yaml" + ;; + vllm-tp2-criu) + # Multi-GPU on the criu-v2 engine. Separate from vllm-tp2 (which stays + # on the rootfs/cachedir path) so the two do not share a manifest. + # Needs the agent started with NVSNAP_MULTI_GPU_CRIU=1 and the run + # invoked with CAPTURE_PATH=criu-v2. Every cross-GPU transport is off + # in the manifest: cuda-checkpoint blocks if any peer mapping exists. + # See docs/proposals/multi-gpu-criu-v2.md. + POD_NAME="vllm-tp2-criu" + CONTAINER_NAME="vllm" + RESTORE_POD_NAME="vllm-tp2-criu-restored" + RESTORE_CONTAINER_NAME="restore" + PORT=8000 + MODEL="TinyLlama/TinyLlama-1.1B-Chat-v1.0" + INFER_ENDPOINT="/v1/completions" + INFER_DATA='{"model":"TinyLlama/TinyLlama-1.1B-Chat-v1.0","prompt":"Hello","max_tokens":5}' + POST_INFER_DATA='{"model":"TinyLlama/TinyLlama-1.1B-Chat-v1.0","prompt":"The meaning of life is","max_tokens":10}' + SOURCE_MANIFEST="$PROJECT_ROOT/deploy/k8s/workloads/vllm-tp2-criu.yaml" + RESTORE_MANIFEST_TEMPLATE="$PROJECT_ROOT/deploy/k8s/workloads/vllm-tp2-criu-restore.yaml" + ;; vllm-tp2) # E1 multi-GPU ladder: TinyLlama TP=2 eager. Force the CRIU engine # with CAPTURE_PATH=criu-v2 (multi-GPU otherwise defaults to rootfs). @@ -276,22 +343,25 @@ NAMESPACE="nvsnap-system" # Timeouts (seconds) — 70B needs longer for model download + GPU memory dump/restore if [[ "$WORKLOAD" == *"70b"* ]]; then - POD_READY_TIMEOUT=1800 # 30min: 70B model download + load + # A COLD 70B run does not fit 30min: the HF pull alone is ~140G and a + # measured cold start needed >32min just to reach Ready. Warm caches finish + # well inside this; the ceiling is here for the cold case. + POD_READY_TIMEOUT=${POD_READY_TIMEOUT_OVERRIDE:-4200} # 70min: 140G pull + load MODELS_POLL_TIMEOUT=1200 # 20min INFERENCE_POLL_TIMEOUT=300 RESTORE_READY_TIMEOUT=1200 # 20min: CRIU + 4x GPU memory restore elif [[ "$WORKLOAD" == trtllm-* ]]; then - POD_READY_TIMEOUT=1800 # 30min: ~25GB image pull + TRT engine compilation + POD_READY_TIMEOUT=${POD_READY_TIMEOUT_OVERRIDE:-1800} # 30min: ~25GB image pull + TRT engine compilation MODELS_POLL_TIMEOUT=1200 # 20min INFERENCE_POLL_TIMEOUT=300 RESTORE_READY_TIMEOUT=600 # 10min -elif [[ "$WORKLOAD" == *"qwen32b"* ]]; then - POD_READY_TIMEOUT=1800 # 30min: ~64GB fp16 model download + load +elif [[ "$WORKLOAD" == *"qwen32b"* || "$WORKLOAD" == *"qwen3-32b"* ]]; then + POD_READY_TIMEOUT=${POD_READY_TIMEOUT_OVERRIDE:-1800} # 30min: ~64GB fp16 model download + load MODELS_POLL_TIMEOUT=1200 # 20min INFERENCE_POLL_TIMEOUT=300 RESTORE_READY_TIMEOUT=1200 # 20min: CRIU restore of ~64GB host-staged GPU memory else - POD_READY_TIMEOUT=600 # 10min + POD_READY_TIMEOUT=${POD_READY_TIMEOUT_OVERRIDE:-600} # 10min MODELS_POLL_TIMEOUT=600 INFERENCE_POLL_TIMEOUT=300 RESTORE_READY_TIMEOUT=600 # 10min