feat(nvsnap): multi-GPU checkpoint/restore on criu-v2

src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml

@@ -47,7 +47,7 @@ spec:
     # scripts/sync-versions.sh) — same restore-entrypoint binary the
     # production deploy uses.
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.42
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.65
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-qwen3-32b-criu-restore.yaml

@@ -0,0 +1,90 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# GENERATED by internal/manifests -- do not edit.
+# Regenerate with: go generate ./internal/manifests/...
+#
+# Restore placeholder for a criu-v2 (in-namespace) checkpoint of nim-qwen3-32b-criu.
+#
+# Dumb reaper: same image as the source (CRIU's path-based file checks resolve
+# against an identical rootfs), bash pid1 reaps orphans, no restore-entrypoint
+# and no hostPID -- the pod keeps its own fresh pid namespace, which is where
+# the in-namespace CRIU restores the dumped session. The agent drives
+# everything on POST /v1/restore. See internal/agent/restore_v2.go.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nim-qwen3-32b-criu-restored
+  namespace: nvsnap-system
+  labels:
+    app: nim-qwen3-32b-criu-restored
+    nvsnap.io/demo: "true"
+spec:
+  automountServiceAccountToken: false
+  # IMPORTANT: must run on the same node as the source pod's checkpoint.
+  # test-e2e.sh substitutes __NODE_NAME__ from the source pod's status.
+  nodeName: __NODE_NAME__
+
+  imagePullSecrets:
+    - name: nvsnap-pull-secret
+    - name: nim-pull-secret
+
+  containers:
+    - name: restore
+      image: nvcr.io/nim/qwen/qwen3-32b:1.0.0
+      imagePullPolicy: IfNotPresent
+      command: ["/bin/bash", "-lc"]
+      args:
+        - |
+          set -e
+          # Push this pod's own pid allocations high so the low pid range the
+          # dump captured stays free for CRIU's exact-pid forks.
+          echo 100000 > /proc/sys/kernel/ns_last_pid || echo "WARNING: ns_last_pid bump FAILED - restore will hit pid collisions"
+          # Restored workload stdio is a plain-file fd on /opt/nim/nim.out (the
+          # source manifest's setsid convention); surface it via kubelet.
+          touch /opt/nim/nim.out
+          tail -F /opt/nim/nim.out &
+          while true; do sleep 30; done
+      env:
+        # CHECKPOINT_ID intentionally in block style -- test-e2e.sh's sed
+        # substitution advances to the NEXT line.
+        - name: CHECKPOINT_ID
+          value: "__CHECKPOINT_ID__"
+      readinessProbe:
+        httpGet:
+          path: /v1/health/ready
+          port: 8000
+        initialDelaySeconds: 5
+        periodSeconds: 5
+        timeoutSeconds: 5
+        failureThreshold: 80
+      securityContext:
+        privileged: true
+        # Root, regardless of what the source image runs as. The placeholder has
+        # to write /proc/sys/kernel/ns_last_pid, and privileged does not confer
+        # root: an image defaulting to a non-root uid (NIM runs as 1000) fails
+        # that write, leaves its pid range unreserved, and the restore then dies
+        # with "Can't fork for <pid>: File exists". The placeholder is a throwaway
+        # reaper and the restored workload's own uid comes from the checkpoint,
+        # so this does not change what the workload runs as.
+        runAsUser: 0
+      resources:
+        limits:
+          nvidia.com/gpu: "2"
+        requests:
+          nvidia.com/gpu: "2"
+      volumeMounts:
+        - { name: checkpoints, mountPath: /checkpoints }
+        - { name: dev-shm, mountPath: /dev/shm }
+
+  volumes:
+    - name: checkpoints
+      hostPath:
+        path: /var/lib/containerd/nvsnap-checkpoints
+        type: Directory
+    - name: dev-shm
+      emptyDir:
+        medium: Memory
+        sizeLimit: 16Gi
+
+  restartPolicy: Never

src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-qwen3-32b-criu.yaml

@@ -0,0 +1,113 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# nim-qwen3-32b-criu SOURCE pod: Qwen3-32B TP=2 on the criu-v2 engine.
+#
+# Separate from nim-qwen3-32b, which stays on the cachedir path. The two
+# engines do not share a manifest.
+#
+# The stock NIM image has no command: it runs /opt/nvidia/nvidia_entrypoint.sh
+# with cmd `bash -c $SERVER_START_SCRIPT_PATH`. criu-v2 needs the setsid
+# convention instead (workload in its own session, stdio to a rootfs file), so
+# the command below reproduces the image's own startup inside that convention
+# rather than replacing it: the entrypoint still runs, and still execs the
+# server start script the image names.
+#
+# Stdio goes to /opt/nim/nim.out rather than /nim.out because the image runs as
+# uid 1000 (nvs) and cannot write the container root. Same reason NIM's restore
+# placeholder needs runAsUser 0 for the pid reservation: privileged does not
+# confer root.
+#
+# Peer-state sever: cuda-checkpoint blocks if cross-GPU state exists. The NCCL
+# knobs are engine-independent so they apply here as they do to vLLM. There is
+# no CUDA-graph disable: TRT-LLM configures graphs at engine build time and the
+# image exposes no env for it. That is deliberate for a first run, because graphs
+# fail late (restore inference) rather than at capture, so their absence does not
+# block finding out whether capture works at all.
+# See docs/proposals/multi-gpu-criu-v2.md.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nim-qwen3-32b-criu
+  namespace: nvsnap-system
+  labels:
+    app: nim-qwen3-32b-criu
+    nvsnap.io/demo: "true"
+  annotations:
+    nvsnap.io/demo-name: "NIM criu-v2"
+    nvsnap.io/desc: "criu-v2 Qwen3-32B multi-GPU TP=2"
+    nvsnap.io/model: "qwen/qwen3-32b"
+    nvsnap.io/port: "8000"
+    nvsnap.io/gpus: "2"
+    nvsnap.io/path: "criu"
+spec:
+  tolerations:
+    - key: "nvidia.com/gpu"
+      operator: "Exists"
+      effect: "NoSchedule"
+
+  imagePullSecrets:
+    - name: nvsnap-pull-secret
+    - name: nim-pull-secret
+
+  containers:
+    - name: nim
+      image: nvcr.io/nim/qwen/qwen3-32b:1.0.0
+      imagePullPolicy: IfNotPresent
+      command: ["/bin/bash", "-lc"]
+      args:
+        - |
+          set -e
+          # criu-v2 convention: the workload runs in its own session via setsid,
+          # so the agent dumps the GPU leader's session rather than container
+          # init, which cannot be restored into a placeholder whose pid 1 is
+          # already occupied. bash stays pid 1 to reap orphans and tails the
+          # output file so kubectl logs keeps working.
+          nohup setsid /opt/nvidia/nvidia_entrypoint.sh \
+            /bin/bash -c "$SERVER_START_SCRIPT_PATH" > /opt/nim/nim.out 2>&1 < /dev/null &
+          tail -F /opt/nim/nim.out &
+          while true; do sleep 30; done
+      env:
+        - name: NGC_API_KEY
+          valueFrom:
+            secretKeyRef:
+              name: ngc-api-key
+              key: NGC_API_KEY
+        - { name: NIM_CACHE_PATH, value: "/opt/nim/.cache" }
+        - { name: HF_HUB_DISABLE_XET, value: "1" }
+        - { name: CUDA_VISIBLE_DEVICES, value: "0,1" }
+        # Peer-state sever (see header).
+        - { name: NCCL_P2P_DISABLE,  value: "1" }
+        - { name: NCCL_NVLS_ENABLE,  value: "0" }
+        - { name: NCCL_SHM_DISABLE,  value: "1" }
+      ports:
+        - containerPort: 8000
+          name: http
+      readinessProbe:
+        httpGet: { path: /v1/health/ready, port: 8000 }
+        initialDelaySeconds: 60
+        periodSeconds: 15
+        failureThreshold: 80
+      resources:
+        limits:
+          nvidia.com/gpu: "2"
+        requests:
+          nvidia.com/gpu: "2"
+      securityContext:
+        privileged: true
+      volumeMounts:
+        - { name: shm, mountPath: /dev/shm }
+        - { name: nim-cache, mountPath: /opt/nim/.cache }
+
+  volumes:
+    - name: shm
+      emptyDir: { medium: Memory, sizeLimit: 16Gi }
+    - name: nim-cache
+      # emptyDir, despite the ~62G re-download per run. A hostPath is created
+      # root-owned by DirectoryOrCreate and NIM runs as uid 1000, which fails at
+      # manifest download with "Permission denied (os error 13)". Making it
+      # persistent needs an initContainer to chown, which is not worth coupling
+      # to the question this workload exists to answer.
+      emptyDir: {}
+
+  restartPolicy: Never

src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-tp2-criu-restore.yaml

@@ -0,0 +1,89 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# GENERATED by internal/manifests -- do not edit.
+# Regenerate with: go generate ./internal/manifests/...
+#
+# Restore placeholder for a criu-v2 (in-namespace) checkpoint of sglang-tp2-criu.
+#
+# Dumb reaper: same image as the source (CRIU's path-based file checks resolve
+# against an identical rootfs), bash pid1 reaps orphans, no restore-entrypoint
+# and no hostPID -- the pod keeps its own fresh pid namespace, which is where
+# the in-namespace CRIU restores the dumped session. The agent drives
+# everything on POST /v1/restore. See internal/agent/restore_v2.go.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sglang-tp2-criu-restored
+  namespace: nvsnap-system
+  labels:
+    app: sglang-tp2-criu-restored
+    nvsnap.io/demo: "true"
+spec:
+  automountServiceAccountToken: false
+  # IMPORTANT: must run on the same node as the source pod's checkpoint.
+  # test-e2e.sh substitutes __NODE_NAME__ from the source pod's status.
+  nodeName: __NODE_NAME__
+
+  imagePullSecrets:
+    - name: nvsnap-pull-secret
+
+  containers:
+    - name: restore
+      image: lmsysorg/sglang:v0.5.15.post1-cu129
+      imagePullPolicy: IfNotPresent
+      command: ["/bin/bash", "-lc"]
+      args:
+        - |
+          set -e
+          # Push this pod's own pid allocations high so the low pid range the
+          # dump captured stays free for CRIU's exact-pid forks.
+          echo 100000 > /proc/sys/kernel/ns_last_pid || echo "WARNING: ns_last_pid bump FAILED - restore will hit pid collisions"
+          # Restored workload stdio is a plain-file fd on /sglang.out (the
+          # source manifest's setsid convention); surface it via kubelet.
+          touch /sglang.out
+          tail -F /sglang.out &
+          while true; do sleep 30; done
+      env:
+        # CHECKPOINT_ID intentionally in block style -- test-e2e.sh's sed
+        # substitution advances to the NEXT line.
+        - name: CHECKPOINT_ID
+          value: "__CHECKPOINT_ID__"
+      readinessProbe:
+        httpGet:
+          path: /v1/models
+          port: 30000
+        initialDelaySeconds: 5
+        periodSeconds: 5
+        timeoutSeconds: 5
+        failureThreshold: 60
+      securityContext:
+        privileged: true
+        # Root, regardless of what the source image runs as. The placeholder has
+        # to write /proc/sys/kernel/ns_last_pid, and privileged does not confer
+        # root: an image defaulting to a non-root uid (NIM runs as 1000) fails
+        # that write, leaves its pid range unreserved, and the restore then dies
+        # with "Can't fork for <pid>: File exists". The placeholder is a throwaway
+        # reaper and the restored workload's own uid comes from the checkpoint,
+        # so this does not change what the workload runs as.
+        runAsUser: 0
+      resources:
+        limits:
+          nvidia.com/gpu: "2"
+        requests:
+          nvidia.com/gpu: "2"
+      volumeMounts:
+        - { name: checkpoints, mountPath: /checkpoints }
+        - { name: dev-shm, mountPath: /dev/shm }
+
+  volumes:
+    - name: checkpoints
+      hostPath:
+        path: /var/lib/containerd/nvsnap-checkpoints
+        type: Directory
+    - name: dev-shm
+      emptyDir:
+        medium: Memory
+        sizeLimit: 16Gi
+
+  restartPolicy: Never