feat(nvsnap): elect one downloader per cache hash at admission

src/compute-plane-services/nvsnap/cmd/agent/main.go

@@ -167,6 +167,13 @@ func main() {
 	// Empty disables the inject (back-compat: PVC mount still works
 	// once the rox PVC binds; kubelet may stall in
 	// ContainerCreating in the meantime).
+	// One-downloader-per-hash admission election for chart-shaped model
+	// workloads (docs/proposals/helm-chart-cache-election.md). Needs L2.
+	flag.BoolVar(&config.Election.Enabled, "election", false,
+		"Elect one capture leader per cache hash at admission and gate the other model workers until the capture is promoted (needs L2)")
+	flag.DurationVar(&config.Election.Deadline, "election-deadline", 0,
+		"Bound on a leader's cold start plus capture; past it nvsnap-server evicts the gated followers for re-election (default 60m)")
+
 	flag.StringVar(&config.Webhook.L2WaitImage, "webhook-l2-wait-image", "",
 		"Image ref for the nvsnap-l2-wait init container injected onto restore pods (nvsnap#147)")
 

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-daemonset.yaml

@@ -112,6 +112,11 @@ spec:
             # stamped in the manifest.
             - --cachedir-env-file=/etc/nvsnap/cachedir-env/env
             {{- end }}
+            {{- if and .Values.agent.election .Values.agent.election.enabled }}
+            # one-downloader election (values: agent.election)
+            - --election
+            - --election-deadline={{ .Values.agent.election.deadline | default "60m" }}
+            {{- end }}
             - --rootfs-cm-namespace={{ .Release.Namespace }}
             # nvsnap#194: OverlayFS scratch root. MUST match the
             # nvsnap-overlays volumeMount path below and the hostPath

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-rbac.yaml

@@ -86,6 +86,13 @@ rules:
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots"]
     verbs: ["get", "list", "watch", "create", "delete"]
+  # volumesnapshotcontents: a namespace-local restore claim on
+  # snapshot-clone storage is a clone of a pre-provisioned snapshot whose
+  # content points at the promote's CSI snapshot handle
+  # (SnapshotClonePromoter.EnsureClaim).
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list", "watch", "create", "delete"]
   # storageclasses.get: the L2 backend probes the configured SC at
   # startup (internal/agent/l2_integration.go validateL2StorageClass)
   # to verify it exists and log its provisioner. Without this rule

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/server.yaml

@@ -64,9 +64,11 @@ rules:
   # Lease ('nvsnap-promote-<short>') to serialize concurrent promote
   # attempts on the same hash. Deleted on hash delete so a future
   # re-capture isn't blocked by a stale lease holder.
+  # get/list/watch: the election reconciler lists nvsnap-capture-<short>
+  # Leases to check leader liveness and deletes them on release.
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
-    verbs: ["delete"]
+    verbs: ["get", "list", "watch", "delete"]
   # services.get: the observability proxy (internal/server/observability_proxy.go)
   # probes whether the grafana / jaeger / prometheus Services exist
   # before rendering the matching UI nav tile. Without this rule the

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml

@@ -107,6 +107,20 @@ agent:
   # capture (no gemm capture, no prewarm).
   podCacheDir: "/opt/nvsnap"
 
+  # One downloader per cache hash for chart-shaped model workloads
+  # (docs/proposals/helm-chart-cache-election.md). At admission the
+  # webhook composes the cache hash of every GPU pod that names a model;
+  # if a promoted capture exists the pod restores from it, otherwise one
+  # pod is elected to capture and the rest are held by a scheduling gate
+  # (no node, no GPU) until nvsnap-server sees the promote and releases
+  # them. Needs agent.l2. Off until qualified on a cluster.
+  election:
+    enabled: false
+    # Bound on the leader's cold start plus capture. Past it the server
+    # evicts the gated followers so their controller recreates them and a
+    # new leader is elected.
+    deadline: 60m
+
   # cacheEnvTemplate is the cachedir env set (nvsnap #244), rendered into
   # the nvsnap-cachedir-env ConfigMap when podCacheDir is set. The agent
   # webhook reads it on CAPTURE inject — edit it (helm value or

src/compute-plane-services/nvsnap/deploy/k8s/charts/vllm-workers/Chart.yaml

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: v2
+name: vllm-workers
+version: 0.1.0
+description: >-
+  Stock vLLM Deployment with N identical GPU workers and no nvsnap markers.
+  Test chart for the one-downloader election (docs/proposals/helm-chart-cache-election.md).

src/compute-plane-services/nvsnap/deploy/k8s/charts/vllm-workers/templates/deployment.yaml

@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}
+  labels: { app: {{ .Release.Name }} }
+spec:
+  replicas: {{ .Values.replicas }}
+  selector:
+    matchLabels: { app: {{ .Release.Name }} }
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}
+    spec:
+      automountServiceAccountToken: false
+      {{- with .Values.excludeNodes }}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values: {{ toJson . }}
+      {{- end }}
+      tolerations:
+        - { key: "nvidia.com/gpu", operator: Exists, effect: NoSchedule }
+      {{- with .Values.imagePullSecret }}
+      imagePullSecrets:
+        - name: {{ . }}
+      {{- end }}
+      containers:
+        - name: vllm
+          image: {{ .Values.image }}
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/bash", "-lc"]
+          args:
+            - |
+              set -e
+              nohup setsid vllm serve --model {{ .Values.model }} --host 0.0.0.0 --port 8000 --max-model-len {{ .Values.maxModelLen }} --tensor-parallel-size {{ .Values.tensorParallel }} --gpu-memory-utilization {{ .Values.gpuMemoryUtilization }} {{ .Values.extraArgs }} > /vllm.out 2>&1 < /dev/null &
+              tail -F /vllm.out &
+              while true; do sleep 30; done
+          env:
+            - { name: PYTHONUNBUFFERED, value: "1" }
+            - { name: HF_HOME, value: "/root/.cache/huggingface" }
+            - { name: VLLM_ENABLE_V1_MULTIPROCESSING, value: "1" }
+            - { name: HF_HUB_DISABLE_XET, value: "1" }
+            {{- with .Values.env }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          ports: [{ containerPort: 8000, name: http }]
+          readinessProbe:
+            httpGet: { path: /v1/models, port: 8000 }
+            initialDelaySeconds: 20
+            periodSeconds: 5
+            failureThreshold: {{ .Values.readinessFailureThreshold }}
+          resources:
+            limits: { nvidia.com/gpu: {{ .Values.tensorParallel | quote }} }
+          volumeMounts: [{ name: shm, mountPath: /dev/shm }]
+      volumes:
+        - name: shm
+          emptyDir: { medium: Memory, sizeLimit: {{ .Values.shmSize }} }

src/compute-plane-services/nvsnap/deploy/k8s/charts/vllm-workers/values.yaml

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# A plain vLLM chart, deliberately free of nvsnap labels and annotations:
+# the election has to work on what customers deploy. Every worker is
+# identical; the webhook decides per pod who downloads.
+replicas: 2
+image: vllm/vllm-openai:v0.20.0
+imagePullSecret: ""            # e.g. nvsnap-pull-secret
+model: Qwen/Qwen2.5-32B-Instruct
+tensorParallel: 4              # also the GPU request per worker
+maxModelLen: 4096
+gpuMemoryUtilization: 0.85
+extraArgs: ""                  # appended to vllm serve, e.g. "--enforce-eager"
+env: []                        # extra env, e.g. [{name: HF_TOKEN, valueFrom: {secretKeyRef: {name: hf-token, key: token}}}]
+shmSize: 32Gi
+readinessFailureThreshold: 400 # x 5 s; cold starts of large models take minutes
+excludeNodes: []               # hostnames to keep the workers off

src/compute-plane-services/nvsnap/docs/BENCHMARK.md

@@ -42,6 +42,9 @@ recompiling.
 
 | Model | Engine | Cold | Restore | Speedup |
 |---|---|---:|---:|---:|
+| Qwen3-235B-A22B FP8 (chart, 2 replicas, 2 nodes) | vLLM TP=8 EP | 870 s | 367 s | 2.4x |
+| Qwen2.5-32B-Instruct (chart, 2 replicas) | vLLM TP=4 | 325 s | 170 s | 1.9x |
+| Llama-3.1-70B-Instruct | vLLM TP=4 | 632 s | 246 s | 2.6x |
 | DeepSeek-V4-Flash | SGLang TP=8 | 1162 s | 289 s | 4.0x |
 | gemma-4-31B-it | SGLang | 492 s | 109 s | 4.5x |
 | gpt-oss-120b | vLLM TP=4 | ~550 s | 171 s | 3.2x |
@@ -51,6 +54,16 @@ recompiling.
 | e5-mistral-7b-instruct | vLLM | 73 s | 90 s | -- |
 | whisper-large-v3 | NIM (Riva) | 72 s | 74 s | -- |
 
+The two rows above the June table are from dev1 (EKS, NVMesh, 2026-09-25);
+the Qwen row is a two-replica Helm chart through the admission election
+(`docs/proposals/helm-chart-cache-election.md`): first deploy pays one cold
+leader plus capture, every later start of the chart is the restore number.
+Phase split for Qwen3-235B, cold to warm: model download and load 641 s to
+17 s, torch.compile 94 s to 15 s, init engine 292 s to 48 s; the warm 367 s is
+mostly the 230 s prewarm read of 237 GB. Qwen2.5-32B: model download and load 247 s to 6 s,
+torch.compile 57 s to 8.5 s, init engine 77 s to 22 s, CUDA graph capture
+10 s either way. Llama-70B: model load 481 s to 14 s, compile 65 s to 3 s.
+
 Wins scale with the cold JIT/compile cost; neutral on compile-light and
 framework-bound workloads. Two implementation details enable the cache reuse: