feat(nvsnap): download each model once per cluster into a shared volume for Helm functions

src/compute-plane-services/nvsnap/cmd/agent/main.go

@@ -174,6 +174,15 @@ func main() {
 	flag.DurationVar(&config.Election.Deadline, "election-deadline", 0,
 		"Bound on a leader's cold start plus capture; past it nvsnap-server evicts the gated followers for re-election (default 60m)")
 
+	// Write-once model volume for Helm functions
+	// (docs/proposals/helm-shared-model-volume.md). Needs L2.
+	flag.BoolVar(&config.ModelVolume.Enabled, "model-volume", false,
+		"Download each model once per cluster into a shared volume and attach it to every other pod that names it (needs L2)")
+	flag.StringVar(&config.ModelVolume.HostRoot, "model-volume-host-root", "",
+		"Host directory (mounted Bidirectional into the agent at the same path) where completed model volumes are bound for readers on block storage (default /var/lib/containerd/nvsnap-models)")
+	flag.DurationVar(&config.ModelVolume.WaitDeadline, "model-volume-wait-deadline", 0,
+		"How long a reader waits for the writer's download before downloading itself (default 1h)")
+
 	flag.StringVar(&config.Webhook.L2WaitImage, "webhook-l2-wait-image", "",
 		"Image ref for the nvsnap-l2-wait init container injected onto restore pods (nvsnap#147)")
 

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-daemonset.yaml

@@ -112,6 +112,12 @@ spec:
             # stamped in the manifest.
             - --cachedir-env-file=/etc/nvsnap/cachedir-env/env
             {{- end }}
+            {{- if and .Values.agent.modelVolume .Values.agent.modelVolume.enabled }}
+            # write-once model volume (values: agent.modelVolume)
+            - --model-volume
+            - --model-volume-wait-deadline={{ .Values.agent.modelVolume.waitDeadline | default "1h" }}
+            - --model-volume-host-root={{ .Values.agent.hostPaths.nvsnapModels | default "/var/lib/containerd/nvsnap-models" }}
+            {{- end }}
             {{- if and .Values.agent.election .Values.agent.election.enabled }}
             # one-downloader election (values: agent.election)
             - --election
@@ -317,6 +323,15 @@ spec:
             - name: nvsnap-overlays
               mountPath: {{ .Values.agent.hostPaths.nvsnapOverlays | default "/var/lib/containerd/nvsnap-overlays" }}
               mountPropagation: Bidirectional
+            {{- if and .Values.agent.modelVolume .Values.agent.modelVolume.enabled }}
+            # Completed model volumes are bound here for readers on block
+            # storage; same path in the agent and on the host, Bidirectional
+            # so the bind reaches kubelet. Not under the overlays root: its
+            # sweeper removes entries it does not own.
+            - name: nvsnap-models
+              mountPath: {{ .Values.agent.hostPaths.nvsnapModels | default "/var/lib/containerd/nvsnap-models" }}
+              mountPropagation: Bidirectional
+            {{- end }}
             {{- if .Values.webhook.enabled }}
             - name: webhook-tls
               mountPath: /etc/nvsnap/webhook
@@ -345,6 +360,12 @@ spec:
           hostPath:
             path: {{ .Values.agent.hostPaths.checkpoints }}
             type: DirectoryOrCreate
+        {{- if and .Values.agent.modelVolume .Values.agent.modelVolume.enabled }}
+        - name: nvsnap-models
+          hostPath:
+            path: {{ .Values.agent.hostPaths.nvsnapModels | default "/var/lib/containerd/nvsnap-models" }}
+            type: DirectoryOrCreate
+        {{- end }}
         {{- if .Values.agent.podCacheDir }}
         - name: cachedir-env
           configMap:

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-rbac.yaml

@@ -41,14 +41,16 @@ metadata:
 rules:
   - apiGroups: [""]
     resources: ["pods"]
+    # patch: the model volume controller un-pends readers once their
+    # volume is bound (internal/agent/modelvolume_controller.go).
     # v0.0.51: create+delete needed for the mount-holder pod in the
     # source workload's namespace (see
     # internal/checkpointstore/mount_holder.go +
     # docs/design/ROOTFS-EVERYWHERE.md). The holder is a Kyverno-
     # compliant pause pod owned by the rwx PVC; the agent creates it
     # to trigger kubelet to mount the PVC, then deletes it after the
     # in-process file-tree copy finishes.
-    verbs: ["get", "list", "watch", "create", "delete"]
+    verbs: ["get", "list", "watch", "create", "delete", "patch"]
   - apiGroups: [""]
     resources: ["configmaps"]
     verbs: ["get", "list", "watch", "create", "update", "delete"]
@@ -63,8 +65,10 @@ rules:
   # nvsnap-system); we keep them ClusterRole for now so the agent can
   # operate in any namespace via flag.
   - apiGroups: [""]
+    # update/patch: the model volume controller labels the writer claim
+    # complete (internal/agent/modelvolume_controller.go).
     resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch", "create", "delete"]
+    verbs: ["get", "list", "watch", "create", "delete", "update", "patch"]
   # persistentvolumes (cluster-scoped): the shared-volume promoter
   # (NVMesh/EFS cachedir + ember path) Gets the writer's bound primary PV,
   # Updates its reclaim policy to Retain, and Creates a secondary

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml

@@ -114,6 +114,22 @@ agent:
   # pod is elected to capture and the rest are held by a scheduling gate
   # (no node, no GPU) until nvsnap-server sees the promote and releases
   # them. Needs agent.l2. Off until qualified on a cluster.
+  # Write-once model volume for Helm functions
+  # (docs/proposals/helm-shared-model-volume.md): the model is downloaded
+  # once per cluster into a per-identity volume and every other pod that
+  # names it attaches that volume; compile caches are shared the same way.
+  # Mode comes from the storage profile: block on NVMesh (the L2 class),
+  # rwx when the profile declares a distributed-filesystem class. Block
+  # readers reference a read-only claim by default (profile
+  # modelVolume.readerMode: pvc); readerMode: hostPath schedules readers
+  # at once and binds the volume in, for gang-scheduled charts where
+  # policy allows hostPath. Needs agent.l2. Off until qualified.
+  modelVolume:
+    enabled: false
+    # How long a reader waits for the writer's download before it downloads
+    # itself. Never a deadlock: past this, every pod is self-sufficient.
+    waitDeadline: 1h
+
   election:
     enabled: false
     # Bound on the leader's cold start plus capture. Past it the server
@@ -242,6 +258,9 @@ agent:
     checkpoints: /var/lib/containerd/nvsnap-checkpoints
     containerdSock: /run/containerd/containerd.sock
     containerdStorage: /var/lib/containerd
+    # Bind root for completed model volumes (agent.modelVolume); Bidirectional
+    # in the agent at the same path.
+    nvsnapModels: /var/lib/containerd/nvsnap-models
     # Cache, staging and overlays sit under the containerd root on purpose.
     # They are the bulk writers -- a 70B cachedir capture is ~132 GB -- and on
     # a typical GPU node /var/lib is the boot volume (network-backed, ~125

src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md

@@ -11,19 +11,19 @@ Status: design, 2026-09-26. Supersedes the gate-and-promote path of
 
 ```mermaid
 sequenceDiagram
-    participant P0 as pod-0 (writer)
-    participant P1 as pod-1..N (readers, any node, any namespace)
+    participant J as Job nvsnap-model-dl-<key>
+    participant P as pods 0..N (readers, any node, any namespace)
     participant WH as webhook
     participant A as agent
-    participant V as model volume nvsnap-model-<id>
-    WH->>WH: identity, landing path, group; elect writer (Lease)
-    WH-->>P0: init "download" -> V (rw), main mounts V ro
-    WH-->>P1: landing path -> V (DFS) or emptyDir + wait init (NVMesh)
-    Note over P0,P1: all pods schedule immediately
-    P0->>V: download; init exits 0
-    A->>V: complete: marker (DFS) / ro attach + bind into P1 (NVMesh)
-    P1->>P1: wait init sees marker, engine starts
-    Note over P0,P1: engines start together; multi-node groups form as today
+    participant V as model volume nvsnap-model-<key>
+    WH->>WH: identity, landing path, group
+    WH->>J: create (idempotent): the chart's download step, into V
+    WH-->>P: landing path -> V (DFS) or hostPath + wait init (NVMesh)
+    Note over P: all pods schedule immediately
+    J->>V: download; touch .nvsnap-complete; exit 0, volume released
+    A->>V: complete: label claim; NVMesh: ro PV + bind into P's hostPath
+    P->>P: wait init sees marker, engine starts
+    Note over P: engines start together; multi-node groups form as today
 ```
 
 ## Two artifacts, two lifecycles
@@ -55,14 +55,28 @@ not.
    volume there is already a PVC, hostPath or OCI image, skip the pod.
    Role flags and wiring env stay out of the hash (`stripRoleFlags`).
 
-2. One download step per identity per cluster (webhook + Lease). If the
-   chart downloads in an init container, that init is the download step.
-   If the engine downloads itself, the webhook injects an init
-   (`huggingface-cli download <repo>` into the landing path) and starts the
-   engine offline. The Lease `nvsnap-model-<id>` elects the writer among
-   concurrent admissions; the writer's init runs the download, its main
-   container mounts the model read-only. Every other pod is a reader and
-   its download init is replaced by a wait.
+2. One download step per identity per cluster, as a Job (webhook). The
+   webhook creates Job `nvsnap-model-dl-<key>` in the pod's namespace on
+   first sight; create is idempotent, so concurrent admissions need no
+   election. The Job's pod is the chart's own download init (image,
+   command, env, secrets, pull secrets, tolerations copied from the
+   admitted pod) wrapped to touch `<volume>/.nvsnap-complete` on success;
+   when the engine downloads itself the Job runs `hf download <repo>`
+   with the engine image and credentials. Every workload pod is a reader:
+   its download init becomes a wait for the marker, and an engine that
+   downloaded itself is started offline.
+
+   Why a Job and not the first pod: on NVMesh a volume attached read-write
+   by a running pod cannot be attached read-only anywhere else (dev1,
+   2026-09-26: `NVMesh Attach Failed` on the read-only PV while the writer
+   pod held the primary). The download step has to exit and release the
+   volume before readers attach, so it cannot live inside a pod that goes
+   on to serve. The Job's pod must also be removed after success
+   (`ttlSecondsAfterFinished`): a Succeeded pod keeps its volumes attached,
+   and on dev1 the read-only attach worked on the Job's node but failed on
+   every other node until that pod was deleted. A Job also decouples the download from the workload's
+   scheduling: it runs on any node with the image, and the workload pods
+   of a multi-node group or a gang all schedule as plain readers.
 
 3. Model volume per identity, immutable after download (agent + storage
    profile). Distributed filesystem: one RWX volume; writer and readers
@@ -73,15 +87,27 @@ not.
    namespaces (`EnsureClaim`, done) and new versions of the function all
    attach the same volume. There is no capture copy of the model anymore.
 
-4. Readers never block scheduling (webhook + agent). On a distributed
-   filesystem the RWX claim exists and is bound at admission, so the pod
-   schedules. On NVMesh, before the download is complete, a reader cannot
-   reference a bindable claim, so it gets an emptyDir at the landing path
-   plus `nvsnap-wait-model`; once complete, the agent on the reader's node
-   attaches the read-only volume (mount-holder, existing) and bind-mounts
-   it over the emptyDir, then drops the marker the wait init is polling.
-   After completion, NVMesh readers reference the read-only claim directly.
-   No pod ever needs network access to nvsnap; function namespaces block it.
+4. Readers reach the volume without help from inside the pod (webhook +
+   agent). On a distributed filesystem the RWX claim exists and is bound
+   at admission, so the pod schedules. On NVMesh the storage profile picks
+   one of two reader modes (`modelVolume.readerMode`):
+   - `pvc` (default). The reader references the read-only claim
+     `nvsnap-model-<key>-ro` in its own namespace. Complete already: the
+     webhook mints the claim at admission and the pod binds at once. Not
+     yet: the pod stays Pending on volume binding; when the Job succeeds
+     and the primary is detached, any agent mints the claim and kubelet
+     starts the pod. No hostPath, so it passes Kyverno's
+     `disallow-host-path` in NVCF function namespaces (enforced there).
+     A pod Pending on a claim holds a gang scheduler, so this mode is for
+     Deployments, StatefulSets and LWS, which is every NVCF chart today.
+   - `hostPath`. The reader gets a hostPath landing under the agent's
+     model root and schedules at once; the agent on its node attaches the
+     read-only volume (mount-holder) and bind-mounts it over the landing,
+     and the marker the wait init polls appears. For gang-scheduled
+     workloads (Grove, kai-scheduler) where policy allows hostPath.
+   In both modes the init waits for the marker and falls back to its own
+   download at the deadline. No pod ever needs network access to nvsnap;
+   function namespaces block it.
 
 5. Compile caches (webhook env + agent). All caches are redirected to a
    cache location keyed by image digest plus identity plus role-neutral
@@ -123,23 +149,26 @@ between them. Everything after that first start is a full hit.
 
 | Failure | Effect | Recovery |
 |---|---|---|
-| writer dies before complete | readers' wait reaches the Lease deadline | readers download locally (NVMesh) or into the volume (DFS; per-file atomic); Lease expires; next admission elects a new writer |
+| download Job fails or never completes | Job retries with backoff; readers' wait reaches the deadline | readers download locally (NVMesh) or into the volume (DFS; per-file atomic); the next admission recreates a missing Job |
 | volume full | writer's download fails, init restarts | same as above; retention by last use with a size budget is part of this design's follow-up, since a full volume fails every writer |
 | agent down on a reader node (NVMesh) | no bind arrives | wait deadline, local download |
 | writer pod restarts after complete | volume immutable, unaffected | none needed |
 | identity changes (revision, quantization, image) | different URI or cache key | separate volume; old one ages out |
-| gang scheduler | readers always schedulable (RWX bound, or emptyDir); writer PVC binds in seconds on Immediate storage classes | none needed |
+| gang scheduler | readers always schedulable in RWX or hostPath mode; in `pvc` mode readers pend on binding until the download completes, so gang-scheduled charts use `hostPath` | profile `readerMode: hostPath` |
+| function namespace with Kyverno enforced (`disallow-host-path`, requests and limits, no SA token) | `pvc` mode uses no hostPath; the download Job carries every mount the chart's init had (registry key secret, script ConfigMap), default requests and limits, seccomp, dropped capabilities and no token | none needed |
+| identity deleted while a read-only PV is still Terminating (NVMesh) | the read-only PV name is deterministic per identity and namespace, so a re-download of the same identity cannot mint until the old PV finalizes; the attacher's detach timed out for minutes after the volume was gone | retention deletes read-only claims and PVs before the primary, and the controller retries minting; a stale VolumeAttachment on a deleted volume needs the finalizer cleared (seen on dev1 2026-09-26) |
+| last reader of an identity leaves a node (NVMesh) | the agent's bind mount keeps the volume published; kubelet cannot unmount and the attacher's detach times out (seen on dev1 2026-09-26 during cleanup) | the agent must unbind and drop its mount-holder when no pod on the node uses the identity; part of retention (follow-up) |
 
 ## What changes in the code
 
-Stays: classifier (extended per mechanism 1), role-neutral hash, Lease
-election (elects the writer), `EnsureClaim`, storage profiles, cache env
+Stays: classifier (extended per mechanism 1), role-neutral hash,
+`EnsureClaim`, storage profiles, cache env
 injection and seed init, mount-holder and bind injection (L1), the server
 reconciler, `vllm-workers` chart and runner.
 
-New: identity from init containers and group inheritance; download-init
-injection for engine-internal downloads; init wrapping for marker and
-wait; per-identity model volume created at admission from the profile's
+New: identity from init containers and group inheritance; the download
+Job derived from the chart's init or from `hf download`; init wrapping
+for the wait; per-identity model volume created at admission from the profile's
 class (RWX on DFS, RWO writer PVC on NVMesh); agent completion handler
 (init exit 0 -> marker / ro PV + bind); cache volume capture (caches only)
 on NVMesh; `cacheMode`; last-use labels.
@@ -158,3 +187,61 @@ Removed for Helm: `schedulingGates`, promote-to-ROX of the whole tree,
 5. Retire the gate; e2e on dev1 in all matrix rows that dev1 can host
    (NVMesh; DFS stands in with an NFS class), each measured cold, first
    deploy with two pods per instance, redeploy, second namespace.
+
+## Results, dev1 2026-09-26 (NVMesh, block mode)
+
+Stock `vllm-workers` chart, Qwen2.5-32B-Instruct TP=4, replicas=2 on two
+nodes, no nvsnap markers in the chart, agent v0.2.76-mv5.
+
+```
+first deploy (nothing on the cluster)
+  t+0       both pods admitted as readers (hostPath landing, wait init); one Job created
+  t+354s    Job succeeded: 65 GB via `hf download` into the RWO claim; claim released
+  t+400s    both readers un-pended (read-only PV minted per namespace, attached via
+            mount-holder, bound under /var/lib/containerd/nvsnap-models/<key>)
+  t+591s    both Ready; 0 downloads in either pod; serve " Paris. Correct!"
+uninstall + reinstall
+  t+12s     both un-pended (identity complete: no Job)
+  t+136s    both Ready
+earlier run, identity already complete on the cluster
+  t+205s    both Ready, wait init 10-15 s, engine reads the volume directly
+```
+
+PVC reader mode (the default since the function-namespace fixes), same
+chart, Qwen2.5-14B-Instruct TP=4, replicas=2, agent v0.2.76-mv6:
+
+```
+first deploy (fresh identity)
+  t+0       both pods admitted as readers referencing nvsnap-model-<key>-ro; Pending on
+            "persistentvolumeclaim not found"; one Job created (2 CPU / 4 Gi requests,
+            no SA token, seccomp, dropped capabilities)
+  t+135s    Job succeeded: 28 GB via `hf download`
+  t+195s    read-only PV and claim minted in the pod namespace after the primary
+            detached; both readers un-pended and scheduled; init found the marker
+  t+344s    both Ready; 0 downloads; /root/.cache/huggingface is the NVMesh volume
+            mounted ro,norecovery,nouuid; serve " Paris. The capital"
+uninstall + reinstall
+  t+118s    both Ready (claim minted at admission, no pending phase)
+```
+
+No hostPath and no agent bind in this mode. Kyverno on dev1 is Audit, so
+the remaining warnings were the stock chart's own containers plus, until
+v0.2.76-mv8, the injected init (no resources, capabilities not dropped,
+no runAsNonRoot); the enforced rejection itself is not testable on dev1.
+
+Cold start of the same pod on the same node: 325 s. The reinstall number
+is the engine's own load and compile from a read-only NVMesh mount with no
+prewarm; compile caches on block storage are still the follow-up (step 4).
+
+Findings that changed the design during these runs:
+
+- NVMesh refuses a read-only attach on any node while the volume is
+  attached read-write anywhere, including a Succeeded Job pod that still
+  exists. Hence the download Job, its TTL, releasing the claim on
+  completion, and the detach check before minting.
+- The overlays root is swept by the L1 overlay GC; binds live under their
+  own Bidirectional hostPath.
+- Memory is not a mount table: binds are verified against the mounted
+  device and the volume handle and redone when missing or stale.
+- The agent's binds pin the volume on the node; unbinding when the last
+  reader leaves is part of retention (open).