From 86ef3547d54e05552c2c5410de9bdd5a047b5276 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 07:26:43 -0700 Subject: [PATCH 01/16] feat(nvsnap): resolve model identity and landing volume from any chart First step of docs/proposals/helm-shared-model-volume.md. For any GPU pod at admission, internal/modelid answers which artifact the pod will download, as a URI shared by every chart, version and namespace that names it, and where the bytes land and what backs that path today. Identity sources, as the field uses them: engine args (--model, --model-path, --model=, positional `vllm serve `, --revision), engine env (HF_MODEL_ID, MODEL_ID, MODEL_PATH, with $VAR expansion from the container's own env), download init containers (NGC CLI with NGC_MODEL_NAME, huggingface-cli download with --local-dir, aws s3 sync, KServe storage-initializer args), NIM images with NIM_MODEL_PROFILE, and for LeaderWorkerSet workers that name no model the leader template of their group. URIs: hf://org/repo[@rev], ngc://org/team/model:ver, s3://bucket/key, nim://image@profile, path:///abs for pre-filled paths. The landing volume is the mount at or above the download destination in the main container: emptyDir or rootfs are substitutable, a customer's PVC, hostPath or other volume means sharing is already solved and the pod is left alone. KServe pvc:// is likewise not a downloader. Tests are built from real specs: the NVCF Helm function observed on prd11 (NGC init download, positional MODEL_PATH), the Dynamo operator sample, SGLang, NIM, KServe, huggingface-cli and S3 inits, the upstream LWS vLLM example, and non-downloaders (Dynamo frontend, Ray worker, etcd). Mutation-checked: PVC marked substitutable, revision split dropped and env expansion dropped each turn tests red. Co-Authored-By: Balaji Ganesan --- .../nvsnap/internal/modelid/BUILD.bazel | 33 ++ .../nvsnap/internal/modelid/group.go | 90 ++++ .../nvsnap/internal/modelid/modelid.go | 387 ++++++++++++++++++ .../nvsnap/internal/modelid/modelid_test.go | 244 +++++++++++ 4 files changed, 754 insertions(+) create mode 100644 src/compute-plane-services/nvsnap/internal/modelid/BUILD.bazel create mode 100644 src/compute-plane-services/nvsnap/internal/modelid/group.go create mode 100644 src/compute-plane-services/nvsnap/internal/modelid/modelid.go create mode 100644 src/compute-plane-services/nvsnap/internal/modelid/modelid_test.go diff --git a/src/compute-plane-services/nvsnap/internal/modelid/BUILD.bazel b/src/compute-plane-services/nvsnap/internal/modelid/BUILD.bazel new file mode 100644 index 0000000000..091796aaf9 --- /dev/null +++ b/src/compute-plane-services/nvsnap/internal/modelid/BUILD.bazel @@ -0,0 +1,33 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "modelid", + srcs = [ + "group.go", + "modelid.go", + ], + importpath = "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/modelid", + visibility = ["//src/compute-plane-services/nvsnap:__subpackages__"], + deps = [ + "@io_k8s_api//core/v1:core", + "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured", + "@io_k8s_apimachinery//pkg/runtime", + "@io_k8s_apimachinery//pkg/runtime/schema", + "@io_k8s_client_go//dynamic", + ], +) + +go_test( + name = "modelid_test", + srcs = ["modelid_test.go"], + embed = [":modelid"], + deps = [ + "@io_k8s_api//core/v1:core", + "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured", + "@io_k8s_apimachinery//pkg/runtime", + "@io_k8s_apimachinery//pkg/runtime/schema", + "@io_k8s_client_go//dynamic/fake", + ], +) diff --git a/src/compute-plane-services/nvsnap/internal/modelid/group.go b/src/compute-plane-services/nvsnap/internal/modelid/group.go new file mode 100644 index 0000000000..647b4338ad --- /dev/null +++ b/src/compute-plane-services/nvsnap/internal/modelid/group.go @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package modelid + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" +) + +// Group inheritance. A LeaderWorkerSet worker runs `vllm serve --headless` +// or `ray start --block` and names no model; the leader template does. +// The worker still needs the same bytes on its node before the group can +// form, so it inherits the leader's identity. StatefulSet and Dynamo +// members carry their own model argument and never reach this path. + +// LWS pod labels and the LeaderWorkerSet resource. +const ( + lwsNameLabel = "leaderworkerset.sigs.k8s.io/name" + lwsWorkerIndexLabel = "leaderworkerset.sigs.k8s.io/worker-index" +) + +var lwsGVR = schema.GroupVersionResource{Group: "leaderworkerset.x-k8s.io", Version: "v1", Resource: "leaderworkersets"} + +// GroupResolver finds the identity of the group a pod belongs to. +type GroupResolver interface { + // ResolveGroup returns the group's result and true when pod is a + // member of a group whose leader template names a model. + ResolveGroup(ctx context.Context, pod *corev1.Pod) (Result, bool, error) +} + +// LWSResolver reads the LeaderWorkerSet's leader template. +type LWSResolver struct { + Dyn dynamic.Interface +} + +// ResolveGroup implements GroupResolver for LWS workers. +func (r *LWSResolver) ResolveGroup(ctx context.Context, pod *corev1.Pod) (Result, bool, error) { + if r == nil || r.Dyn == nil || pod == nil { + return Result{}, false, nil + } + name := pod.Labels[lwsNameLabel] + if name == "" { + return Result{}, false, nil + } + if idx := pod.Labels[lwsWorkerIndexLabel]; idx == "0" || idx == "" { + return Result{}, false, nil // the leader resolves on its own + } + lws, err := r.Dyn.Resource(lwsGVR).Namespace(pod.Namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return Result{}, false, fmt.Errorf("get LeaderWorkerSet %s/%s: %w", pod.Namespace, name, err) + } + tmpl, found, err := unstructured.NestedMap(lws.Object, "spec", "leaderWorkerTemplate", "leaderTemplate") + if err != nil || !found { + return Result{}, false, nil + } + var leader corev1.PodTemplateSpec + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(tmpl, &leader); err != nil { + return Result{}, false, fmt.Errorf("decode leader template: %w", err) + } + leaderPod := &corev1.Pod{ObjectMeta: leader.ObjectMeta, Spec: leader.Spec} + res, ok := Resolve(leaderPod, 0) + if !ok { + return Result{}, false, nil + } + // The worker's own landing volume is what it mounts; the leader's + // tells us only the identity and the path convention. + res.Landing = landingFor(pod, &pod.Spec.Containers[0], res.Landing.Path, res.Landing.Downloader, "") + res.Source = "lws leader template: " + res.Source + return res, true, nil +} + +// ResolveWithGroup is Resolve followed by group inheritance for pods that +// name no model themselves. +func ResolveWithGroup(ctx context.Context, pod *corev1.Pod, mainContainer int, groups GroupResolver) (Result, bool, error) { + if res, ok := Resolve(pod, mainContainer); ok { + return res, true, nil + } + if groups == nil { + return Result{}, false, nil + } + return groups.ResolveGroup(ctx, pod) +} diff --git a/src/compute-plane-services/nvsnap/internal/modelid/modelid.go b/src/compute-plane-services/nvsnap/internal/modelid/modelid.go new file mode 100644 index 0000000000..d8f7cbf9b7 --- /dev/null +++ b/src/compute-plane-services/nvsnap/internal/modelid/modelid.go @@ -0,0 +1,387 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package modelid answers, for any GPU pod at admission, the first two of +// the four questions in docs/proposals/helm-shared-model-volume.md: +// +// - Identity: which model artifact will this pod download, as a URI +// (hf://org/repo[@rev], ngc://org/team/model:ver, s3://bucket/path, +// nim://image[@profile]) so every chart, version and namespace that +// names the same artifact shares one volume. +// - Landing: where inside the pod the download writes and what volume +// backs that path today, so the webhook knows what to substitute and +// when to leave the pod alone. +// +// Sources, in the order the field uses them: the engine's own arguments +// (--model, --model-path, positional `vllm serve `), engine env +// (HF_MODEL_ID, MODEL_ID, MODEL_PATH), an init container that downloads +// (NGC CLI, huggingface-cli, aws s3, KServe storage-initializer), a NIM +// image, and for group members with no identity of their own (LWS +// workers) the group's leader template. +package modelid + +import ( + "path" + "regexp" + "sort" + "strings" + + corev1 "k8s.io/api/core/v1" +) + +// Identity is the normalized artifact reference. +type Identity struct { + Scheme string // hf, ngc, s3, nim, path + Ref string // org/repo, org/team/model:ver, bucket/key, image, /abs/path + Revision string // hf revision when given + Profile string // NIM_MODEL_PROFILE when given +} + +// URI renders the identity as the cache key everything else hangs off. +func (id Identity) URI() string { + if id.Scheme == "" || id.Ref == "" { + return "" + } + s := id.Scheme + "://" + id.Ref + if id.Revision != "" { + s += "@" + id.Revision + } + if id.Profile != "" { + s += "@" + id.Profile + } + return s +} + +// Downloader says which container performs the download. +type Downloader string + +// Downloaders: the engine fetches at start (vllm serve --model X), an init +// container fetches before the engine, or the bytes are already present +// (PVC, hostPath, image). +const ( + DownloaderEngine Downloader = "engine" + DownloaderInit Downloader = "init" + DownloaderNone Downloader = "none" +) + +// VolumeKind classifies what backs the landing path. +type VolumeKind string + +// Volume kinds behind the landing path. emptyDir and rootfs are ours to +// replace; a PVC, hostPath or other volume is the customer's own storage. +const ( + VolumeEmptyDir VolumeKind = "emptyDir" // substitutable + VolumeRootfs VolumeKind = "rootfs" // no volume: container filesystem, substitutable + VolumePVC VolumeKind = "pvc" // customer's own shared storage: skip + VolumeHostPath VolumeKind = "hostPath" // customer's node cache: skip + VolumeOther VolumeKind = "other" // image volume, CSI ephemeral, etc.: skip +) + +// Landing is where the download writes and what is under it. +type Landing struct { + // Path is the directory the download populates inside the container. + Path string + // VolumeName is the pod volume mounted at (or above) Path; empty for rootfs. + VolumeName string + // MountPath is that volume's mount path; empty for rootfs. + MountPath string + Kind VolumeKind + // InitContainer is the name of the downloading init, when Downloader is init. + InitContainer string + Downloader Downloader +} + +// Substitutable reports whether the webhook may replace the landing volume +// with the shared model volume. A PVC, hostPath or image volume means the +// customer already solved sharing; leave it alone. +func (l Landing) Substitutable() bool { + return l.Kind == VolumeEmptyDir || l.Kind == VolumeRootfs +} + +// Result is what Resolve returns for a pod. +type Result struct { + Identity Identity + Landing Landing + // Source names where the identity came from, for logs. + Source string +} + +const ( + defaultHFHome = "/root/.cache/huggingface" + kserveDest = "/mnt/models" +) + +var ( + modelFlagRe = regexp.MustCompile(`(?:^|\s)--model(?:-path)?(?:=|\s+)(?:'([^']+)'|"([^"]+)"|([^\s\\'"]+))`) + revisionRe = regexp.MustCompile(`(?:^|\s)--revision(?:=|\s+)([^\s\\'"]+)`) + servePosRe = regexp.MustCompile(`\bvllm\s+serve\s+(?:'([^']+)'|"([^"]+)"|([^\s\\'"-][^\s\\'"]*))`) + ngcDownloadRe = regexp.MustCompile(`ngc\s+registry\s+model\s+download-version(?:\s+--\S+(?:\s+[^\s-]\S*)?)*\s+"?([A-Za-z0-9_./-]+:[A-Za-z0-9_.-]+)"?`) + ngcDestRe = regexp.MustCompile(`--dest(?:=|\s+)"?([^\s"]+)"?`) + hfCLIRe = regexp.MustCompile(`(?:huggingface-cli|hf)\s+download\s+(?:--\S+\s+\S+\s+)*([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)`) + hfLocalDirRe = regexp.MustCompile(`--local-dir(?:=|\s+)"?([^\s"]+)"?`) + s3Re = regexp.MustCompile(`s3://([A-Za-z0-9_./-]+)`) + shellVarRe = regexp.MustCompile(`\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?`) +) + +// Resolve derives identity and landing for the pod's main container. +// ok is false when the pod names no model: it is not a downloader and the +// webhook leaves it alone. +func Resolve(pod *corev1.Pod, mainContainer int) (Result, bool) { + if pod == nil || mainContainer < 0 || mainContainer >= len(pod.Spec.Containers) { + return Result{}, false + } + main := pod.Spec.Containers[mainContainer] + // 1. An init container that downloads is the strongest signal: it + // names the artifact and the destination explicitly. + for i := range pod.Spec.InitContainers { + if r, ok := fromInit(pod, &pod.Spec.InitContainers[i], &main); ok { + return r, true + } + } + // 2. The engine downloads itself. + if id, src, ok := fromEngine(&main); ok { + land := landingFor(pod, &main, engineCachePath(&main), DownloaderEngine, "") + return Result{Identity: id, Landing: land, Source: src}, true + } + return Result{}, false +} + +// fromInit recognizes the download init containers seen in the field. +func fromInit(pod *corev1.Pod, init, main *corev1.Container) (Result, bool) { + env := envMap(init) + text := joinArgv(init, env) + // NVCF Helm functions: NGC CLI download with the artifact in env. + if name := env["NGC_MODEL_NAME"]; name != "" { + dest := firstNonEmpty(env["NGC_MODEL_MOUNT"], ngcDest(text), mountRoot(init)) + return Result{Identity: Identity{Scheme: "ngc", Ref: name}, Landing: landingFor(pod, main, dest, DownloaderInit, init.Name), Source: "init env NGC_MODEL_NAME"}, true + } + if m := ngcDownloadRe.FindStringSubmatch(text); m != nil { + dest := firstNonEmpty(ngcDest(text), env["NGC_MODEL_MOUNT"], mountRoot(init)) + return Result{Identity: Identity{Scheme: "ngc", Ref: m[1]}, Landing: landingFor(pod, main, dest, DownloaderInit, init.Name), Source: "init ngc registry model download-version"}, true + } + if m := hfCLIRe.FindStringSubmatch(text); m != nil { + dest := firstNonEmpty(hfLocalDir(text), env["HF_HOME"], mountRoot(init)) + id := Identity{Scheme: "hf", Ref: m[1]} + if r := revisionRe.FindStringSubmatch(text); r != nil { + id.Revision = r[1] + } + return Result{Identity: id, Landing: landingFor(pod, main, dest, DownloaderInit, init.Name), Source: "init huggingface-cli download"}, true + } + if m := s3Re.FindStringSubmatch(text); m != nil && (strings.Contains(text, "s3 sync") || strings.Contains(text, "s3 cp") || strings.Contains(text, "s5cmd")) { + return Result{Identity: Identity{Scheme: "s3", Ref: strings.TrimSuffix(m[1], "/")}, Landing: landingFor(pod, main, mountRoot(init), DownloaderInit, init.Name), Source: "init s3"}, true + } + // KServe storage-initializer: args [storageUri, destDir]. + if strings.Contains(init.Image, "storage-initializer") && len(init.Args) >= 2 { + if id, ok := parseURI(init.Args[0]); ok { + return Result{Identity: id, Landing: landingFor(pod, main, init.Args[1], DownloaderInit, init.Name), Source: "kserve storage-initializer"}, true + } + if strings.HasPrefix(init.Args[0], "pvc://") { + return Result{Identity: Identity{}, Landing: Landing{Path: init.Args[1], Kind: VolumePVC, Downloader: DownloaderNone}, Source: "kserve pvc"}, false + } + } + return Result{}, false +} + +// fromEngine reads the engine's own arguments and env. +func fromEngine(main *corev1.Container) (Identity, string, bool) { + env := envMap(main) + text := joinArgv(main, env) + rev := "" + if r := revisionRe.FindStringSubmatch(text); r != nil { + rev = r[1] + } + if m := modelFlagRe.FindStringSubmatch(text); m != nil { + return classifyRef(first(m[1:]), rev, env), "--model", true + } + if m := servePosRe.FindStringSubmatch(text); m != nil { + return classifyRef(first(m[1:]), rev, env), "vllm serve ", true + } + for _, k := range []string{"HF_MODEL_ID", "MODEL_ID"} { + if v := env[k]; v != "" { + return classifyRef(v, rev, env), "env " + k, true + } + } + if v := env["MODEL_PATH"]; v != "" { + return classifyRef(v, rev, env), "env MODEL_PATH", true + } + if isNIMImage(main.Image) { + return Identity{Scheme: "nim", Ref: main.Image, Profile: env["NIM_MODEL_PROFILE"]}, "nim image", true + } + return Identity{}, "", false +} + +// classifyRef turns a --model value into an identity: an absolute path is +// a local path (an init filled it, or it is baked into the image); a +// URI keeps its scheme; anything else is a Hugging Face repo id. +func classifyRef(ref, rev string, env map[string]string) Identity { + ref = expandEnv(ref, env) + if id, ok := parseURI(ref); ok { + return id + } + if strings.HasPrefix(ref, "/") { + return Identity{Scheme: "path", Ref: path.Clean(ref)} + } + return Identity{Scheme: "hf", Ref: ref, Revision: rev} +} + +func parseURI(s string) (Identity, bool) { + for _, scheme := range []string{"hf", "ngc", "s3", "gs", "oci"} { + if strings.HasPrefix(s, scheme+"://") { + ref := strings.TrimPrefix(s, scheme+"://") + id := Identity{Scheme: scheme, Ref: strings.TrimSuffix(ref, "/")} + if scheme == "hf" { + if i := strings.LastIndex(id.Ref, "@"); i > 0 { + id.Revision, id.Ref = id.Ref[i+1:], id.Ref[:i] + } + } + return id, true + } + } + return Identity{}, false +} + +// engineCachePath is where an engine-internal download lands. +func engineCachePath(main *corev1.Container) string { + env := envMap(main) + if isNIMImage(main.Image) && env["NIM_CACHE_PATH"] != "" { + return env["NIM_CACHE_PATH"] + } + if v := env["HF_HOME"]; v != "" { + return v + } + if v := env["HF_HUB_CACHE"]; v != "" { + return v + } + return defaultHFHome +} + +// landingFor finds the volume mounted at or above dest in the main +// container and classifies it. The download init may mount the volume +// under a different path; the main container's view is what the engine +// reads, so it is the reference. +func landingFor(pod *corev1.Pod, main *corev1.Container, dest string, dl Downloader, initName string) Landing { + dest = path.Clean(dest) + l := Landing{Path: dest, Kind: VolumeRootfs, Downloader: dl, InitContainer: initName} + best := "" + for _, vm := range main.VolumeMounts { + mp := path.Clean(vm.MountPath) + if (dest == mp || strings.HasPrefix(dest, mp+"/")) && len(mp) > len(best) { + best = mp + l.VolumeName, l.MountPath = vm.Name, mp + } + } + if l.VolumeName == "" { + return l + } + for i := range pod.Spec.Volumes { + v := &pod.Spec.Volumes[i] + if v.Name != l.VolumeName { + continue + } + switch { + case v.EmptyDir != nil: + l.Kind = VolumeEmptyDir + case v.PersistentVolumeClaim != nil: + l.Kind = VolumePVC + case v.HostPath != nil: + l.Kind = VolumeHostPath + default: + l.Kind = VolumeOther + } + } + return l +} + +// mountRoot is the first non-system mount of a container, the usual +// destination of a download init that does not say where it writes. +func mountRoot(c *corev1.Container) string { + paths := []string{} + for _, vm := range c.VolumeMounts { + if strings.HasPrefix(vm.MountPath, "/var/run/secrets") || vm.MountPath == "/dev/shm" { + continue + } + paths = append(paths, vm.MountPath) + } + sort.Strings(paths) + if len(paths) == 0 { + return kserveDest + } + return paths[0] +} + +func ngcDest(text string) string { + if m := ngcDestRe.FindStringSubmatch(text); m != nil { + return m[1] + } + return "" +} + +func hfLocalDir(text string) string { + if m := hfLocalDirRe.FindStringSubmatch(text); m != nil { + return m[1] + } + return "" +} + +func envMap(c *corev1.Container) map[string]string { + m := map[string]string{} + for _, e := range c.Env { + if e.Value != "" { + m[e.Name] = e.Value + } + } + return m +} + +// joinArgv flattens command+args into one string and expands $VAR from +// the container's own literal env, so `vllm serve ${MODEL_PATH}` resolves. +func joinArgv(c *corev1.Container, env map[string]string) string { + parts := append(append([]string{}, c.Command...), c.Args...) + return expandEnv(strings.Join(parts, " "), env) +} + +func expandEnv(s string, env map[string]string) string { + return shellVarRe.ReplaceAllStringFunc(s, func(tok string) string { + name := shellVarRe.FindStringSubmatch(tok)[1] + if v, ok := env[name]; ok { + return v + } + return tok + }) +} + +func isNIMImage(image string) bool { + return strings.Contains(image, "nvcr.io/nim/") || strings.Contains(image, "/nim/") +} + +func first(groups []string) string { + for _, g := range groups { + if g != "" { + return g + } + } + return "" +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "" +} diff --git a/src/compute-plane-services/nvsnap/internal/modelid/modelid_test.go b/src/compute-plane-services/nvsnap/internal/modelid/modelid_test.go new file mode 100644 index 0000000000..5e2fd6ad2d --- /dev/null +++ b/src/compute-plane-services/nvsnap/internal/modelid/modelid_test.go @@ -0,0 +1,244 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package modelid + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" +) + +func emptyDir(name string) corev1.Volume { + return corev1.Volume{Name: name, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}} +} + +// The NVCF Helm function observed on prd11 (2026-09-25): NGC CLI download +// in an init container into an emptyDir, engine started from a positional +// path taken from MODEL_PATH, no --model anywhere. +func prd11Function() *corev1.Pod { + return &corev1.Pod{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{ + Name: "download-ngc-model", Image: "nvcr.io/org/llm_nim/ultra:vllm", + Command: []string{"/bin/bash", "-c"}, + Args: []string{"set -euo pipefail\n/tmp/ngc-cli/ngc registry model download-version --dest \"${NGC_MODEL_MOUNT}\" \"${NGC_MODEL_NAME}\"\n"}, + Env: []corev1.EnvVar{ + {Name: "NGC_MODEL_NAME", Value: "qc69jvmznzxy/llm_nim/nemotron3-ultra-genrm:bf16-fixed"}, + {Name: "NGC_MODEL_MOUNT", Value: "/config/models"}, + {Name: "NGC_STABLE_MODEL_PATH", Value: "/config/models/nemotron3-ultra-genrm"}, + }, + VolumeMounts: []corev1.VolumeMount{{Name: "ngc-models", MountPath: "/config/models"}}, + }}, + Containers: []corev1.Container{{ + Name: "kimi-k3", Image: "nvcr.io/org/llm_nim/ultra:vllm", + Command: []string{"/bin/bash", "/opt/kimi-k3/start.sh"}, + Env: []corev1.EnvVar{ + {Name: "MODEL_PATH", Value: "/config/models/nemotron3-ultra-genrm"}, + {Name: "HF_HUB_OFFLINE", Value: "1"}, + }, + VolumeMounts: []corev1.VolumeMount{{Name: "dshm", MountPath: "/dev/shm"}, {Name: "scripts", MountPath: "/opt/kimi-k3", ReadOnly: true}, {Name: "ngc-models", MountPath: "/config/models"}}, + }}, + Volumes: []corev1.Volume{emptyDir("dshm"), {Name: "scripts", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{}}}, emptyDir("ngc-models")}, + }} +} + +func TestResolve_NVCFFunction_NGCInitDownload(t *testing.T) { + r, ok := Resolve(prd11Function(), 0) + if !ok { + t.Fatal("prd11 function must resolve") + } + if r.Identity.URI() != "ngc://qc69jvmznzxy/llm_nim/nemotron3-ultra-genrm:bf16-fixed" { + t.Errorf("identity = %q", r.Identity.URI()) + } + l := r.Landing + if l.Downloader != DownloaderInit || l.InitContainer != "download-ngc-model" || l.Path != "/config/models" || l.VolumeName != "ngc-models" || l.Kind != VolumeEmptyDir || !l.Substitutable() { + t.Errorf("landing = %+v", l) + } +} + +func TestResolve_EngineDownloads(t *testing.T) { + cases := map[string]struct { + c corev1.Container + vols []corev1.Volume + wantURI string + wantSrc string + path string + kind VolumeKind + }{ + "stock vllm, default HF_HOME on rootfs": { + c: corev1.Container{Image: "vllm/vllm-openai:v0.20.0", Command: []string{"/bin/bash", "-lc"}, Args: []string{"vllm serve --model Qwen/Qwen2.5-32B-Instruct --tensor-parallel-size 4 > /vllm.out 2>&1 &"}}, + wantURI: "hf://Qwen/Qwen2.5-32B-Instruct", wantSrc: "--model", path: "/root/.cache/huggingface", kind: VolumeRootfs, + }, + "positional model with revision, HF_HOME on emptyDir": { + c: corev1.Container{Image: "vllm/vllm-openai", Args: []string{"vllm serve meta-llama/Llama-3.1-70B-Instruct --revision abc123 --tensor-parallel-size 4"}, + Env: []corev1.EnvVar{{Name: "HF_HOME", Value: "/models/hf"}}, VolumeMounts: []corev1.VolumeMount{{Name: "hf", MountPath: "/models"}}}, + vols: []corev1.Volume{emptyDir("hf")}, + wantURI: "hf://meta-llama/Llama-3.1-70B-Instruct@abc123", wantSrc: "vllm serve ", path: "/models/hf", kind: VolumeEmptyDir, + }, + "dynamo list form": { + c: corev1.Container{Image: "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.1.1", Command: []string{"python3", "-m", "dynamo.vllm"}, Args: []string{"--model", "Qwen/Qwen3-0.6B", "--is-prefill-worker"}}, + wantURI: "hf://Qwen/Qwen3-0.6B", wantSrc: "--model", path: "/root/.cache/huggingface", kind: VolumeRootfs, + }, + "sglang model-path": { + c: corev1.Container{Image: "lmsysorg/sglang", Args: []string{"python3 -m sglang.launch_server --model-path google/gemma-4-31B-it --tp 2"}}, + wantURI: "hf://google/gemma-4-31B-it", wantSrc: "--model", path: "/root/.cache/huggingface", kind: VolumeRootfs, + }, + "customer already mounts a PVC at HF_HOME": { + c: corev1.Container{Image: "vllm/vllm-openai", Args: []string{"--model", "Qwen/Qwen3-0.6B"}, VolumeMounts: []corev1.VolumeMount{{Name: "cache", MountPath: "/root/.cache/huggingface"}}}, + vols: []corev1.Volume{{Name: "cache", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "models"}}}}, + wantURI: "hf://Qwen/Qwen3-0.6B", wantSrc: "--model", path: "/root/.cache/huggingface", kind: VolumePVC, + }, + "NIM image with profile": { + c: corev1.Container{Image: "nvcr.io/nim/meta/llama-3.1-8b-instruct:1.8.3", Env: []corev1.EnvVar{{Name: "NIM_CACHE_PATH", Value: "/opt/nim/.cache"}, {Name: "NIM_MODEL_PROFILE", Value: "tensorrt_llm-h100-fp8"}}}, + wantURI: "nim://nvcr.io/nim/meta/llama-3.1-8b-instruct:1.8.3@tensorrt_llm-h100-fp8", wantSrc: "nim image", path: "/opt/nim/.cache", kind: VolumeRootfs, + }, + "HF_MODEL_ID env": { + c: corev1.Container{Image: "x", Command: []string{"/start.sh"}, Env: []corev1.EnvVar{{Name: "HF_MODEL_ID", Value: "openai/whisper-large-v3"}}}, + wantURI: "hf://openai/whisper-large-v3", wantSrc: "env HF_MODEL_ID", path: "/root/.cache/huggingface", kind: VolumeRootfs, + }, + } + for name, tc := range cases { + pod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{tc.c}, Volumes: tc.vols}} + r, ok := Resolve(pod, 0) + if !ok { + t.Errorf("%s: must resolve", name) + continue + } + if r.Identity.URI() != tc.wantURI || r.Source != tc.wantSrc { + t.Errorf("%s: identity %q via %q, want %q via %q", name, r.Identity.URI(), r.Source, tc.wantURI, tc.wantSrc) + } + if r.Landing.Path != tc.path || r.Landing.Kind != tc.kind || r.Landing.Downloader != DownloaderEngine { + t.Errorf("%s: landing %+v, want path %s kind %s", name, r.Landing, tc.path, tc.kind) + } + if tc.kind == VolumePVC && r.Landing.Substitutable() { + t.Errorf("%s: a customer PVC must not be substitutable", name) + } + } +} + +func TestResolve_OtherInitDownloaders(t *testing.T) { + hf := &corev1.Pod{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "fetch", Image: "python:3", Command: []string{"sh", "-c", "huggingface-cli download --revision v2 meta-llama/Llama-3.1-8B-Instruct --local-dir /models/llama"}, VolumeMounts: []corev1.VolumeMount{{Name: "m", MountPath: "/models"}}}}, + Containers: []corev1.Container{{Name: "vllm", Image: "vllm/vllm-openai", Args: []string{"vllm serve /models/llama"}, VolumeMounts: []corev1.VolumeMount{{Name: "m", MountPath: "/models"}}}}, + Volumes: []corev1.Volume{emptyDir("m")}, + }} + r, ok := Resolve(hf, 0) + if !ok || r.Identity.URI() != "hf://meta-llama/Llama-3.1-8B-Instruct@v2" || r.Landing.Path != "/models/llama" || r.Landing.VolumeName != "m" || r.Landing.InitContainer != "fetch" { + t.Errorf("huggingface-cli init: ok=%v %+v", ok, r) + } + s3 := &corev1.Pod{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "sync", Image: "amazon/aws-cli", Args: []string{"s3", "sync", "s3://my-bucket/models/llama-70b/", "/mnt/models"}, VolumeMounts: []corev1.VolumeMount{{Name: "m", MountPath: "/mnt/models"}}}}, + Containers: []corev1.Container{{Name: "vllm", Image: "vllm/vllm-openai", Env: []corev1.EnvVar{{Name: "MODEL_PATH", Value: "/mnt/models"}}, Command: []string{"sh", "-c", "vllm serve $MODEL_PATH"}, VolumeMounts: []corev1.VolumeMount{{Name: "m", MountPath: "/mnt/models"}}}}, + Volumes: []corev1.Volume{emptyDir("m")}, + }} + if r, ok := Resolve(s3, 0); !ok || r.Identity.URI() != "s3://my-bucket/models/llama-70b" || r.Landing.Path != "/mnt/models" { + t.Errorf("s3 init: ok=%v %+v", ok, r) + } + kserve := &corev1.Pod{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "storage-initializer", Image: "kserve/storage-initializer:v0.15", Args: []string{"hf://Qwen/Qwen3-0.6B", "/mnt/models"}, VolumeMounts: []corev1.VolumeMount{{Name: "kserve-provision-location", MountPath: "/mnt/models"}}}}, + Containers: []corev1.Container{{Name: "kserve-container", Image: "kserve/vllm", Args: []string{"--model_dir=/mnt/models"}, VolumeMounts: []corev1.VolumeMount{{Name: "kserve-provision-location", MountPath: "/mnt/models", ReadOnly: true}}}}, + Volumes: []corev1.Volume{emptyDir("kserve-provision-location")}, + }} + if r, ok := Resolve(kserve, 0); !ok || r.Identity.URI() != "hf://Qwen/Qwen3-0.6B" || r.Landing.Path != "/mnt/models" || r.Landing.Kind != VolumeEmptyDir { + t.Errorf("kserve hf: ok=%v %+v", ok, r) + } + kserve.Spec.InitContainers[0].Args[0] = "pvc://models/qwen" + if _, ok := Resolve(kserve, 0); ok { + t.Error("kserve pvc:// means the customer shares already; must not resolve as a downloader") + } +} + +// URIs carry the revision after '@'; engine argv expands the container's +// own env so `vllm serve ${MODEL_PATH}` resolves to the path, not to the +// literal. +func TestResolve_URIRevisionAndEnvExpansion(t *testing.T) { + id, ok := parseURI("hf://Qwen/Qwen3-0.6B@v1") + if !ok || id.Ref != "Qwen/Qwen3-0.6B" || id.Revision != "v1" || id.URI() != "hf://Qwen/Qwen3-0.6B@v1" { + t.Errorf("parseURI revision: ok=%v %+v", ok, id) + } + pod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Image: "vllm/vllm-openai", Command: []string{"sh", "-c", "exec vllm serve ${MODEL_PATH} --port 8000"}, + Env: []corev1.EnvVar{{Name: "MODEL_PATH", Value: "/models/llama"}}, + }}}} + r, ok := Resolve(pod, 0) + if !ok || r.Identity.URI() != "path:///models/llama" { + t.Errorf("env-expanded positional model: ok=%v %q", ok, r.Identity.URI()) + } +} + +// An init's own env is expanded before the argv is read, so `--dest +// ${DEST}` yields the real path when no NGC_MODEL_MOUNT is set. +func TestResolve_InitArgvExpandsOwnEnv(t *testing.T) { + pod := &corev1.Pod{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "dl", Image: "x", Command: []string{"sh", "-c", "ngc registry model download-version --dest ${DEST} org/team/model:1"}, + Env: []corev1.EnvVar{{Name: "DEST", Value: "/data/models"}}, VolumeMounts: []corev1.VolumeMount{{Name: "m", MountPath: "/data"}}}}, + Containers: []corev1.Container{{Name: "engine", Image: "x", Command: []string{"/start.sh"}, VolumeMounts: []corev1.VolumeMount{{Name: "m", MountPath: "/data"}}}}, + Volumes: []corev1.Volume{emptyDir("m")}, + }} + r, ok := Resolve(pod, 0) + if !ok || r.Identity.URI() != "ngc://org/team/model:1" || r.Landing.Path != "/data/models" || r.Landing.VolumeName != "m" { + t.Errorf("ok=%v %+v", ok, r) + } +} + +func TestResolve_NotADownloader(t *testing.T) { + for name, c := range map[string]corev1.Container{ + "dynamo frontend": {Command: []string{"python3", "-m", "dynamo.frontend"}, Args: []string{"--router-mode", "kv"}}, + "ray worker": {Image: "vllm/vllm-openai", Command: []string{"sh", "-c", "ray start --address=$(LWS_LEADER_ADDRESS):6379 --block"}}, + "etcd": {Image: "quay.io/coreos/etcd", Command: []string{"etcd"}}, + } { + if _, ok := Resolve(&corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{c}}}, 0); ok { + t.Errorf("%s must not resolve", name) + } + } +} + +// LWS worker: no model of its own; inherits from the leader template. +func TestResolveWithGroup_LWSWorkerInheritsLeader(t *testing.T) { + leaderTmpl := map[string]any{ + "spec": map[string]any{"containers": []any{map[string]any{ + "name": "vllm-leader", "image": "vllm/vllm-openai", + "command": []any{"sh", "-c", "bash multi-node-serving.sh leader --ray_cluster_size=$(LWS_GROUP_SIZE); python3 -m vllm.entrypoints.openai.api_server --port 8080 --model meta-llama/Meta-Llama-3.1-405B-Instruct --tensor-parallel-size 8 --pipeline_parallel_size 2"}, + }}}, + } + lws := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "leaderworkerset.x-k8s.io/v1", "kind": "LeaderWorkerSet", + "metadata": map[string]any{"name": "vllm", "namespace": "fn"}, + "spec": map[string]any{"leaderWorkerTemplate": map[string]any{"leaderTemplate": leaderTmpl}}, + }} + scheme := runtime.NewScheme() + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{lwsGVR: "LeaderWorkerSetList"}, lws) + worker := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: "fn", Labels: map[string]string{lwsNameLabel: "vllm", lwsWorkerIndexLabel: "1"}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "vllm-worker", Image: "vllm/vllm-openai", + Command: []string{"sh", "-c", "bash multi-node-serving.sh worker --ray_address=$(LWS_LEADER_ADDRESS)"}, + VolumeMounts: []corev1.VolumeMount{{Name: "hf", MountPath: "/root/.cache/huggingface"}}}}, + Volumes: []corev1.Volume{emptyDir("hf")}}, + } + r, ok, err := ResolveWithGroup(context.Background(), worker, 0, &LWSResolver{Dyn: dyn}) + if err != nil || !ok { + t.Fatalf("worker must inherit: ok=%v err=%v", ok, err) + } + if r.Identity.URI() != "hf://meta-llama/Meta-Llama-3.1-405B-Instruct" || r.Landing.VolumeName != "hf" || r.Landing.Kind != VolumeEmptyDir { + t.Errorf("inherited %+v", r) + } + // The leader itself resolves on its own and never consults the group. + leader := worker.DeepCopy() + leader.Labels[lwsWorkerIndexLabel] = "0" + leader.Spec.Containers[0].Command = []string{"sh", "-c", "vllm serve --model meta-llama/Meta-Llama-3.1-405B-Instruct"} + if r, ok, _ := ResolveWithGroup(context.Background(), leader, 0, &LWSResolver{Dyn: dynamicfake.NewSimpleDynamicClient(scheme)}); !ok || r.Source != "--model" { + t.Errorf("leader must resolve itself: ok=%v src=%q", ok, r.Source) + } + // A worker of an unknown group fails open (not a downloader). + unknown := worker.DeepCopy() + unknown.Labels[lwsNameLabel] = "missing" + if _, ok, err := ResolveWithGroup(context.Background(), unknown, 0, &LWSResolver{Dyn: dyn}); ok || err == nil { + t.Errorf("missing group: ok=%v err=%v, want not ok with error", ok, err) + } +} From 1d882cc492321f12986e2a4fd032823cfe62b2e5 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 07:33:57 -0700 Subject: [PATCH 02/16] feat(nvsnap): write-once model volume at admission for Helm functions Step 2 of docs/proposals/helm-shared-model-volume.md. For a GPU pod whose model identity resolves (internal/modelid), the webhook replaces the volume the download lands in with the per-identity model volume and turns the download into a write-once step; no pod is gated. internal/modelvolume names the claim per identity (nvsnap-model-), creates the writer claim (ReadWriteMany on a distributed filesystem, ReadWriteOnce on NVMesh), looks the identity up cluster-wide and records completion as a label. The webhook elects the writer with the existing Lease elector keyed by identity; the writer's landing emptyDir becomes the claim, its engine mounts the model read-only, and its download init is wrapped to touch /.nvsnap-complete on success. Readers on a distributed filesystem mount the same claim and their init waits for the marker, downloading themselves after the deadline (decided: always fall back). Readers on NVMesh keep their emptyDir, are labelled pending and wait for the agent to bind the completed volume in (step 3). Charts whose engine downloads itself get an injected huggingface-cli init that lands the model in the volume, with the engine started offline and the registry credentials forwarded. Compile caches are redirected into /.nvsnap/cache/ on a distributed filesystem and into the local cachedir on NVMesh; the model env of the cachedir template is left alone because the model lives in the landing volume now. Tests use the prd11 function shape and a stock vLLM Deployment: writer on Block, pending reader on Block, reader on RWX, injected init with forwarded token and offline engine, completed volume skipping the election, and the customer-PVC / no-GPU / no-model pods left alone. Mutation-checked: read-only mount dropped, reader fallback dropped, Block reader substituting the volume, and writer marker dropped each turn tests red. Co-Authored-By: Balaji Ganesan --- .../internal/checkpointstore/BUILD.bazel | 3 + .../nvsnap/internal/election/BUILD.bazel | 1 + .../nvsnap/internal/modelvolume/BUILD.bazel | 27 ++ .../internal/modelvolume/modelvolume.go | 197 +++++++++ .../internal/modelvolume/modelvolume_test.go | 72 ++++ .../nvsnap/internal/webhook/BUILD.bazel | 6 + .../nvsnap/internal/webhook/model_volume.go | 397 ++++++++++++++++++ .../internal/webhook/model_volume_test.go | 286 +++++++++++++ .../nvsnap/internal/webhook/mutate.go | 20 + 9 files changed, 1009 insertions(+) create mode 100644 src/compute-plane-services/nvsnap/internal/modelvolume/BUILD.bazel create mode 100644 src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go create mode 100644 src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go create mode 100644 src/compute-plane-services/nvsnap/internal/webhook/model_volume.go create mode 100644 src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go diff --git a/src/compute-plane-services/nvsnap/internal/checkpointstore/BUILD.bazel b/src/compute-plane-services/nvsnap/internal/checkpointstore/BUILD.bazel index 42a62617c1..44f721dff7 100644 --- a/src/compute-plane-services/nvsnap/internal/checkpointstore/BUILD.bazel +++ b/src/compute-plane-services/nvsnap/internal/checkpointstore/BUILD.bazel @@ -47,6 +47,7 @@ go_test( "mounter_test.go", "percapture_pvc_detach_test.go", "percapture_pvc_test.go", + "promoter_namespace_test.go", "promoter_shared_test.go", "storage_profile_test.go", "volumelayout_test.go", @@ -59,7 +60,9 @@ go_test( "@io_k8s_apimachinery//pkg/api/errors", "@io_k8s_apimachinery//pkg/api/resource", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured", "@io_k8s_apimachinery//pkg/runtime", + "@io_k8s_apimachinery//pkg/runtime/schema", "@io_k8s_apimachinery//pkg/types", "@io_k8s_client_go//dynamic/fake", "@io_k8s_client_go//kubernetes/fake", diff --git a/src/compute-plane-services/nvsnap/internal/election/BUILD.bazel b/src/compute-plane-services/nvsnap/internal/election/BUILD.bazel index 66602f9272..5d32bb3b1e 100644 --- a/src/compute-plane-services/nvsnap/internal/election/BUILD.bazel +++ b/src/compute-plane-services/nvsnap/internal/election/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "@io_k8s_api//core/v1:core", "@io_k8s_apimachinery//pkg/api/errors", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_apimachinery//pkg/util/uuid", "@io_k8s_client_go//kubernetes", ], ) diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/BUILD.bazel b/src/compute-plane-services/nvsnap/internal/modelvolume/BUILD.bazel new file mode 100644 index 0000000000..d17f0bdd07 --- /dev/null +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/BUILD.bazel @@ -0,0 +1,27 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "modelvolume", + srcs = ["modelvolume.go"], + importpath = "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/modelvolume", + visibility = ["//src/compute-plane-services/nvsnap:__subpackages__"], + deps = [ + "@io_k8s_api//core/v1:core", + "@io_k8s_apimachinery//pkg/api/errors", + "@io_k8s_apimachinery//pkg/api/resource", + "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_client_go//kubernetes", + ], +) + +go_test( + name = "modelvolume_test", + srcs = ["modelvolume_test.go"], + embed = [":modelvolume"], + deps = [ + "@io_k8s_api//core/v1:core", + "@io_k8s_apimachinery//pkg/api/resource", + "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_client_go//kubernetes/fake", + ], +) diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go new file mode 100644 index 0000000000..5764af85bc --- /dev/null +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go @@ -0,0 +1,197 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package modelvolume owns the per-identity model volume of +// docs/proposals/helm-shared-model-volume.md: one volume per model URI per +// cluster, written once by the elected writer's download step, immutable +// afterwards, attached by every other pod. +// +// Two modes, chosen by the storage profile: +// +// - ModeRWX (distributed filesystem): one ReadWriteMany claim per +// identity per namespace, mounted by writer and readers alike at +// admission. Completion is a marker file the writer's download step +// leaves at the volume root. +// - ModeBlock (NVMesh): the writer's ReadWriteOnce claim is the artifact. +// When its download step exits 0 the agent marks the claim complete and +// mints the read-only secondary PV and claim; readers admitted before +// that keep an emptyDir and wait for the agent to bind the volume in. +package modelvolume + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// Mode is how the volume is shared. +type Mode string + +// Modes: RWX on a distributed filesystem, Block on NVMesh. +const ( + ModeRWX Mode = "rwx" + ModeBlock Mode = "block" +) + +// Labels and annotations on volumes and pods. +const ( + // IdentityLabel is the short identity key on claims, PVs and pods. + IdentityLabel = "nvsnap.io/model" + // IdentityAnnotation is the full model URI. + IdentityAnnotation = "nvsnap.io/model-uri" + // RoleLabel is writer | reader on pods. + RoleLabel = "nvsnap.io/model-role" + // CompleteLabel is "true" on the writer claim once the download exited 0. + CompleteLabel = "nvsnap.io/model-complete" + // PendingLabel is "true" on a Block-mode reader that still needs the + // agent to bind the volume into its emptyDir. + PendingLabel = "nvsnap.io/model-pending" + // LandingAnnotation on a pod records the mount path the model must + // appear at, for the agent's bind. + LandingAnnotation = "nvsnap.io/model-landing" + // MarkerFile at the volume root says the download completed. + MarkerFile = ".nvsnap-complete" + + managedBy = "nvsnap" +) + +// Config comes from the storage profile. +type Config struct { + Mode Mode + StorageClass string + // Size requested for a new volume; the model size is unknown at + // admission, so this is a ceiling. Thin-provisioned classes make it + // cheap. + Size resource.Quantity +} + +// Key is the short stable token for a model URI, used in object names +// and labels (label values may be at most 63 characters). +func Key(uri string) string { + sum := sha256.Sum256([]byte(uri)) + return hex.EncodeToString(sum[:8]) +} + +// ClaimName is the writer claim in RWX mode and the shared claim in both. +func ClaimName(uri string) string { return "nvsnap-model-" + Key(uri) } + +// ReadOnlyClaimName is the Block-mode read-only claim minted after completion. +func ReadOnlyClaimName(uri string) string { return "nvsnap-model-" + Key(uri) + "-ro" } + +// State of an identity on the cluster, as the webhook needs it. +type State struct { + // Exists: a writer claim exists (a download is in flight or done). + Exists bool + // Complete: the download finished; readers may attach. + Complete bool + // ClaimNamespace is where the writer claim lives. + ClaimNamespace string +} + +// Provisioner creates and inspects model volumes. +type Provisioner struct { + Kube kubernetes.Interface + Cfg Config +} + +// EnsureWriterClaim creates the claim the writer downloads into, in ns. +// Idempotent. RWX mode creates a ReadWriteMany claim readers share; Block +// mode a ReadWriteOnce claim that becomes the read-only artifact. +func (p *Provisioner) EnsureWriterClaim(ctx context.Context, uri, ns string) (string, error) { + name := ClaimName(uri) + if _, err := p.Kube.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name, metav1.GetOptions{}); err == nil { + return name, nil + } else if !apierrors.IsNotFound(err) { + return "", fmt.Errorf("get claim %s/%s: %w", ns, name, err) + } + mode := corev1.ReadWriteOnce + if p.Cfg.Mode == ModeRWX { + mode = corev1.ReadWriteMany + } + sc := p.Cfg.StorageClass + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: ns, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": managedBy, + IdentityLabel: Key(uri), + }, + Annotations: map[string]string{IdentityAnnotation: uri}, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{mode}, + StorageClassName: &sc, + Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: p.Cfg.Size}}, + }, + } + if _, err := p.Kube.CoreV1().PersistentVolumeClaims(ns).Create(ctx, pvc, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { + return "", fmt.Errorf("create claim %s/%s: %w", ns, name, err) + } + return name, nil +} + +// Lookup finds the writer claim for uri anywhere on the cluster and reports +// whether its download completed. +func (p *Provisioner) Lookup(ctx context.Context, uri string) (State, error) { + list, err := p.Kube.CoreV1().PersistentVolumeClaims("").List(ctx, metav1.ListOptions{LabelSelector: IdentityLabel + "=" + Key(uri)}) + if err != nil { + return State{}, fmt.Errorf("list claims for %s: %w", uri, err) + } + st := State{} + for i := range list.Items { + c := &list.Items[i] + if c.Name != ClaimName(uri) { + continue + } + st.Exists = true + st.ClaimNamespace = c.Namespace + if c.Labels[CompleteLabel] == "true" { + st.Complete = true + return st, nil + } + } + return st, nil +} + +// MarkComplete labels the writer claim complete. The agent calls it when +// the writer's download step exits 0 (Block mode); in RWX mode the marker +// file is authoritative and this label is informational. +func (p *Provisioner) MarkComplete(ctx context.Context, uri, ns string) error { + name := ClaimName(uri) + pvc, err := p.Kube.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get claim %s/%s: %w", ns, name, err) + } + if pvc.Labels[CompleteLabel] == "true" { + return nil + } + if pvc.Labels == nil { + pvc.Labels = map[string]string{} + } + pvc.Labels[CompleteLabel] = "true" + if _, err := p.Kube.CoreV1().PersistentVolumeClaims(ns).Update(ctx, pvc, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("label claim %s/%s complete: %w", ns, name, err) + } + return nil +} diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go new file mode 100644 index 0000000000..7d64b3787d --- /dev/null +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package modelvolume + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +const uri = "hf://Qwen/Qwen2.5-32B-Instruct" + +func TestProvisioner_WriterClaimModes(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + mode Mode + want corev1.PersistentVolumeAccessMode + }{{ModeRWX, corev1.ReadWriteMany}, {ModeBlock, corev1.ReadWriteOnce}} { + kc := fake.NewSimpleClientset() + p := &Provisioner{Kube: kc, Cfg: Config{Mode: tc.mode, StorageClass: "sc", Size: resource.MustParse("512Gi")}} + name, err := p.EnsureWriterClaim(ctx, uri, "fn") + if err != nil || name != ClaimName(uri) { + t.Fatalf("%s: %v %q", tc.mode, err, name) + } + pvc, err := kc.CoreV1().PersistentVolumeClaims("fn").Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if pvc.Spec.AccessModes[0] != tc.want || *pvc.Spec.StorageClassName != "sc" || pvc.Labels[IdentityLabel] != Key(uri) || pvc.Annotations[IdentityAnnotation] != uri { + t.Errorf("%s: claim %+v", tc.mode, pvc) + } + if _, err := p.EnsureWriterClaim(ctx, uri, "fn"); err != nil { + t.Errorf("%s: second call must be a no-op: %v", tc.mode, err) + } + } +} + +func TestProvisioner_LookupAndComplete(t *testing.T) { + ctx := context.Background() + kc := fake.NewSimpleClientset() + p := &Provisioner{Kube: kc, Cfg: Config{Mode: ModeBlock, StorageClass: "sc", Size: resource.MustParse("1Gi")}} + if st, err := p.Lookup(ctx, uri); err != nil || st.Exists || st.Complete { + t.Errorf("nothing yet: %+v %v", st, err) + } + if _, err := p.EnsureWriterClaim(ctx, uri, "fn-a"); err != nil { + t.Fatal(err) + } + if st, err := p.Lookup(ctx, uri); err != nil || !st.Exists || st.Complete || st.ClaimNamespace != "fn-a" { + t.Errorf("in flight: %+v %v", st, err) + } + if err := p.MarkComplete(ctx, uri, "fn-a"); err != nil { + t.Fatal(err) + } + if st, _ := p.Lookup(ctx, uri); !st.Complete { + t.Errorf("after MarkComplete: %+v", st) + } + if err := p.MarkComplete(ctx, uri, "fn-a"); err != nil { + t.Errorf("second MarkComplete must be a no-op: %v", err) + } + // Another identity is unaffected. + if st, _ := p.Lookup(ctx, "hf://other/model"); st.Exists { + t.Error("lookup must be per identity") + } + if len(Key(uri)) != 16 || ClaimName(uri) != "nvsnap-model-"+Key(uri) || ReadOnlyClaimName(uri) != ClaimName(uri)+"-ro" { + t.Errorf("names: %s %s", ClaimName(uri), ReadOnlyClaimName(uri)) + } +} diff --git a/src/compute-plane-services/nvsnap/internal/webhook/BUILD.bazel b/src/compute-plane-services/nvsnap/internal/webhook/BUILD.bazel index a90f668440..8e5668d0cf 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/BUILD.bazel +++ b/src/compute-plane-services/nvsnap/internal/webhook/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "election.go", "extract_coalesce.go", "l2_mount.go", + "model_volume.go", "mount_prep_init.go", "mutate.go", "restore_entrypoint.go", @@ -22,6 +23,8 @@ go_library( deps = [ "//src/compute-plane-services/nvsnap/internal/checkpointstore", "//src/compute-plane-services/nvsnap/internal/election", + "//src/compute-plane-services/nvsnap/internal/modelid", + "//src/compute-plane-services/nvsnap/internal/modelvolume", "//src/compute-plane-services/nvsnap/internal/rootfsonly", "//src/compute-plane-services/nvsnap/internal/tracing", "@com_github_sirupsen_logrus//:logrus", @@ -43,6 +46,7 @@ go_test( "extract_coalesce_test.go", "l2_mount_test.go", "mergeplan_test.go", + "model_volume_test.go", "mutate_test.go", "restore_entrypoint_test.go", "rootfs_l2_mount_test.go", @@ -53,6 +57,7 @@ go_test( deps = [ "//src/compute-plane-services/nvsnap/internal/checkpointstore", "//src/compute-plane-services/nvsnap/internal/election", + "//src/compute-plane-services/nvsnap/internal/modelvolume", "//src/compute-plane-services/nvsnap/internal/rootfsonly", "@io_k8s_api//admission/v1:admission", "@io_k8s_api//core/v1:core", @@ -60,5 +65,6 @@ go_test( "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", "@io_k8s_apimachinery//pkg/runtime", "@io_k8s_apimachinery//pkg/types", + "@io_k8s_client_go//kubernetes/fake", ], ) diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go new file mode 100644 index 0000000000..8f549153b4 --- /dev/null +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go @@ -0,0 +1,397 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package webhook + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "path" + "strings" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/checkpointstore" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/election" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/modelid" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/modelvolume" +) + +// Model volume decoration (docs/proposals/helm-shared-model-volume.md). +// For a pod that will download a model, the webhook replaces the volume +// the download lands in with the per-identity model volume and turns the +// download step into a write-once: the elected writer downloads and marks +// completion, every other pod waits for the marker and skips its own +// download. No pod is gated; waiting is an init container. +// +// Roles: +// - writer: download step runs, then touches /.nvsnap-complete. +// Its engine mounts the model read-only so the volume stays immutable. +// - reader, RWX mode: mounts the same claim; its download step becomes +// "wait for the marker, else download" (fallback after the deadline). +// - reader, Block mode: keeps its emptyDir at the landing path and waits +// for the agent to bind the completed volume in and drop the marker. + +const ( + modelVolumeName = "nvsnap-model" + waitScriptDeadline = 3600 // seconds, when no deadline is configured +) + +// modelVolumePatches is the Helm-function decision. (nil, nil) means the +// pod is not a downloader or the feature is off, and Mutate continues +// with the older paths. +func (m *Mutator) modelVolumePatches(ctx context.Context, pod *corev1.Pod) ([]PatchOp, error) { + if m.ModelVolume == nil || m.Elector == nil { + return nil, nil + } + if gpuRequest(pod) == 0 { + return nil, nil + } + res, ok, err := modelid.ResolveWithGroup(ctx, pod, m.MainContainer, m.Groups) + if err != nil { + return nil, fmt.Errorf("resolve model identity: %w", err) + } + if !ok || res.Identity.Scheme == "path" { + return nil, nil + } + uri := res.Identity.URI() + log := m.logger().WithFields(logrus.Fields{"pod": election.PodIdentity(pod), "model": uri, "source": res.Source}) + if !res.Landing.Substitutable() { + log.WithField("kind", res.Landing.Kind).Info("model volume: landing volume is the customer's; leaving pod alone") + return nil, nil + } + main := &pod.Spec.Containers[m.MainContainer] + land := res.Landing + mp := newMetaPatcher(pod) + patches := mp.annotation(modelvolume.IdentityAnnotation, uri) + patches = append(patches, mp.label(modelvolume.IdentityLabel, modelvolume.Key(uri))...) + + st, err := m.ModelVolume.Lookup(ctx, uri) + if err != nil { + return nil, err + } + role := election.RoleFollower + if !st.Complete { + r, _, err := m.Elector.Elect(ctx, leaseHash(uri), pod) + if err != nil { + return nil, err + } + role = r + } + switch { + case role == election.RoleLeader: + claim, err := m.ModelVolume.EnsureWriterClaim(ctx, uri, pod.Namespace) + if err != nil { + return nil, err + } + patches = append(patches, mp.label(modelvolume.RoleLabel, "writer")...) + patches = append(patches, m.substituteLandingVolume(pod, main, land, claim)...) + patches = append(patches, m.downloadStepPatches(pod, main, land, res.Identity, true)...) + patches = append(patches, m.modelCacheEnvPatches(pod, main, land, uri)...) + log.WithField("claim", claim).Info("model volume: writer; download lands in the shared volume") + case m.ModelVolume.Cfg.Mode == modelvolume.ModeRWX: + // Readers share the claim in the writer's namespace when it is + // ours, else get one minted in theirs (EnsureClaim, cross + // namespace, is the same volume on a distributed filesystem). + claim, err := m.ModelVolume.EnsureWriterClaim(ctx, uri, pod.Namespace) + if err != nil { + return nil, err + } + patches = append(patches, mp.label(modelvolume.RoleLabel, "reader")...) + patches = append(patches, m.substituteLandingVolume(pod, main, land, claim)...) + patches = append(patches, m.downloadStepPatches(pod, main, land, res.Identity, false)...) + patches = append(patches, m.modelCacheEnvPatches(pod, main, land, uri)...) + log.WithFields(logrus.Fields{"claim": claim, "complete": st.Complete}).Info("model volume: reader on shared filesystem; waits for the marker") + default: + // Block mode reader: the emptyDir stays; the agent binds the + // completed read-only volume over it and drops the marker. + patches = append(patches, mp.label(modelvolume.RoleLabel, "reader")...) + patches = append(patches, mp.label(modelvolume.PendingLabel, "true")...) + patches = append(patches, mp.annotation(modelvolume.LandingAnnotation, landingMount(land))...) + if land.Kind == modelid.VolumeRootfs { + patches = append(patches, m.addLandingEmptyDir(pod, main, land)...) + } + patches = append(patches, m.downloadStepPatches(pod, main, land, res.Identity, false)...) + patches = append(patches, m.modelCacheEnvPatches(pod, main, land, uri)...) + log.WithField("complete", st.Complete).Info("model volume: reader on block storage; agent binds the volume after completion") + } + return patches, nil +} + +// leaseHash keys the writer election by model identity, in the 64-hex form +// the Lease naming expects. +func leaseHash(uri string) string { + sum := sha256.Sum256([]byte("model:" + uri)) + return hex.EncodeToString(sum[:]) +} + +func gpuRequest(pod *corev1.Pod) int64 { + var n int64 + for i := range pod.Spec.Containers { + if q, ok := pod.Spec.Containers[i].Resources.Limits["nvidia.com/gpu"]; ok { + n += q.Value() + } + } + return n +} + +// landingMount is the mount path the model volume occupies: the existing +// volume's mount when there is one, else the landing path itself. +func landingMount(l modelid.Landing) string { + if l.MountPath != "" { + return l.MountPath + } + return l.Path +} + +// substituteLandingVolume replaces the emptyDir under the landing path +// with the claim, or adds the claim as a new volume mounted at the landing +// path when the download wrote into the container filesystem. The main +// container's mount becomes read-only: the model is immutable once +// downloaded, and nothing in the engine may dirty the shared copy. +func (m *Mutator) substituteLandingVolume(pod *corev1.Pod, main *corev1.Container, land modelid.Landing, claim string) []PatchOp { + src := corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: claim}} + if land.VolumeName != "" { + for i := range pod.Spec.Volumes { + if pod.Spec.Volumes[i].Name != land.VolumeName { + continue + } + patches := []PatchOp{{Op: "replace", Path: fmt.Sprintf("/spec/volumes/%d", i), Value: corev1.Volume{Name: land.VolumeName, VolumeSource: src}}} + for j := range main.VolumeMounts { + if main.VolumeMounts[j].Name == land.VolumeName { + patches = append(patches, PatchOp{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts/%d/readOnly", m.MainContainer, j), Value: true}) + } + } + return patches + } + } + patches := []PatchOp{} + if pod.Spec.Volumes == nil { + patches = append(patches, PatchOp{Op: "add", Path: "/spec/volumes", Value: []any{}}) + } + if main.VolumeMounts == nil { + patches = append(patches, PatchOp{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts", m.MainContainer), Value: []any{}}) + } + patches = append(patches, + PatchOp{Op: "add", Path: "/spec/volumes/-", Value: corev1.Volume{Name: modelVolumeName, VolumeSource: src}}, + PatchOp{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts/-", m.MainContainer), Value: corev1.VolumeMount{Name: modelVolumeName, MountPath: land.Path, ReadOnly: true}}, + ) + return patches +} + +// addLandingEmptyDir gives a Block-mode reader whose download wrote to the +// container filesystem an emptyDir at the landing path, so the agent has a +// mount point to bind the completed volume over. +func (m *Mutator) addLandingEmptyDir(pod *corev1.Pod, main *corev1.Container, land modelid.Landing) []PatchOp { + patches := []PatchOp{} + if pod.Spec.Volumes == nil { + patches = append(patches, PatchOp{Op: "add", Path: "/spec/volumes", Value: []any{}}) + } + if main.VolumeMounts == nil { + patches = append(patches, PatchOp{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts", m.MainContainer), Value: []any{}}) + } + return append(patches, + PatchOp{Op: "add", Path: "/spec/volumes/-", Value: corev1.Volume{Name: modelVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, + PatchOp{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts/-", m.MainContainer), Value: corev1.VolumeMount{Name: modelVolumeName, MountPath: land.Path}}, + ) +} + +// downloadStepPatches turns the download into a write-once step. With an +// init container: the writer's init is wrapped to touch the marker on +// success, a reader's init to wait for the marker and skip. Without one +// (the engine downloads): an init is injected that runs the download for +// the writer and waits for readers, and the engine is started offline so +// it reads the volume instead of the network. +func (m *Mutator) downloadStepPatches(pod *corev1.Pod, main *corev1.Container, land modelid.Landing, id modelid.Identity, writer bool) []PatchOp { + marker := path.Join(landingMount(land), modelvolume.MarkerFile) + deadline := m.waitDeadlineSeconds() + if land.Downloader == modelid.DownloaderInit { + for i := range pod.Spec.InitContainers { + init := &pod.Spec.InitContainers[i] + if init.Name != land.InitContainer { + continue + } + imarker := path.Join(initMountFor(init, land), modelvolume.MarkerFile) + orig := shellJoin(append(append([]string{}, init.Command...), init.Args...)) + script := writerScript(orig, imarker) + if !writer { + script = readerScript(orig, imarker, deadline) + } + return []PatchOp{ + {Op: "replace", Path: fmt.Sprintf("/spec/initContainers/%d/command", i), Value: []string{"/bin/sh", "-c"}}, + {Op: "replace", Path: fmt.Sprintf("/spec/initContainers/%d/args", i), Value: []string{script}}, + } + } + return nil + } + // Engine downloads itself. Only Hugging Face repos can be fetched by + // an injected init; NIM and others complete when the engine is Ready + // (the agent marks them), and readers still wait for the marker. + var patches []PatchOp + if pod.Spec.InitContainers == nil { + patches = append(patches, PatchOp{Op: "add", Path: "/spec/initContainers", Value: []any{}}) + } + download := "" + if id.Scheme == "hf" { + download = "huggingface-cli download " + shellQuote(id.Ref) + if id.Revision != "" { + download += " --revision " + shellQuote(id.Revision) + } + } + script := readerScript(download, marker, deadline) + if writer { + if download == "" { + return nil // engine writes; completion is Ready, marked by the agent + } + script = writerScript(download, marker) + } + init := corev1.Container{ + Name: "nvsnap-model-download", + Image: main.Image, + Command: []string{"/bin/sh", "-c"}, + Args: []string{script}, + Env: append([]corev1.EnvVar{{Name: "HF_HOME", Value: land.Path}}, tokenEnv(main)...), + } + for _, vm := range main.VolumeMounts { + if vm.Name == land.VolumeName || vm.Name == modelVolumeName { + init.VolumeMounts = append(init.VolumeMounts, corev1.VolumeMount{Name: vm.Name, MountPath: vm.MountPath}) + } + } + if len(init.VolumeMounts) == 0 { + init.VolumeMounts = []corev1.VolumeMount{{Name: modelVolumeName, MountPath: land.Path}} + } + patches = append(patches, PatchOp{Op: "add", Path: "/spec/initContainers/0", Value: init}) + if id.Scheme == "hf" { + if main.Env == nil { + patches = append(patches, PatchOp{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/env", m.MainContainer), Value: []any{}}) + } + patches = append(patches, appendEnv(m.MainContainer, corev1.EnvVar{Name: "HF_HUB_OFFLINE", Value: "1"})) + } + return patches +} + +// initMountFor is the path the download init sees the landing volume at; +// it may differ from the main container's mount. +func initMountFor(init *corev1.Container, land modelid.Landing) string { + for _, vm := range init.VolumeMounts { + if vm.Name == land.VolumeName { + return vm.MountPath + } + } + return landingMount(land) +} + +func writerScript(download, marker string) string { + return fmt.Sprintf("set -e\nif [ -f %[1]s ]; then echo 'nvsnap: model already complete'; exit 0; fi\n%[2]s\nsync\ntouch %[1]s\n", shellQuote(marker), download) +} + +// readerScript waits for the marker; past the deadline it runs the +// download itself (decided: always fall back, never deadlock). +func readerScript(download, marker string, deadline int) string { + fallback := "echo 'nvsnap: no download step to fall back to'; exit 0" + if download != "" { + fallback = download + } + return fmt.Sprintf("set -e\nd=0\nwhile [ ! -f %[1]s ]; do if [ $d -ge %[2]d ]; then echo 'nvsnap: marker deadline passed; downloading locally'; %[3]s; exit 0; fi; sleep 5; d=$((d+5)); done\necho 'nvsnap: model complete, skipping download'\n", shellQuote(marker), deadline, fallback) +} + +func (m *Mutator) waitDeadlineSeconds() int { + if m.ModelWaitDeadline > 0 { + return int(m.ModelWaitDeadline.Seconds()) + } + return waitScriptDeadline +} + +// tokenEnv copies registry credentials the engine carries to the injected +// download init, by value or by reference. +func tokenEnv(main *corev1.Container) []corev1.EnvVar { + var out []corev1.EnvVar + for _, e := range main.Env { + switch e.Name { + case "HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HF_HUB_ENABLE_HF_TRANSFER", "HF_HUB_DISABLE_XET", "HF_ENDPOINT": + out = append(out, e) + } + } + return out +} + +// modelCacheEnvPatches redirects the compile caches. RWX: into the shared +// volume under a key of image plus identity plus role-neutral args, so +// every pod of that engine config shares one set. Block: into the pod's +// local cachedir (captured after Ready by the existing path). The model +// entries of the template are dropped: the model lives in the landing +// volume now, not under the cachedir. +func (m *Mutator) modelCacheEnvPatches(pod *corev1.Pod, main *corev1.Container, land modelid.Landing, uri string) []PatchOp { + root := "" + switch { + case m.ModelVolume.Cfg.Mode == modelvolume.ModeRWX: + key := modelvolume.Key(uri) + if m.Composer != nil { + key = checkpointstore.ShortHash(checkpointstore.ComputeHash(m.Composer.Compose(pod, m.MainContainer)))[:16] + } + root = path.Join(landingMount(land), ".nvsnap", "cache", key) + case m.CacheDir != "": + root = path.Join(m.CacheDir, "cache") + default: + return nil + } + patches := make([]PatchOp, 0, 8) + if main.Env == nil { + patches = append(patches, PatchOp{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/env", m.MainContainer), Value: []any{}}) + } + for _, e := range m.cacheEnvVars(m.CacheDir) { + if !strings.HasPrefix(e.Value, path.Join(m.CacheDir, "cache")) { + continue // model entries (HF_HOME, NIM_CACHE_PATH) stay with the landing volume + } + rel := strings.TrimPrefix(e.Value, path.Join(m.CacheDir, "cache")) + patches = append(patches, appendEnv(m.MainContainer, corev1.EnvVar{Name: e.Name, Value: root + rel})) + } + if m.ModelVolume.Cfg.Mode != modelvolume.ModeRWX && m.CacheDir != "" { + // Block mode keeps the local cachedir emptyDir the capture reads. + patches = append(patches, m.cacheDirVolumeOnly(pod, main)...) + } + return patches +} + +// cacheDirVolumeOnly adds the /opt/nvsnap emptyDir for compile caches +// without the model env of the capture decoration. +func (m *Mutator) cacheDirVolumeOnly(pod *corev1.Pod, main *corev1.Container) []PatchOp { + for _, vm := range main.VolumeMounts { + if vm.Name == cacheDirVolumeName || vm.MountPath == m.CacheDir { + return nil + } + } + patches := []PatchOp{} + if pod.Spec.Volumes == nil { + patches = append(patches, PatchOp{Op: "add", Path: "/spec/volumes", Value: []any{}}) + } + if main.VolumeMounts == nil { + patches = append(patches, PatchOp{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts", m.MainContainer), Value: []any{}}) + } + return append(patches, + PatchOp{Op: "add", Path: "/spec/volumes/-", Value: corev1.Volume{Name: cacheDirVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, + PatchOp{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts/-", m.MainContainer), Value: corev1.VolumeMount{Name: cacheDirVolumeName, MountPath: m.CacheDir}}, + ) +} + +// shellJoin renders an exec argv as one shell command line. +func shellJoin(argv []string) string { + parts := make([]string, 0, len(argv)) + for _, a := range argv { + parts = append(parts, shellQuote(a)) + } + return strings.Join(parts, " ") +} + +func shellQuote(s string) string { + if s == "" { + return "''" + } + if !strings.ContainsAny(s, " \t\n'\"\\$`&|;<>()*?[]{}!#~") { + return s + } + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go new file mode 100644 index 0000000000..64f8d6c3b8 --- /dev/null +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go @@ -0,0 +1,286 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package webhook + +import ( + "context" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/election" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/modelvolume" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/rootfsonly" +) + +func mvMutator(t *testing.T, mode modelvolume.Mode, role election.Role, kc *fake.Clientset) (*Mutator, *fakeElector) { + t.Helper() + el := &fakeElector{role: role} + return &Mutator{ + Backend: newBackend(t), + CacheDir: "/opt/nvsnap", + Composer: &rootfsonly.HashInputComposer{CUDADriverMajor: 580}, + Elector: el, + ModelVolume: &modelvolume.Provisioner{Kube: kc, Cfg: modelvolume.Config{Mode: mode, StorageClass: "sc", Size: resource.MustParse("512Gi")}}, + }, el +} + +// The prd11 function shape: NGC init download into an emptyDir, engine +// from a positional MODEL_PATH, two pods per instance. +func ngcFunctionPod() *corev1.Pod { + gpu := corev1.ResourceRequirements{Limits: corev1.ResourceList{"nvidia.com/gpu": resource.MustParse("4")}} + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: "sr-fn", GenerateName: "mini-service-kimi-k3-"}, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{ + Name: "download-ngc-model", Image: "nvcr.io/org/ultra:vllm", Command: []string{"/bin/bash", "-c"}, + Args: []string{"set -euo pipefail\nngc registry model download-version --dest \"${NGC_MODEL_MOUNT}\" \"${NGC_MODEL_NAME}\"\n"}, + Env: []corev1.EnvVar{{Name: "NGC_MODEL_NAME", Value: "org/team/nemotron3-ultra-genrm:bf16-fixed"}, {Name: "NGC_MODEL_MOUNT", Value: "/config/models"}}, + VolumeMounts: []corev1.VolumeMount{{Name: "ngc-models", MountPath: "/config/models"}}, + }}, + Containers: []corev1.Container{{ + Name: "kimi-k3", Image: "nvcr.io/org/ultra:vllm", Command: []string{"/bin/bash", "/opt/kimi-k3/start.sh"}, + Env: []corev1.EnvVar{{Name: "MODEL_PATH", Value: "/config/models/nemotron3-ultra-genrm"}, {Name: "HF_HUB_OFFLINE", Value: "1"}}, + Resources: gpu, + VolumeMounts: []corev1.VolumeMount{{Name: "dshm", MountPath: "/dev/shm"}, {Name: "ngc-models", MountPath: "/config/models"}}, + }}, + Volumes: []corev1.Volume{ + {Name: "dshm", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "ngc-models", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + }, + }, + } +} + +func stockVLLMPod() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: "fn", GenerateName: "vllm-"}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "vllm", Image: "vllm/vllm-openai:v0.20.0", Command: []string{"/bin/bash", "-lc"}, + Args: []string{"vllm serve --model Qwen/Qwen2.5-32B-Instruct --tensor-parallel-size 4"}, + Env: []corev1.EnvVar{{Name: "HF_TOKEN", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{Key: "token"}}}}, + Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{"nvidia.com/gpu": resource.MustParse("4")}}, + }}}, + } +} + +type mvView struct { + labels, annotations map[string]string + volumes map[string]corev1.Volume // by name, from replace/add ops + roMounts []string + initScripts map[string]string + newInits []corev1.Container + env map[string]string +} + +func viewMV(pod *corev1.Pod, patches []PatchOp) mvView { + v := mvView{labels: map[string]string{}, annotations: map[string]string{}, volumes: map[string]corev1.Volume{}, initScripts: map[string]string{}, env: map[string]string{}} + for _, p := range patches { + switch { + case strings.HasPrefix(p.Path, "/metadata/labels/"): + v.labels[strings.ReplaceAll(strings.TrimPrefix(p.Path, "/metadata/labels/"), "~1", "/")] = p.Value.(string) + case strings.HasPrefix(p.Path, "/metadata/annotations/"): + v.annotations[strings.ReplaceAll(strings.TrimPrefix(p.Path, "/metadata/annotations/"), "~1", "/")] = p.Value.(string) + case strings.HasSuffix(p.Path, "/readOnly") && p.Value == true: + v.roMounts = append(v.roMounts, p.Path) + case strings.HasPrefix(p.Path, "/spec/initContainers/") && strings.HasSuffix(p.Path, "/args"): + idx := strings.Split(strings.TrimPrefix(p.Path, "/spec/initContainers/"), "/")[0] + v.initScripts[pod.Spec.InitContainers[atoi(idx)].Name] = p.Value.([]string)[0] + } + switch val := p.Value.(type) { + case corev1.Volume: + v.volumes[val.Name] = val + case corev1.Container: + if strings.HasPrefix(p.Path, "/spec/initContainers") { + v.newInits = append(v.newInits, val) + } + case corev1.EnvVar: + v.env[val.Name] = val.Value + } + } + return v +} + +func atoi(s string) int { + n := 0 + for _, c := range s { + n = n*10 + int(c-'0') + } + return n +} + +func TestModelVolume_WriterBlock_NGCInit(t *testing.T) { + kc := fake.NewSimpleClientset() + m, el := mvMutator(t, modelvolume.ModeBlock, election.RoleLeader, kc) + pod := ngcFunctionPod() + patches, err := m.Mutate(context.Background(), pod) + if err != nil { + t.Fatal(err) + } + v := viewMV(pod, patches) + uri := "ngc://org/team/nemotron3-ultra-genrm:bf16-fixed" + if el.called != 1 || v.annotations[modelvolume.IdentityAnnotation] != uri || v.labels[modelvolume.RoleLabel] != "writer" { + t.Errorf("writer stamp: elected=%d ann=%v labels=%v", el.called, v.annotations, v.labels) + } + vol, ok := v.volumes["ngc-models"] + if !ok || vol.PersistentVolumeClaim == nil || vol.PersistentVolumeClaim.ClaimName != modelvolume.ClaimName(uri) { + t.Errorf("landing emptyDir must be replaced by the writer claim, got %+v", vol) + } + if pvc, err := kc.CoreV1().PersistentVolumeClaims("sr-fn").Get(context.Background(), modelvolume.ClaimName(uri), metav1.GetOptions{}); err != nil || pvc.Spec.AccessModes[0] != corev1.ReadWriteOnce { + t.Errorf("writer claim must exist RWO in the pod namespace: %v", err) + } + if len(v.roMounts) != 1 || !strings.Contains(v.roMounts[0], "/spec/containers/0/volumeMounts/1/") { + t.Errorf("engine's model mount must become read-only, got %v", v.roMounts) + } + s := v.initScripts["download-ngc-model"] + if !strings.Contains(s, "ngc registry model download-version") || !strings.Contains(s, "touch /config/models/.nvsnap-complete") || !strings.Contains(s, "already complete") { + t.Errorf("writer init must run the original download then touch the marker:\n%s", s) + } + if v.env["TORCHINDUCTOR_CACHE_DIR"] != "/opt/nvsnap/cache/torchinductor" || v.env["HF_HOME"] != "" || v.env["NIM_CACHE_PATH"] != "" { + t.Errorf("Block mode: compile caches to the local cachedir, model env untouched: %v", v.env) + } + if _, ok := v.volumes[cacheDirVolumeName]; !ok { + t.Error("Block mode must add the local cachedir emptyDir for the compile caches") + } +} + +func TestModelVolume_ReaderBlock_PendingBind(t *testing.T) { + kc := fake.NewSimpleClientset() + m, _ := mvMutator(t, modelvolume.ModeBlock, election.RoleFollower, kc) + pod := ngcFunctionPod() + patches, err := m.Mutate(context.Background(), pod) + if err != nil { + t.Fatal(err) + } + v := viewMV(pod, patches) + if v.labels[modelvolume.RoleLabel] != "reader" || v.labels[modelvolume.PendingLabel] != "true" || v.annotations[modelvolume.LandingAnnotation] != "/config/models" { + t.Errorf("Block reader stamp: %v %v", v.labels, v.annotations) + } + if _, replaced := v.volumes["ngc-models"]; replaced { + t.Error("Block reader keeps its emptyDir; the agent binds the volume over it") + } + s := v.initScripts["download-ngc-model"] + if !strings.Contains(s, "while [ ! -f /config/models/.nvsnap-complete ]") || !strings.Contains(s, "ngc registry model download-version") || !strings.Contains(s, "deadline passed") { + t.Errorf("reader init must wait for the marker and fall back to its own download:\n%s", s) + } + for _, p := range patches { + if strings.HasPrefix(p.Path, "/spec/schedulingGates") { + t.Fatal("no pod is ever gated on the model volume path") + } + } + if pvcs, _ := kc.CoreV1().PersistentVolumeClaims("").List(context.Background(), metav1.ListOptions{}); len(pvcs.Items) != 0 { + t.Error("a Block reader must not create claims") + } +} + +func TestModelVolume_ReaderRWX_SharesClaim(t *testing.T) { + kc := fake.NewSimpleClientset() + m, _ := mvMutator(t, modelvolume.ModeRWX, election.RoleFollower, kc) + pod := ngcFunctionPod() + patches, err := m.Mutate(context.Background(), pod) + if err != nil { + t.Fatal(err) + } + v := viewMV(pod, patches) + uri := "ngc://org/team/nemotron3-ultra-genrm:bf16-fixed" + vol := v.volumes["ngc-models"] + if vol.PersistentVolumeClaim == nil || vol.PersistentVolumeClaim.ClaimName != modelvolume.ClaimName(uri) { + t.Errorf("RWX reader mounts the shared claim, got %+v", vol) + } + if pvc, err := kc.CoreV1().PersistentVolumeClaims("sr-fn").Get(context.Background(), modelvolume.ClaimName(uri), metav1.GetOptions{}); err != nil || pvc.Spec.AccessModes[0] != corev1.ReadWriteMany { + t.Errorf("shared claim must be RWX: %v", err) + } + if v.labels[modelvolume.PendingLabel] != "" { + t.Error("RWX readers are not pending; the filesystem delivers the marker") + } + if !strings.HasPrefix(v.env["TORCHINDUCTOR_CACHE_DIR"], "/config/models/.nvsnap/cache/") || !strings.HasSuffix(v.env["TORCHINDUCTOR_CACHE_DIR"], "/torchinductor") { + t.Errorf("RWX mode: compile caches live in the shared volume under a config key, got %q", v.env["TORCHINDUCTOR_CACHE_DIR"]) + } +} + +func TestModelVolume_EngineDownload_InjectedInit(t *testing.T) { + kc := fake.NewSimpleClientset() + m, _ := mvMutator(t, modelvolume.ModeRWX, election.RoleLeader, kc) + pod := stockVLLMPod() + patches, err := m.Mutate(context.Background(), pod) + if err != nil { + t.Fatal(err) + } + v := viewMV(pod, patches) + if len(v.newInits) != 1 || v.newInits[0].Name != "nvsnap-model-download" { + t.Fatalf("engine-download writer needs an injected download init, got %v", v.newInits) + } + init := v.newInits[0] + s := init.Args[0] + if !strings.Contains(s, "huggingface-cli download Qwen/Qwen2.5-32B-Instruct") || !strings.Contains(s, "touch /root/.cache/huggingface/.nvsnap-complete") { + t.Errorf("injected init script:\n%s", s) + } + if init.Image != pod.Spec.Containers[0].Image || len(init.VolumeMounts) != 1 || init.VolumeMounts[0].MountPath != "/root/.cache/huggingface" { + t.Errorf("init must reuse the engine image and mount the model volume at HF_HOME: %+v", init) + } + var sawToken bool + for _, e := range init.Env { + if e.Name == "HF_TOKEN" && e.ValueFrom != nil { + sawToken = true + } + } + if !sawToken { + t.Error("registry credentials must be forwarded to the download init") + } + if v.env["HF_HUB_OFFLINE"] != "1" { + t.Error("engine must start offline and read the volume") + } + vol, ok := v.volumes[modelVolumeName] + if !ok || vol.PersistentVolumeClaim == nil { + t.Errorf("rootfs landing gets a new claim volume: %+v", vol) + } +} + +func TestModelVolume_CompleteSkipsElection(t *testing.T) { + kc := fake.NewSimpleClientset() + m, el := mvMutator(t, modelvolume.ModeRWX, election.RoleLeader, kc) + uri := "hf://Qwen/Qwen2.5-32B-Instruct" + if _, err := m.ModelVolume.EnsureWriterClaim(context.Background(), uri, "fn"); err != nil { + t.Fatal(err) + } + if err := m.ModelVolume.MarkComplete(context.Background(), uri, "fn"); err != nil { + t.Fatal(err) + } + patches, err := m.Mutate(context.Background(), stockVLLMPod()) + if err != nil { + t.Fatal(err) + } + v := viewMV(stockVLLMPod(), patches) + if el.called != 0 || v.labels[modelvolume.RoleLabel] != "reader" { + t.Errorf("complete volume: no election, reader role; elected=%d labels=%v", el.called, v.labels) + } +} + +func TestModelVolume_LeavesOthersAlone(t *testing.T) { + kc := fake.NewSimpleClientset() + m, el := mvMutator(t, modelvolume.ModeRWX, election.RoleLeader, kc) + customer := stockVLLMPod() + customer.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "models", MountPath: "/root/.cache/huggingface"}} + customer.Spec.Volumes = []corev1.Volume{{Name: "models", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "their-nfs"}}}} + if p, err := m.modelVolumePatches(context.Background(), customer); err != nil || p != nil { + t.Errorf("customer PVC at HF_HOME must be left alone: %v %v", p, err) + } + noGPU := stockVLLMPod() + noGPU.Spec.Containers[0].Resources = corev1.ResourceRequirements{} + if p, _ := m.modelVolumePatches(context.Background(), noGPU); p != nil { + t.Error("a pod without GPUs is not a model worker") + } + frontend := stockVLLMPod() + frontend.Spec.Containers[0].Args = []string{"python3 -m dynamo.frontend --router-mode kv"} + if p, _ := m.modelVolumePatches(context.Background(), frontend); p != nil { + t.Error("a pod naming no model is left alone") + } + if el.called != 0 { + t.Error("no election for pods that are left alone") + } +} diff --git a/src/compute-plane-services/nvsnap/internal/webhook/mutate.go b/src/compute-plane-services/nvsnap/internal/webhook/mutate.go index 265ba08bc9..654747474d 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/mutate.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/mutate.go @@ -32,6 +32,7 @@ import ( "fmt" "regexp" "strings" + "time" "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" @@ -39,6 +40,8 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/checkpointstore" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/election" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/modelid" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/modelvolume" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/rootfsonly" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/tracing" ) @@ -255,6 +258,17 @@ type Mutator struct { // capture and explicit-hash restore paths only. See election.go. Elector election.Elector + // ModelVolume, when set, turns on the write-once model volume for + // Helm-function pods (docs/proposals/helm-shared-model-volume.md): + // the download lands in a per-identity shared volume, one writer, + // readers wait for the completion marker. Takes precedence over the + // gate-and-promote election above. Groups resolves identity for + // group members that name no model (LWS workers); may be nil. + // ModelWaitDeadline bounds a reader's wait before it downloads itself. + ModelVolume *modelvolume.Provisioner + Groups modelid.GroupResolver + ModelWaitDeadline time.Duration + // L2WaitImage is the nvsnap-l2-wait init-container image ref // (nvsnap#147). When non-empty, tryL2Mount prepends a // nvsnap-l2-wait init container that polls nvsnap-server's @@ -441,6 +455,12 @@ func (m *Mutator) Mutate(ctx context.Context, pod *corev1.Pod) ([]PatchOp, error // the label-driven capture inject below keeps its behaviour. An // error is logged and admits the pod unchanged: the election is // an optimisation, never a gate. + if vp, err := m.modelVolumePatches(ctx, pod); err != nil { + m.logger().WithError(err).WithField("pod", election.PodIdentity(pod)). + Warn("model volume decision failed; admitting pod unchanged") + } else if vp != nil { + return mergePatchPlan(append(injectPatches, vp...)), nil + } if ep, err := m.electionPatches(ctx, pod); err != nil { m.logger().WithError(err).WithField("pod", election.PodIdentity(pod)). Warn("election failed; admitting pod unchanged") From 6eaf5a8a28d570fe70c2fde8dcedd26d7f78d2ca Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 07:43:57 -0700 Subject: [PATCH 03/16] feat(nvsnap): complete model volumes and bind them into readers on NVMesh Step 3 of docs/proposals/helm-shared-model-volume.md. The agent gains a model volume controller. For writers, on any node: when the download init the webhook named on the pod exits 0, the writer claim is labelled complete and, on block storage, SharedVolumePromoter.MintReadOnly exposes it as a read-only claim (primary PV retained, secondary static PV with the namespace-rewritten NVMesh handle, pre-bound claim). The writer claim stays because the writer still runs on it and later writers must find it complete. For pending readers on this node: once the identity is complete the read-only claim is minted in the reader's namespace, attached to the node through a mount-holder pod, bind-mounted read-only onto the reader's hostPath landing under the Bidirectional overlays root (/models/), and the pod is un-pended. The writer touched the marker inside the volume, so the reader's wait init sees it as soon as the bind lands. Every agent watches writers (idempotent marking and minting); only the reader's own agent binds. The webhook gives Block-mode readers a hostPath landing with HostToContainer propagation on the engine and init mounts, and names the writer's download init for the agent. Storage profiles gain `modelVolume: {mode, storageClass, size}`; block is the default for shared-volume strategies, rwx must be declared. Agent flags --model-volume and --model-volume-wait-deadline, Helm agent.modelVolume.{enabled,waitDeadline}, and pods patch in the agent ClusterRole. Tests with fake clients: writer completion marks and mints the read-only PV and claim with the handle rewritten and the primary retained; a pending reader on its node gets the claim in its own namespace, one attach, one bind at the hostPath, and is un-pended, a second reader of the same identity reuses the bind, readers on other nodes are ignored. Mutation-checked: node filter dropped, completion on any exit code, binding before completion, and un-pend dropped each turn tests red. Co-Authored-By: Balaji Ganesan --- .../nvsnap/cmd/agent/main.go | 7 + .../nvsnap/templates/agent-daemonset.yaml | 5 + .../helm/nvsnap/templates/agent-rbac.yaml | 4 +- .../nvsnap/deploy/helm/nvsnap/values.yaml | 13 + .../nvsnap/internal/agent/BUILD.bazel | 6 + .../nvsnap/internal/agent/agent.go | 37 +++ .../nvsnap/internal/agent/l2_integration.go | 75 ++++- .../internal/agent/modelvolume_controller.go | 263 ++++++++++++++++++ .../agent/modelvolume_controller_test.go | 171 ++++++++++++ .../internal/agent/webhook_integration.go | 5 + .../checkpointstore/promoter_shared.go | 61 ++++ .../checkpointstore/storage_profile.go | 18 ++ .../internal/modelvolume/modelvolume.go | 9 + .../nvsnap/internal/webhook/model_volume.go | 65 ++++- .../internal/webhook/model_volume_test.go | 17 +- .../nvsnap/internal/webhook/mutate.go | 4 + 16 files changed, 734 insertions(+), 26 deletions(-) create mode 100644 src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go create mode 100644 src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go diff --git a/src/compute-plane-services/nvsnap/cmd/agent/main.go b/src/compute-plane-services/nvsnap/cmd/agent/main.go index b7e27605d4..ec588b30b2 100644 --- a/src/compute-plane-services/nvsnap/cmd/agent/main.go +++ b/src/compute-plane-services/nvsnap/cmd/agent/main.go @@ -174,6 +174,13 @@ func main() { flag.DurationVar(&config.Election.Deadline, "election-deadline", 0, "Bound on a leader's cold start plus capture; past it nvsnap-server evicts the gated followers for re-election (default 60m)") + // Write-once model volume for Helm functions + // (docs/proposals/helm-shared-model-volume.md). Needs L2. + flag.BoolVar(&config.ModelVolume.Enabled, "model-volume", false, + "Download each model once per cluster into a shared volume and attach it to every other pod that names it (needs L2)") + flag.DurationVar(&config.ModelVolume.WaitDeadline, "model-volume-wait-deadline", 0, + "How long a reader waits for the writer's download before downloading itself (default 1h)") + flag.StringVar(&config.Webhook.L2WaitImage, "webhook-l2-wait-image", "", "Image ref for the nvsnap-l2-wait init container injected onto restore pods (nvsnap#147)") diff --git a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-daemonset.yaml b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-daemonset.yaml index 47c763558e..9855664caa 100644 --- a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-daemonset.yaml +++ b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-daemonset.yaml @@ -112,6 +112,11 @@ spec: # stamped in the manifest. - --cachedir-env-file=/etc/nvsnap/cachedir-env/env {{- end }} + {{- if and .Values.agent.modelVolume .Values.agent.modelVolume.enabled }} + # write-once model volume (values: agent.modelVolume) + - --model-volume + - --model-volume-wait-deadline={{ .Values.agent.modelVolume.waitDeadline | default "1h" }} + {{- end }} {{- if and .Values.agent.election .Values.agent.election.enabled }} # one-downloader election (values: agent.election) - --election diff --git a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-rbac.yaml b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-rbac.yaml index ece0b2b4fd..94e25bf06c 100644 --- a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-rbac.yaml +++ b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-rbac.yaml @@ -41,6 +41,8 @@ metadata: rules: - apiGroups: [""] resources: ["pods"] + # patch: the model volume controller un-pends readers once their + # volume is bound (internal/agent/modelvolume_controller.go). # v0.0.51: create+delete needed for the mount-holder pod in the # source workload's namespace (see # internal/checkpointstore/mount_holder.go + @@ -48,7 +50,7 @@ rules: # compliant pause pod owned by the rwx PVC; the agent creates it # to trigger kubelet to mount the PVC, then deletes it after the # in-process file-tree copy finishes. - verbs: ["get", "list", "watch", "create", "delete"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create", "update", "delete"] diff --git a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml index a1508d9774..0b18ecbf60 100644 --- a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml +++ b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml @@ -114,6 +114,19 @@ agent: # pod is elected to capture and the rest are held by a scheduling gate # (no node, no GPU) until nvsnap-server sees the promote and releases # them. Needs agent.l2. Off until qualified on a cluster. + # Write-once model volume for Helm functions + # (docs/proposals/helm-shared-model-volume.md): the model is downloaded + # once per cluster into a per-identity volume and every other pod that + # names it attaches that volume; compile caches are shared the same way. + # Mode comes from the storage profile: block on NVMesh (the L2 class), + # rwx when the profile declares a distributed-filesystem class. Needs + # agent.l2. Off until qualified. + modelVolume: + enabled: false + # How long a reader waits for the writer's download before it downloads + # itself. Never a deadlock: past this, every pod is self-sufficient. + waitDeadline: 1h + election: enabled: false # Bound on the leader's cold start plus capture. Past it the server diff --git a/src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel b/src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel index 9f0c3148b0..65ef021ebd 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel +++ b/src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel @@ -21,6 +21,7 @@ go_library( "l2_integration.go", "l2_promote_async.go", "l2_writer.go", + "modelvolume_controller.go", "nim_backend.go", "pathsafe.go", "peer_load.go", @@ -51,6 +52,7 @@ go_library( "//src/compute-plane-services/nvsnap/internal/cuda", "//src/compute-plane-services/nvsnap/internal/election", "//src/compute-plane-services/nvsnap/internal/metrics", + "//src/compute-plane-services/nvsnap/internal/modelvolume", "//src/compute-plane-services/nvsnap/internal/objectstore", "//src/compute-plane-services/nvsnap/internal/rootfsonly", "//src/compute-plane-services/nvsnap/internal/runtime", @@ -66,7 +68,9 @@ go_library( "@io_k8s_api//coordination/v1:coordination", "@io_k8s_api//core/v1:core", "@io_k8s_apimachinery//pkg/api/errors", + "@io_k8s_apimachinery//pkg/api/resource", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_apimachinery//pkg/types", "@io_k8s_client_go//dynamic", "@io_k8s_client_go//informers", "@io_k8s_client_go//kubernetes", @@ -94,6 +98,7 @@ go_test( "l2_profile_prewarm_test.go", "l2_promote_async_test.go", "l2_writer_test.go", + "modelvolume_controller_test.go", "nim_backend_test.go", "pathsafe_test.go", "peer_fanout_test.go", @@ -116,6 +121,7 @@ go_test( embed = [":agent"], deps = [ "//src/compute-plane-services/nvsnap/internal/checkpointstore", + "//src/compute-plane-services/nvsnap/internal/modelvolume", "//src/compute-plane-services/nvsnap/internal/objectstore", "@com_github_gorilla_mux//:mux", "@com_github_sirupsen_logrus//:logrus", diff --git a/src/compute-plane-services/nvsnap/internal/agent/agent.go b/src/compute-plane-services/nvsnap/internal/agent/agent.go index 5983b54d20..ee08038ad1 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/agent.go +++ b/src/compute-plane-services/nvsnap/internal/agent/agent.go @@ -45,6 +45,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/cuda" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/election" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/metrics" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/modelvolume" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/objectstore" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/runtime" _ "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/runtime/crio" // register CRI-O factory @@ -164,6 +165,9 @@ type Config struct { // promoted rox); ignored when L2 is off. Election ElectionConfig + // ModelVolume enables the write-once model volume for Helm functions. + ModelVolume ModelVolumeConfig + // Replication is the opt-in cross-cluster replication config (the L4 // tier). See docs/design/cross-cluster-replication.md. When // Replication.ObjectStore.Provider AND HomeBucket are both non-empty, @@ -222,6 +226,16 @@ type ElectionConfig struct { Deadline time.Duration } +// ModelVolumeConfig configures docs/proposals/helm-shared-model-volume.md. +type ModelVolumeConfig struct { + // Enabled turns the write-once model volume on. Needs L2 and a storage + // profile that resolves a mode (block on NVMesh, rwx when declared). + Enabled bool + // WaitDeadline bounds a reader's wait for the marker before it downloads + // itself. Zero means one hour. + WaitDeadline time.Duration +} + // L2BackendConfig is the per-capture PVC L2 backend (nvsnap#63). See // docs/L2-PVC-CRIU-DESIGN.md. type L2BackendConfig struct { @@ -294,6 +308,10 @@ type Agent struct { // elector is the admission election, built with the L2 backend when // Election.Enabled; nil keeps the webhook on its explicit paths. elector election.Elector + // modelVolume and modelMinter are built with the L2 backend when + // ModelVolume.Enabled; the webhook and the controller share them. + modelVolume *modelvolume.Provisioner + modelMinter *checkpointstore.SharedVolumePromoter // kubeClient is the shared K8s API client used by the rootfs-only // capture watcher AND the admission-webhook cascade-fetch path @@ -642,6 +660,25 @@ func (a *Agent) Run(ctx context.Context) error { if err := a.startWebhook(ctx, a.config.Webhook, backend); err != nil { a.log.WithError(err).Error("agent admission webhook failed to start; continuing without it") } + if a.modelVolume != nil { + // Completes writer volumes and binds them into pending readers on + // this node (docs/proposals/helm-shared-model-volume.md). + mvc := &ModelVolumeController{ + Kube: a.kubeClient, + Provisioner: a.modelVolume, + Minter: a.modelMinter, + NodeName: a.config.NodeName, + HostRoot: filepath.Join(a.config.OverlayRoot, "models"), + HolderImage: a.config.L2.WriterImage, + HolderPullSecrets: l2PullSecrets(a.config.L2), + Log: a.log.WithField("subsys", "modelvolume"), + } + go func() { + if err := mvc.Run(ctx); err != nil { + a.log.WithError(err).Error("model volume controller stopped") + } + }() + } // nvsnap#194: OverlayFS cleanup-on-pod-delete + startup sweep. Safe // to call regardless of whether the webhook is enabled — if no diff --git a/src/compute-plane-services/nvsnap/internal/agent/l2_integration.go b/src/compute-plane-services/nvsnap/internal/agent/l2_integration.go index d73b2e3b39..19b63bdd80 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/l2_integration.go +++ b/src/compute-plane-services/nvsnap/internal/agent/l2_integration.go @@ -32,6 +32,7 @@ import ( "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" @@ -39,6 +40,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/checkpointstore" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/election" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/modelvolume" ) // storageProfilesConfigMap is the optional per-cluster overlay that @@ -162,14 +164,7 @@ func (a *Agent) startL2Backend(_ context.Context, cfg L2BackendConfig) (checkpoi // Default the mount-holder pull secret unless the operator // explicitly cleared it. The "-" sentinel disables (no secret). - pullSecret := cfg.WriterPullSecret - if pullSecret == "" { - pullSecret = DefaultWriterPullSecret - } - var pullSecrets []string - if pullSecret != "" && pullSecret != "-" { - pullSecrets = []string{pullSecret} - } + pullSecrets := l2PullSecrets(cfg) // Resolve the storage strategy from the L2 SC's provisioner + // parameters.type (nvsnap#171). nil ⇒ the backend's applyDefaults @@ -177,10 +172,17 @@ func (a *Agent) startL2Backend(_ context.Context, cfg L2BackendConfig) (checkpoi // behavior) — back-compat for clusters with no profile match. promoter, profile := resolveL2Promoter(context.Background(), kc, dyn, cfg.StorageClass, cfg.Namespace, log) a.l2Profile = profile - if a.config.Election.Enabled { + if a.config.Election.Enabled || a.config.ModelVolume.Enabled { a.elector = &election.LeaseElector{KubeClient: kc, Namespace: cfg.Namespace, Deadline: a.config.Election.Deadline} log.WithField("deadline", a.config.Election.Deadline).Info("admission election enabled (one downloader per hash)") } + if a.config.ModelVolume.Enabled { + if mv, minter, err := buildModelVolume(kc, profile, promoter, cfg.StorageClass, log); err != nil { + log.WithError(err).Warn("model volume disabled") + } else { + a.modelVolume, a.modelMinter = mv, minter + } + } // SnapshotClass is only meaningful for the snapshot-clone strategy. // Shared-volume backends (NVMesh/EFS/Filestore) never snapshot — they @@ -346,3 +348,58 @@ func vramGBFromGPUType(gpuType string) string { } return "" } + +// buildModelVolume derives the write-once model volume setup from the +// storage profile: "block" on shared-volume strategies (NVMesh), whose +// promoter mints the read-only claims; "rwx" when the profile declares a +// distributed-filesystem class. Anything else leaves Helm functions alone. +func buildModelVolume(kc kubernetes.Interface, profile *checkpointstore.StorageProfile, promoter checkpointstore.Promoter, l2Class string, log logrus.FieldLogger) (*modelvolume.Provisioner, *checkpointstore.SharedVolumePromoter, error) { + cfg := modelvolume.Config{StorageClass: l2Class, Size: resource.MustParse("512Gi")} + var mvp *checkpointstore.ModelVolumeProfile + if profile != nil { + mvp = profile.ModelVolume + } + switch { + case mvp != nil && mvp.Mode == string(modelvolume.ModeRWX): + cfg.Mode = modelvolume.ModeRWX + case mvp != nil && mvp.Mode == string(modelvolume.ModeBlock), mvp == nil && profile != nil && profile.Strategy == checkpointstore.StrategySharedVolume: + cfg.Mode = modelvolume.ModeBlock + default: + return nil, nil, errors.New("storage profile resolves no model volume mode (block needs a shared-volume strategy such as NVMesh; rwx must be declared with a distributed filesystem class)") + } + if mvp != nil { + if mvp.StorageClass != "" { + cfg.StorageClass = mvp.StorageClass + } + if mvp.Size != "" { + q, err := resource.ParseQuantity(mvp.Size) + if err != nil { + return nil, nil, fmt.Errorf("modelVolume.size %q: %w", mvp.Size, err) + } + cfg.Size = q + } + } + var minter *checkpointstore.SharedVolumePromoter + if cfg.Mode == modelvolume.ModeBlock { + sp, ok := promoter.(*checkpointstore.SharedVolumePromoter) + if !ok { + return nil, nil, errors.New("block mode needs the shared-volume promoter to mint read-only claims") + } + minter = sp + } + log.WithFields(logrus.Fields{"mode": cfg.Mode, "storage_class": cfg.StorageClass, "size": cfg.Size.String()}).Info("model volume enabled (one download per model per cluster)") + return &modelvolume.Provisioner{Kube: kc, Cfg: cfg}, minter, nil +} + +// l2PullSecrets resolves the mount-holder / writer pull secret list: the +// configured secret, the default unless cleared with "-", or none. +func l2PullSecrets(cfg L2BackendConfig) []string { + pullSecret := cfg.WriterPullSecret + if pullSecret == "" { + pullSecret = DefaultWriterPullSecret + } + if pullSecret != "" && pullSecret != "-" { + return []string{pullSecret} + } + return nil +} diff --git a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go new file mode 100644 index 0000000000..2c594c93c3 --- /dev/null +++ b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go @@ -0,0 +1,263 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package agent + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/checkpointstore" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/modelvolume" +) + +// ModelVolumeController is the agent half of +// docs/proposals/helm-shared-model-volume.md. +// +// - Writers (any node): when the download init named on the pod exits +// 0, the writer claim is labelled complete and, on block storage, the +// read-only claim is minted in the writer's namespace. +// - Pending readers on this node (block storage): once the identity is +// complete, the read-only claim is minted in the reader's namespace, +// attached to this node through a mount-holder, bind-mounted read-only +// onto the reader's hostPath landing (under the Bidirectional overlays +// root, so kubelet's mount sees it), and the pod is un-pended. The +// writer touched the marker inside the volume, so the reader's wait +// init sees it as soon as the bind lands. +// +// Every agent watches writers; marking and minting are idempotent, so the +// race between agents is harmless. Only the agent on the reader's node +// binds for it. +type ModelVolumeController struct { + Kube kubernetes.Interface + Provisioner *modelvolume.Provisioner + // Minter mints read-only claims on block storage; nil in RWX mode. + Minter *checkpointstore.SharedVolumePromoter + // NodeName is this agent's node; readers elsewhere are ignored. + NodeName string + // HostRoot is where model volumes are bound for readers: /. + // Must be under the agent's Bidirectional overlays mount. + HostRoot string + // HolderNamespaceImage is the image for mount-holder pods (the agent image). + HolderImage string + HolderPullSecrets []string + Log logrus.FieldLogger + + // Seams for tests: attach returns the host path a claim is mounted at + // on this node; bind bind-mounts src onto dst read-only. + attach func(ctx context.Context, ns, claim string) (string, error) + bind func(src, dst string) error + + mu sync.Mutex + bound map[string]bool // dst paths already bound + holders map[string]*checkpointstore.MountHolder +} + +// Run starts the informer and blocks until ctx is done. +func (c *ModelVolumeController) Run(ctx context.Context) error { + c.init() + factory := informers.NewSharedInformerFactoryWithOptions(c.Kube, 30*time.Second, + informers.WithTweakListOptions(func(o *metav1.ListOptions) { o.LabelSelector = modelvolume.IdentityLabel })) + informer := factory.Core().V1().Pods().Informer() + if _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { c.handle(ctx, obj) }, + UpdateFunc: func(_, obj any) { c.handle(ctx, obj) }, + }); err != nil { + return fmt.Errorf("AddEventHandler: %w", err) + } + factory.Start(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) { + return fmt.Errorf("model volume informer did not sync") + } + c.log().WithFields(logrus.Fields{"node": c.NodeName, "mode": c.Provisioner.Cfg.Mode, "host_root": c.HostRoot}).Info("model volume controller started") + <-ctx.Done() + return nil +} + +func (c *ModelVolumeController) init() { + if c.bound == nil { + c.bound = map[string]bool{} + } + if c.holders == nil { + c.holders = map[string]*checkpointstore.MountHolder{} + } + if c.attach == nil { + c.attach = c.attachWithHolder + } + if c.bind == nil { + c.bind = bindReadOnly + } +} + +func (c *ModelVolumeController) log() logrus.FieldLogger { + if c.Log != nil { + return c.Log + } + return logrus.NewEntry(logrus.New()).WithField("subsys", "modelvolume") +} + +// handle dispatches one pod event; exported for tests as Handle. +func (c *ModelVolumeController) handle(ctx context.Context, obj any) { + pod, ok := obj.(*corev1.Pod) + if !ok || pod == nil { + return + } + c.init() + uri := pod.Annotations[modelvolume.IdentityAnnotation] + if uri == "" { + return + } + switch pod.Labels[modelvolume.RoleLabel] { + case "writer": + c.handleWriter(ctx, pod, uri) + case "reader": + if pod.Labels[modelvolume.PendingLabel] == "true" && pod.Spec.NodeName == c.NodeName { + c.handlePendingReader(ctx, pod, uri) + } + } +} + +// Handle is the test entry point for one pod event. +func (c *ModelVolumeController) Handle(ctx context.Context, pod *corev1.Pod) { c.handle(ctx, pod) } + +func (c *ModelVolumeController) handleWriter(ctx context.Context, pod *corev1.Pod, uri string) { + initName := pod.Annotations[modelvolume.DownloadInitAnnotation] + if initName == "" || !initExitedZero(pod, initName) { + return + } + log := c.log().WithFields(logrus.Fields{"pod": pod.Namespace + "/" + pod.Name, "model": uri}) + if err := c.Provisioner.MarkComplete(ctx, uri, pod.Namespace); err != nil { + log.WithError(err).Warn("model volume: mark complete failed") + return + } + if c.Provisioner.Cfg.Mode == modelvolume.ModeBlock && c.Minter != nil { + if err := c.Minter.MintReadOnly(ctx, pod.Namespace, modelvolume.ClaimName(uri), modelvolume.ReadOnlyPVName(uri, pod.Namespace), modelvolume.ReadOnlyClaimName(uri), pod.Namespace, modelvolume.Key(uri)); err != nil { + log.WithError(err).Warn("model volume: mint read-only claim failed") + return + } + } + log.Info("model volume: download complete; readers may attach") +} + +func initExitedZero(pod *corev1.Pod, name string) bool { + for i := range pod.Status.InitContainerStatuses { + s := &pod.Status.InitContainerStatuses[i] + if s.Name == name { + return s.State.Terminated != nil && s.State.Terminated.ExitCode == 0 + } + } + return false +} + +func (c *ModelVolumeController) handlePendingReader(ctx context.Context, pod *corev1.Pod, uri string) { + log := c.log().WithFields(logrus.Fields{"pod": pod.Namespace + "/" + pod.Name, "model": uri, "node": c.NodeName}) + st, err := c.Provisioner.Lookup(ctx, uri) + if err != nil { + log.WithError(err).Warn("model volume: lookup failed") + return + } + if !st.Complete { + return // the wait init keeps waiting; the writer's completion re-triggers via its own event + } + dst := filepath.Join(c.HostRoot, modelvolume.Key(uri)) + c.mu.Lock() + already := c.bound[dst] + c.mu.Unlock() + if !already { + if c.Minter != nil { + if err := c.Minter.MintReadOnly(ctx, st.ClaimNamespace, modelvolume.ClaimName(uri), modelvolume.ReadOnlyPVName(uri, pod.Namespace), modelvolume.ReadOnlyClaimName(uri), pod.Namespace, modelvolume.Key(uri)); err != nil { + log.WithError(err).Warn("model volume: mint read-only claim in reader namespace failed") + return + } + } + src, err := c.attach(ctx, pod.Namespace, modelvolume.ReadOnlyClaimName(uri)) + if err != nil { + log.WithError(err).Warn("model volume: attach read-only claim to node failed") + return + } + if err := os.MkdirAll(dst, 0o755); err != nil { + log.WithError(err).Warn("model volume: create bind target failed") + return + } + if err := c.bind(src, dst); err != nil { + log.WithError(err).Warn("model volume: bind failed") + return + } + c.mu.Lock() + c.bound[dst] = true + c.mu.Unlock() + log.WithFields(logrus.Fields{"src": src, "dst": dst}).Info("model volume: bound read-only volume for reader") + } + patch, _ := json.Marshal(map[string]any{"metadata": map[string]any{"labels": map[string]string{modelvolume.PendingLabel: "false"}}}) + if _, err := c.Kube.CoreV1().Pods(pod.Namespace).Patch(ctx, pod.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + log.WithError(err).Warn("model volume: un-pend reader failed") + } +} + +// attachWithHolder mounts the claim on this node through a mount-holder pod +// and returns the agent-visible path of the mounted volume. The holder +// stays for the life of the agent; the volume is read-only and shared. +func (c *ModelVolumeController) attachWithHolder(ctx context.Context, ns, claim string) (string, error) { + key := ns + "/" + claim + c.mu.Lock() + h := c.holders[key] + c.mu.Unlock() + if h == nil { + pvc, err := c.Kube.CoreV1().PersistentVolumeClaims(ns).Get(ctx, claim, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("get claim %s: %w", key, err) + } + entry, _ := c.log().(*logrus.Entry) + if entry == nil { + entry = logrus.NewEntry(logrus.New()) + } + name := "nvsnap-model-holder-" + strings.TrimPrefix(claim, "nvsnap-model-") + "-" + shortNode(c.NodeName) + h = checkpointstore.NewMountHolder(c.Kube, entry, ns, name, c.NodeName, claim, pvc.UID, c.HolderImage, "/host", c.HolderPullSecrets) + if err := h.Create(ctx); err != nil { + return "", fmt.Errorf("create mount-holder: %w", err) + } + if err := h.WaitRunning(ctx); err != nil { + return "", fmt.Errorf("mount-holder not running: %w", err) + } + c.mu.Lock() + c.holders[key] = h + c.mu.Unlock() + } + return h.PVMountPath() +} + +func shortNode(n string) string { + n = strings.SplitN(n, ".", 2)[0] + if len(n) > 20 { + n = n[len(n)-20:] + } + return n +} + +// bindReadOnly bind-mounts src onto dst and remounts it read-only. dst is +// under the agent's Bidirectional overlays root, so the mount propagates +// to the host and into pods whose mounts propagate HostToContainer. +func bindReadOnly(src, dst string) error { + if err := syscall.Mount(src, dst, "", syscall.MS_BIND|syscall.MS_REC, ""); err != nil { + return fmt.Errorf("bind %s -> %s: %w", src, dst, err) + } + if err := syscall.Mount("", dst, "", syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY, ""); err != nil { + return fmt.Errorf("remount %s read-only: %w", dst, err) + } + return nil +} diff --git a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go new file mode 100644 index 0000000000..16a2e0ad3a --- /dev/null +++ b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package agent + +import ( + "context" + "path/filepath" + "testing" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/checkpointstore" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/modelvolume" +) + +const mvURI = "ngc://org/team/nemotron3-ultra-genrm:bf16-fixed" + +// writerFixture: the writer claim bound to an NVMesh PV, as after the +// webhook created it and the CSI provisioner bound it. +func writerFixture(t *testing.T) (*fake.Clientset, *modelvolume.Provisioner) { + t.Helper() + sc := "nvcf-sc" + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "pvc-abc"}, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("512Gi")}, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + PersistentVolumeSource: corev1.PersistentVolumeSource{CSI: &corev1.CSIPersistentVolumeSource{Driver: "nvmesh-csi.excelero.com", VolumeHandle: "cluster:csi-abc:vol:sr-fn"}}, + }, + } + kc := fake.NewSimpleClientset(pv) + p := &modelvolume.Provisioner{Kube: kc, Cfg: modelvolume.Config{Mode: modelvolume.ModeBlock, StorageClass: sc, Size: resource.MustParse("512Gi")}} + if _, err := p.EnsureWriterClaim(context.Background(), mvURI, "sr-fn"); err != nil { + t.Fatal(err) + } + pvc, _ := kc.CoreV1().PersistentVolumeClaims("sr-fn").Get(context.Background(), modelvolume.ClaimName(mvURI), metav1.GetOptions{}) + pvc.Spec.VolumeName = "pvc-abc" + if _, err := kc.CoreV1().PersistentVolumeClaims("sr-fn").Update(context.Background(), pvc, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + return kc, p +} + +func mvController(t *testing.T, kc *fake.Clientset, p *modelvolume.Provisioner, node string) (c *ModelVolumeController, attached *[]string, bound *[][2]string) { + t.Helper() + tx, _ := checkpointstore.LookupVolumeHandleTransform("nvmesh") + minter := &checkpointstore.SharedVolumePromoter{KubeClient: kc, StorageClass: "nvcf-sc", Transform: tx, MountOptions: []string{"ro", "norecovery", "nouuid"}, Log: logrus.New()} + att := []string{} + bnd := [][2]string{} + c = &ModelVolumeController{Kube: kc, Provisioner: p, Minter: minter, NodeName: node, HostRoot: filepath.Join(t.TempDir(), "models"), Log: logrus.New()} + c.attach = func(_ context.Context, ns, claim string) (string, error) { + att = append(att, ns+"/"+claim) + return "/host/var/lib/kubelet/pods/h/volumes/kubernetes.io~csi/pv/mount", nil + } + c.bind = func(src, dst string) error { bnd = append(bnd, [2]string{src, dst}); return nil } + return c, &att, &bnd +} + +func writerPod(exit *int32) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "w-0", Namespace: "sr-fn", + Labels: map[string]string{modelvolume.IdentityLabel: modelvolume.Key(mvURI), modelvolume.RoleLabel: "writer"}, + Annotations: map[string]string{modelvolume.IdentityAnnotation: mvURI, modelvolume.DownloadInitAnnotation: "download-ngc-model"}}, + Spec: corev1.PodSpec{NodeName: "node-a"}, + } + if exit != nil { + p.Status.InitContainerStatuses = []corev1.ContainerStatus{{Name: "download-ngc-model", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: *exit}}}} + } + return p +} + +func readerPod(ns, node string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "r-1", Namespace: ns, + Labels: map[string]string{modelvolume.IdentityLabel: modelvolume.Key(mvURI), modelvolume.RoleLabel: "reader", modelvolume.PendingLabel: "true"}, + Annotations: map[string]string{modelvolume.IdentityAnnotation: mvURI, modelvolume.LandingAnnotation: "/config/models"}}, + Spec: corev1.PodSpec{NodeName: node}, + } +} + +func TestModelVolumeController_WriterCompletionMintsReadOnly(t *testing.T) { + kc, p := writerFixture(t) + c, _, _ := mvController(t, kc, p, "node-a") + ctx := context.Background() + + c.Handle(ctx, writerPod(nil)) // init still running + if st, _ := p.Lookup(ctx, mvURI); st.Complete { + t.Fatal("no completion before the download init exits") + } + one := int32(1) + c.Handle(ctx, writerPod(&one)) // init failed + if st, _ := p.Lookup(ctx, mvURI); st.Complete { + t.Fatal("a failed download must not complete the volume") + } + zero := int32(0) + c.Handle(ctx, writerPod(&zero)) + st, _ := p.Lookup(ctx, mvURI) + if !st.Complete { + t.Fatal("exit 0 must mark the claim complete") + } + ro, err := kc.CoreV1().PersistentVolumeClaims("sr-fn").Get(ctx, modelvolume.ReadOnlyClaimName(mvURI), metav1.GetOptions{}) + if err != nil { + t.Fatalf("read-only claim must be minted in the writer namespace: %v", err) + } + pv, err := kc.CoreV1().PersistentVolumes().Get(ctx, ro.Spec.VolumeName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if pv.Spec.CSI.VolumeHandle != "cluster:csi-abc:vol:sr-fn" || !pv.Spec.CSI.ReadOnly || pv.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain { + t.Errorf("read-only PV: %+v", pv.Spec) + } + primary, _ := kc.CoreV1().PersistentVolumes().Get(ctx, "pvc-abc", metav1.GetOptions{}) + if primary.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain { + t.Error("the writer's PV must be retained; it is the artifact") + } + if _, err := kc.CoreV1().PersistentVolumeClaims("sr-fn").Get(ctx, modelvolume.ClaimName(mvURI), metav1.GetOptions{}); err != nil { + t.Error("the writer claim stays: the writer is still running on it") + } + c.Handle(ctx, writerPod(&zero)) // idempotent +} + +func TestModelVolumeController_PendingReaderBoundOnItsNode(t *testing.T) { + kc, p := writerFixture(t) + ctx := context.Background() + c, attached, bound := mvController(t, kc, p, "node-b") + reader := readerPod("other-ns", "node-b") + if _, err := kc.CoreV1().Pods("other-ns").Create(ctx, reader, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + + c.Handle(ctx, reader) // not complete yet + if len(*attached) != 0 || len(*bound) != 0 { + t.Fatal("nothing may be attached before the download completes") + } + zero := int32(0) + cw, _, _ := mvController(t, kc, p, "node-a") + cw.Handle(ctx, writerPod(&zero)) + + // Readers on other nodes are not this agent's business, even before + // anything is bound here. + c.Handle(ctx, readerPod("other-ns", "node-c")) + if len(*attached) != 0 { + t.Fatal("a reader on another node must be ignored") + } + + c.Handle(ctx, reader) + if len(*attached) != 1 || (*attached)[0] != "other-ns/"+modelvolume.ReadOnlyClaimName(mvURI) { + t.Errorf("reader's read-only claim in its own namespace must be attached, got %v", *attached) + } + if _, err := kc.CoreV1().PersistentVolumeClaims("other-ns").Get(ctx, modelvolume.ReadOnlyClaimName(mvURI), metav1.GetOptions{}); err != nil { + t.Errorf("read-only claim must be minted in the reader namespace: %v", err) + } + wantDst := filepath.Join(c.HostRoot, modelvolume.Key(mvURI)) + if len(*bound) != 1 || (*bound)[0][1] != wantDst || (*bound)[0][0] == "" { + t.Errorf("bind must land on the reader's hostPath %s, got %v", wantDst, *bound) + } + got, _ := kc.CoreV1().Pods("other-ns").Get(ctx, "r-1", metav1.GetOptions{}) + if got.Labels[modelvolume.PendingLabel] != "false" { + t.Errorf("reader must be un-pended, labels %v", got.Labels) + } + // A second reader of the same identity on this node reuses the bind. + c.Handle(ctx, reader) + if len(*bound) != 1 { + t.Error("the bind is per identity per node, not per pod") + } +} diff --git a/src/compute-plane-services/nvsnap/internal/agent/webhook_integration.go b/src/compute-plane-services/nvsnap/internal/agent/webhook_integration.go index ef7c51d914..2a78cb9295 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/webhook_integration.go +++ b/src/compute-plane-services/nvsnap/internal/agent/webhook_integration.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/checkpointstore" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/rootfsonly" @@ -155,6 +156,10 @@ func (a *Agent) startWebhook(ctx context.Context, cfg WebhookConfig, backend che // One-downloader election for chart pods; nil when off or L2 is // off (docs/proposals/helm-chart-cache-election.md). Elector: a.elector, + // Write-once model volume for Helm functions; nil when off. + ModelVolume: a.modelVolume, + ModelWaitDeadline: a.config.ModelVolume.WaitDeadline, + ModelHostRoot: filepath.Join(a.config.OverlayRoot, "models"), Composer: &rootfsonly.HashInputComposer{ CUDADriverMajor: a.config.RootfsCapture.CUDADriverMajor, }, diff --git a/src/compute-plane-services/nvsnap/internal/checkpointstore/promoter_shared.go b/src/compute-plane-services/nvsnap/internal/checkpointstore/promoter_shared.go index 519941a4e3..bf144eef44 100644 --- a/src/compute-plane-services/nvsnap/internal/checkpointstore/promoter_shared.go +++ b/src/compute-plane-services/nvsnap/internal/checkpointstore/promoter_shared.go @@ -535,3 +535,64 @@ func (p *SharedVolumePromoter) deleteNamespacedClaims(ctx context.Context, hash } return errs } + +// MintReadOnly exposes the write-once model volume (a writer claim that +// has finished downloading) as a read-only claim in ns, for +// docs/proposals/helm-shared-model-volume.md on block storage. Same +// mechanics as the promote: the primary PV is kept (Retain), a secondary +// static PV with the namespace-rewritten handle is pre-bound to roClaim +// in ns. Unlike the promote the writer claim is kept: the writer pod is +// still running on it and later writers of the same identity must find +// it complete. Idempotent. +func (p *SharedVolumePromoter) MintReadOnly(ctx context.Context, writerNS, writerClaim, roPVName, roClaim, ns, labelKey string) error { + p.applyDefaults() + writer, err := p.KubeClient.CoreV1().PersistentVolumeClaims(writerNS).Get(ctx, writerClaim, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get writer claim %s/%s: %w", writerNS, writerClaim, err) + } + if writer.Spec.VolumeName == "" { + return fmt.Errorf("writer claim %s/%s has no bound PV yet", writerNS, writerClaim) + } + primary, err := p.KubeClient.CoreV1().PersistentVolumes().Get(ctx, writer.Spec.VolumeName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get primary PV %s: %w", writer.Spec.VolumeName, err) + } + if primary.Spec.CSI == nil || primary.Spec.CSI.VolumeHandle == "" { + return fmt.Errorf("primary PV %s has no CSI volumeHandle", primary.Name) + } + if primary.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain { + primary.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain + if _, err := p.KubeClient.CoreV1().PersistentVolumes().Update(ctx, primary, metav1.UpdateOptions{}); err != nil && !apierrors.IsConflict(err) { + return fmt.Errorf("set primary PV %s reclaim=Retain: %w", primary.Name, err) + } + } + labels := map[string]string{labelNamespace: ns} + if labelKey != "" { + labels["nvsnap.io/model"] = labelKey + } + if err := p.ensureSecondaryPV(ctx, primary, roPVName, roClaim, ns, labels); err != nil { + return err + } + if _, err := p.KubeClient.CoreV1().PersistentVolumeClaims(ns).Get(ctx, roClaim, metav1.GetOptions{}); err == nil { + return nil + } else if !apierrors.IsNotFound(err) { + return fmt.Errorf("get ro claim %s/%s: %w", ns, roClaim, err) + } + sc := p.StorageClass + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: roClaim, Namespace: ns, + Labels: map[string]string{"app.kubernetes.io/managed-by": "nvsnap", "nvsnap.io/role": "reader", labelNamespace: ns, "nvsnap.io/model": labelKey}, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany}, + VolumeName: roPVName, + StorageClassName: &sc, + Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: primary.Spec.Capacity[corev1.ResourceStorage]}}, + }, + } + if _, err := p.KubeClient.CoreV1().PersistentVolumeClaims(ns).Create(ctx, pvc, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create ro claim %s/%s: %w", ns, roClaim, err) + } + return nil +} diff --git a/src/compute-plane-services/nvsnap/internal/checkpointstore/storage_profile.go b/src/compute-plane-services/nvsnap/internal/checkpointstore/storage_profile.go index c71b491c4f..71b74a5464 100644 --- a/src/compute-plane-services/nvsnap/internal/checkpointstore/storage_profile.go +++ b/src/compute-plane-services/nvsnap/internal/checkpointstore/storage_profile.go @@ -81,6 +81,24 @@ type StorageProfile struct { // PrewarmParallelism is the number of concurrent readers in the sweep. // 0 means DefaultPrewarmParallelism. PrewarmParallelism int `json:"prewarmParallelism,omitempty"` + // ModelVolume configures the write-once model volume for Helm + // functions (docs/proposals/helm-shared-model-volume.md). Mode "block" + // is the default for shared-volume strategies (NVMesh): the writer's + // claim on the L2 class becomes the read-only artifact. Mode "rwx" + // needs a ReadWriteMany class of a distributed filesystem. Empty + // mode with no default leaves Helm functions untouched. + ModelVolume *ModelVolumeProfile `json:"modelVolume,omitempty"` +} + +// ModelVolumeProfile is the per-storage-class model volume setting. +type ModelVolumeProfile struct { + // Mode is "rwx" or "block". + Mode string `json:"mode"` + // StorageClass for the volume; empty uses the L2 class. + StorageClass string `json:"storageClass,omitempty"` + // Size requested per volume (a ceiling; the model size is unknown at + // admission). Empty means "512Gi". + Size string `json:"size,omitempty"` } // DefaultPrewarmParallelism is the reader count when a profile does not set diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go index 5764af85bc..7d76c3666e 100644 --- a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go @@ -72,6 +72,9 @@ const ( LandingAnnotation = "nvsnap.io/model-landing" // MarkerFile at the volume root says the download completed. MarkerFile = ".nvsnap-complete" + // DownloadInitAnnotation on a writer names the init container whose + // exit 0 means the download completed. + DownloadInitAnnotation = "nvsnap.io/model-download-init" managedBy = "nvsnap" ) @@ -99,6 +102,12 @@ func ClaimName(uri string) string { return "nvsnap-model-" + Key(uri) } // ReadOnlyClaimName is the Block-mode read-only claim minted after completion. func ReadOnlyClaimName(uri string) string { return "nvsnap-model-" + Key(uri) + "-ro" } +// ReadOnlyPVName is the static PV behind ReadOnlyClaimName in ns. +func ReadOnlyPVName(uri, ns string) string { + sum := sha256.Sum256([]byte(ns)) + return "nvsnap-model-" + Key(uri) + "-ro-" + hex.EncodeToString(sum[:4]) +} + // State of an identity on the cluster, as the webhook needs it. type State struct { // Exists: a writer claim exists (a download is in flight or done). diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go index 8f549153b4..ae727334d0 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go @@ -36,8 +36,9 @@ import ( // for the agent to bind the completed volume in and drop the marker. const ( - modelVolumeName = "nvsnap-model" - waitScriptDeadline = 3600 // seconds, when no deadline is configured + modelVolumeName = "nvsnap-model" + injectedDownloadInit = "nvsnap-model-download" + waitScriptDeadline = 3600 // seconds, when no deadline is configured ) // modelVolumePatches is the Helm-function decision. (nil, nil) means the @@ -88,6 +89,11 @@ func (m *Mutator) modelVolumePatches(ctx context.Context, pod *corev1.Pod) ([]Pa return nil, err } patches = append(patches, mp.label(modelvolume.RoleLabel, "writer")...) + if land.Downloader == modelid.DownloaderInit { + patches = append(patches, mp.annotation(modelvolume.DownloadInitAnnotation, land.InitContainer)...) + } else if res.Identity.Scheme == "hf" { + patches = append(patches, mp.annotation(modelvolume.DownloadInitAnnotation, injectedDownloadInit)...) + } patches = append(patches, m.substituteLandingVolume(pod, main, land, claim)...) patches = append(patches, m.downloadStepPatches(pod, main, land, res.Identity, true)...) patches = append(patches, m.modelCacheEnvPatches(pod, main, land, uri)...) @@ -111,9 +117,7 @@ func (m *Mutator) modelVolumePatches(ctx context.Context, pod *corev1.Pod) ([]Pa patches = append(patches, mp.label(modelvolume.RoleLabel, "reader")...) patches = append(patches, mp.label(modelvolume.PendingLabel, "true")...) patches = append(patches, mp.annotation(modelvolume.LandingAnnotation, landingMount(land))...) - if land.Kind == modelid.VolumeRootfs { - patches = append(patches, m.addLandingEmptyDir(pod, main, land)...) - } + patches = append(patches, m.hostPathLanding(pod, main, land, uri)...) patches = append(patches, m.downloadStepPatches(pod, main, land, res.Identity, false)...) patches = append(patches, m.modelCacheEnvPatches(pod, main, land, uri)...) log.WithField("complete", st.Complete).Info("model volume: reader on block storage; agent binds the volume after completion") @@ -182,10 +186,40 @@ func (m *Mutator) substituteLandingVolume(pod *corev1.Pod, main *corev1.Containe return patches } -// addLandingEmptyDir gives a Block-mode reader whose download wrote to the -// container filesystem an emptyDir at the landing path, so the agent has a -// mount point to bind the completed volume over. -func (m *Mutator) addLandingEmptyDir(pod *corev1.Pod, main *corev1.Container, land modelid.Landing) []PatchOp { +// hostPathLanding gives a Block-mode reader a hostPath at the landing path +// under the agent's model host root (the Bidirectional overlays root, so +// agent mounts reach kubelet). The agent bind-mounts the completed +// read-only volume there once it exists; HostToContainer propagation lets +// the already-running wait init and the engine see it appear. The pod +// schedules immediately: a hostPath never blocks volume binding. +func (m *Mutator) hostPathLanding(pod *corev1.Pod, main *corev1.Container, land modelid.Landing, uri string) []PatchOp { + root := m.ModelHostRoot + if root == "" { + root = "/var/lib/containerd/nvsnap-overlays/models" + } + hp := corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: path.Join(root, modelvolume.Key(uri)), Type: hostPathType(corev1.HostPathDirectoryOrCreate)}} + prop := corev1.MountPropagationHostToContainer + if land.VolumeName != "" { + var patches []PatchOp + for i := range pod.Spec.Volumes { + if pod.Spec.Volumes[i].Name == land.VolumeName { + patches = append(patches, PatchOp{Op: "replace", Path: fmt.Sprintf("/spec/volumes/%d", i), Value: corev1.Volume{Name: land.VolumeName, VolumeSource: hp}}) + } + } + for j := range main.VolumeMounts { + if main.VolumeMounts[j].Name == land.VolumeName { + patches = append(patches, PatchOp{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts/%d/mountPropagation", m.MainContainer, j), Value: prop}) + } + } + for i := range pod.Spec.InitContainers { + for j := range pod.Spec.InitContainers[i].VolumeMounts { + if pod.Spec.InitContainers[i].VolumeMounts[j].Name == land.VolumeName { + patches = append(patches, PatchOp{Op: "add", Path: fmt.Sprintf("/spec/initContainers/%d/volumeMounts/%d/mountPropagation", i, j), Value: prop}) + } + } + } + return patches + } patches := []PatchOp{} if pod.Spec.Volumes == nil { patches = append(patches, PatchOp{Op: "add", Path: "/spec/volumes", Value: []any{}}) @@ -194,11 +228,13 @@ func (m *Mutator) addLandingEmptyDir(pod *corev1.Pod, main *corev1.Container, la patches = append(patches, PatchOp{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts", m.MainContainer), Value: []any{}}) } return append(patches, - PatchOp{Op: "add", Path: "/spec/volumes/-", Value: corev1.Volume{Name: modelVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, - PatchOp{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts/-", m.MainContainer), Value: corev1.VolumeMount{Name: modelVolumeName, MountPath: land.Path}}, + PatchOp{Op: "add", Path: "/spec/volumes/-", Value: corev1.Volume{Name: modelVolumeName, VolumeSource: hp}}, + PatchOp{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts/-", m.MainContainer), Value: corev1.VolumeMount{Name: modelVolumeName, MountPath: land.Path, MountPropagation: &prop}}, ) } +func hostPathType(t corev1.HostPathType) *corev1.HostPathType { return &t } + // downloadStepPatches turns the download into a write-once step. With an // init container: the writer's init is wrapped to touch the marker on // success, a reader's init to wait for the marker and skip. Without one @@ -249,19 +285,20 @@ func (m *Mutator) downloadStepPatches(pod *corev1.Pod, main *corev1.Container, l script = writerScript(download, marker) } init := corev1.Container{ - Name: "nvsnap-model-download", + Name: injectedDownloadInit, Image: main.Image, Command: []string{"/bin/sh", "-c"}, Args: []string{script}, Env: append([]corev1.EnvVar{{Name: "HF_HOME", Value: land.Path}}, tokenEnv(main)...), } + prop := corev1.MountPropagationHostToContainer for _, vm := range main.VolumeMounts { if vm.Name == land.VolumeName || vm.Name == modelVolumeName { - init.VolumeMounts = append(init.VolumeMounts, corev1.VolumeMount{Name: vm.Name, MountPath: vm.MountPath}) + init.VolumeMounts = append(init.VolumeMounts, corev1.VolumeMount{Name: vm.Name, MountPath: vm.MountPath, MountPropagation: &prop}) } } if len(init.VolumeMounts) == 0 { - init.VolumeMounts = []corev1.VolumeMount{{Name: modelVolumeName, MountPath: land.Path}} + init.VolumeMounts = []corev1.VolumeMount{{Name: modelVolumeName, MountPath: land.Path, MountPropagation: &prop}} } patches = append(patches, PatchOp{Op: "add", Path: "/spec/initContainers/0", Value: init}) if id.Scheme == "hf" { diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go index 64f8d6c3b8..3c3b00fac5 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go @@ -124,6 +124,9 @@ func TestModelVolume_WriterBlock_NGCInit(t *testing.T) { } v := viewMV(pod, patches) uri := "ngc://org/team/nemotron3-ultra-genrm:bf16-fixed" + if v.annotations[modelvolume.DownloadInitAnnotation] != "download-ngc-model" { + t.Errorf("writer must name its download init for the agent, got %q", v.annotations[modelvolume.DownloadInitAnnotation]) + } if el.called != 1 || v.annotations[modelvolume.IdentityAnnotation] != uri || v.labels[modelvolume.RoleLabel] != "writer" { t.Errorf("writer stamp: elected=%d ann=%v labels=%v", el.called, v.annotations, v.labels) } @@ -161,8 +164,18 @@ func TestModelVolume_ReaderBlock_PendingBind(t *testing.T) { if v.labels[modelvolume.RoleLabel] != "reader" || v.labels[modelvolume.PendingLabel] != "true" || v.annotations[modelvolume.LandingAnnotation] != "/config/models" { t.Errorf("Block reader stamp: %v %v", v.labels, v.annotations) } - if _, replaced := v.volumes["ngc-models"]; replaced { - t.Error("Block reader keeps its emptyDir; the agent binds the volume over it") + vol, replaced := v.volumes["ngc-models"] + if !replaced || vol.HostPath == nil || vol.HostPath.Path != "/var/lib/containerd/nvsnap-overlays/models/"+modelvolume.Key("ngc://org/team/nemotron3-ultra-genrm:bf16-fixed") || *vol.HostPath.Type != corev1.HostPathDirectoryOrCreate { + t.Errorf("Block reader lands on a hostPath under the model host root for the agent to bind into, got %+v", vol) + } + var propagations int + for _, p := range patches { + if strings.HasSuffix(p.Path, "/mountPropagation") && p.Value == corev1.MountPropagationHostToContainer { + propagations++ + } + } + if propagations != 2 { + t.Errorf("engine and download init mounts must propagate host mounts in, got %d", propagations) } s := v.initScripts["download-ngc-model"] if !strings.Contains(s, "while [ ! -f /config/models/.nvsnap-complete ]") || !strings.Contains(s, "ngc registry model download-version") || !strings.Contains(s, "deadline passed") { diff --git a/src/compute-plane-services/nvsnap/internal/webhook/mutate.go b/src/compute-plane-services/nvsnap/internal/webhook/mutate.go index 654747474d..018904854e 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/mutate.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/mutate.go @@ -268,6 +268,10 @@ type Mutator struct { ModelVolume *modelvolume.Provisioner Groups modelid.GroupResolver ModelWaitDeadline time.Duration + // ModelHostRoot is the host directory (under the agent's Bidirectional + // overlays root) where Block-mode readers get their hostPath and the + // agent binds completed model volumes: /. + ModelHostRoot string // L2WaitImage is the nvsnap-l2-wait init-container image ref // (nvsnap#147). When non-empty, tryL2Mount prepends a From a9b06ae89ebf82c45f161ba5f60c6961aa515159 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 07:46:59 -0700 Subject: [PATCH 04/16] fix(nvsnap): injected download init uses hf, huggingface-cli is a stub in huggingface_hub 1.x Co-Authored-By: Balaji Ganesan --- .../nvsnap/internal/webhook/model_volume.go | 17 +++++++++++++---- .../internal/webhook/model_volume_test.go | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go index ae727334d0..13cfc974ca 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go @@ -272,10 +272,7 @@ func (m *Mutator) downloadStepPatches(pod *corev1.Pod, main *corev1.Container, l } download := "" if id.Scheme == "hf" { - download = "huggingface-cli download " + shellQuote(id.Ref) - if id.Revision != "" { - download += " --revision " + shellQuote(id.Revision) - } + download = hfDownloadCommand(id) } script := readerScript(download, marker, deadline) if writer { @@ -432,3 +429,15 @@ func shellQuote(s string) string { } return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" } + +// hfDownloadCommand fetches a Hugging Face repo into HF_HOME. huggingface_hub +// 1.x renamed the CLI to `hf` and made `huggingface-cli` a stub that only +// prints a deprecation notice (seen in vllm/vllm-openai:v0.20.0), so prefer +// `hf` and fall back for older images. +func hfDownloadCommand(id modelid.Identity) string { + args := shellQuote(id.Ref) + if id.Revision != "" { + args += " --revision " + shellQuote(id.Revision) + } + return fmt.Sprintf("if command -v hf >/dev/null 2>&1; then hf download %[1]s; else huggingface-cli download %[1]s; fi", args) +} diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go index 3c3b00fac5..e3204a6818 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go @@ -230,7 +230,7 @@ func TestModelVolume_EngineDownload_InjectedInit(t *testing.T) { } init := v.newInits[0] s := init.Args[0] - if !strings.Contains(s, "huggingface-cli download Qwen/Qwen2.5-32B-Instruct") || !strings.Contains(s, "touch /root/.cache/huggingface/.nvsnap-complete") { + if !strings.Contains(s, "hf download Qwen/Qwen2.5-32B-Instruct") || !strings.Contains(s, "huggingface-cli download Qwen/Qwen2.5-32B-Instruct") || !strings.Contains(s, "touch /root/.cache/huggingface/.nvsnap-complete") { t.Errorf("injected init script:\n%s", s) } if init.Image != pod.Spec.Containers[0].Image || len(init.VolumeMounts) != 1 || init.VolumeMounts[0].MountPath != "/root/.cache/huggingface" { From 87bff059582bedc72aed72f1c01a3c9373c67039 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 07:57:00 -0700 Subject: [PATCH 05/16] fix(nvsnap): agent may update claims to label the model volume complete Co-Authored-By: Balaji Ganesan --- .../nvsnap/deploy/helm/nvsnap/templates/agent-rbac.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-rbac.yaml b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-rbac.yaml index 94e25bf06c..c68e5ea6e1 100644 --- a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-rbac.yaml +++ b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-rbac.yaml @@ -65,8 +65,10 @@ rules: # nvsnap-system); we keep them ClusterRole for now so the agent can # operate in any namespace via flag. - apiGroups: [""] + # update/patch: the model volume controller labels the writer claim + # complete (internal/agent/modelvolume_controller.go). resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "create", "delete"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] # persistentvolumes (cluster-scoped): the shared-volume promoter # (NVMesh/EFS cachedir + ember path) Gets the writer's bound primary PV, # Updates its reclaim policy to Retain, and Creates a secondary From 5fc949eb96b79914f38bdf7cf648f31deb6e5be4 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 08:15:54 -0700 Subject: [PATCH 06/16] fix(nvsnap): run the model download as a Job so block volumes release The first cluster run of the model volume on NVMesh exposed a constraint the design had missed: a volume attached read-write by a running pod cannot be attached read-only anywhere else (AttachVolume failed on the read-only PV while the writer pod held the primary). The L2 promote never met it because it deletes the writer claim before readers attach. So the download step cannot live inside a pod that goes on to serve. The download is now a Job per identity per cluster, nvsnap-model-dl-, created idempotently by the webhook on first sight: the chart's own download init (image, command, env, pull secrets, tolerations) wrapped to touch the marker, or `hf download` on the engine image when the engine fetches the model itself. Its exit releases the volume. Every workload pod is a reader; there is no writer pod and no Lease election on this path, because Job create is atomic. The agent completes the identity when the Job succeeds and mints the read-only claim on block storage; readers are bound as before. The design doc records the constraint and the reason. Tests: Job spec derived from the NGC init and from a stock vLLM pod (hf download, forwarded token, claim mounted at the init's path), idempotent create, no Job once the volume is complete, controller completes on Job success only. Mutation-checked: Job never created, Job created when complete, and completion on a running Job each turn tests red. Co-Authored-By: Balaji Ganesan --- .../proposals/helm-shared-model-volume.md | 61 ++++++----- .../nvsnap/internal/agent/BUILD.bazel | 2 + .../internal/agent/modelvolume_controller.go | 80 +++++++------- .../agent/modelvolume_controller_test.go | 40 +++---- .../nvsnap/internal/modelvolume/BUILD.bazel | 1 + .../internal/modelvolume/modelvolume.go | 79 ++++++++++++++ .../internal/modelvolume/modelvolume_test.go | 37 +++++++ .../nvsnap/internal/webhook/model_volume.go | 103 +++++++++++------- .../internal/webhook/model_volume_test.go | 77 +++++++------ 9 files changed, 320 insertions(+), 160 deletions(-) diff --git a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md index f25b2d7763..1f5d0506e9 100644 --- a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md +++ b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md @@ -11,19 +11,19 @@ Status: design, 2026-09-26. Supersedes the gate-and-promote path of ```mermaid sequenceDiagram - participant P0 as pod-0 (writer) - participant P1 as pod-1..N (readers, any node, any namespace) + participant J as Job nvsnap-model-dl- + participant P as pods 0..N (readers, any node, any namespace) participant WH as webhook participant A as agent - participant V as model volume nvsnap-model- - WH->>WH: identity, landing path, group; elect writer (Lease) - WH-->>P0: init "download" -> V (rw), main mounts V ro - WH-->>P1: landing path -> V (DFS) or emptyDir + wait init (NVMesh) - Note over P0,P1: all pods schedule immediately - P0->>V: download; init exits 0 - A->>V: complete: marker (DFS) / ro attach + bind into P1 (NVMesh) - P1->>P1: wait init sees marker, engine starts - Note over P0,P1: engines start together; multi-node groups form as today + participant V as model volume nvsnap-model- + WH->>WH: identity, landing path, group + WH->>J: create (idempotent): the chart's download step, into V + WH-->>P: landing path -> V (DFS) or hostPath + wait init (NVMesh) + Note over P: all pods schedule immediately + J->>V: download; touch .nvsnap-complete; exit 0, volume released + A->>V: complete: label claim; NVMesh: ro PV + bind into P's hostPath + P->>P: wait init sees marker, engine starts + Note over P: engines start together; multi-node groups form as today ``` ## Two artifacts, two lifecycles @@ -55,14 +55,25 @@ not. volume there is already a PVC, hostPath or OCI image, skip the pod. Role flags and wiring env stay out of the hash (`stripRoleFlags`). -2. One download step per identity per cluster (webhook + Lease). If the - chart downloads in an init container, that init is the download step. - If the engine downloads itself, the webhook injects an init - (`huggingface-cli download ` into the landing path) and starts the - engine offline. The Lease `nvsnap-model-` elects the writer among - concurrent admissions; the writer's init runs the download, its main - container mounts the model read-only. Every other pod is a reader and - its download init is replaced by a wait. +2. One download step per identity per cluster, as a Job (webhook). The + webhook creates Job `nvsnap-model-dl-` in the pod's namespace on + first sight; create is idempotent, so concurrent admissions need no + election. The Job's pod is the chart's own download init (image, + command, env, secrets, pull secrets, tolerations copied from the + admitted pod) wrapped to touch `/.nvsnap-complete` on success; + when the engine downloads itself the Job runs `hf download ` + with the engine image and credentials. Every workload pod is a reader: + its download init becomes a wait for the marker, and an engine that + downloaded itself is started offline. + + Why a Job and not the first pod: on NVMesh a volume attached read-write + by a running pod cannot be attached read-only anywhere else (dev1, + 2026-09-26: `NVMesh Attach Failed` on the read-only PV while the writer + pod held the primary). The download step has to exit and release the + volume before readers attach, so it cannot live inside a pod that goes + on to serve. A Job also decouples the download from the workload's + scheduling: it runs on any node with the image, and the workload pods + of a multi-node group or a gang all schedule as plain readers. 3. Model volume per identity, immutable after download (agent + storage profile). Distributed filesystem: one RWX volume; writer and readers @@ -123,7 +134,7 @@ between them. Everything after that first start is a full hit. | Failure | Effect | Recovery | |---|---|---| -| writer dies before complete | readers' wait reaches the Lease deadline | readers download locally (NVMesh) or into the volume (DFS; per-file atomic); Lease expires; next admission elects a new writer | +| download Job fails or never completes | Job retries with backoff; readers' wait reaches the deadline | readers download locally (NVMesh) or into the volume (DFS; per-file atomic); the next admission recreates a missing Job | | volume full | writer's download fails, init restarts | same as above; retention by last use with a size budget is part of this design's follow-up, since a full volume fails every writer | | agent down on a reader node (NVMesh) | no bind arrives | wait deadline, local download | | writer pod restarts after complete | volume immutable, unaffected | none needed | @@ -132,14 +143,14 @@ between them. Everything after that first start is a full hit. ## What changes in the code -Stays: classifier (extended per mechanism 1), role-neutral hash, Lease -election (elects the writer), `EnsureClaim`, storage profiles, cache env +Stays: classifier (extended per mechanism 1), role-neutral hash, +`EnsureClaim`, storage profiles, cache env injection and seed init, mount-holder and bind injection (L1), the server reconciler, `vllm-workers` chart and runner. -New: identity from init containers and group inheritance; download-init -injection for engine-internal downloads; init wrapping for marker and -wait; per-identity model volume created at admission from the profile's +New: identity from init containers and group inheritance; the download +Job derived from the chart's init or from `hf download`; init wrapping +for the wait; per-identity model volume created at admission from the profile's class (RWX on DFS, RWO writer PVC on NVMesh); agent completion handler (init exit 0 -> marker / ro PV + bind); cache volume capture (caches only) on NVMesh; `cacheMode`; last-use labels. diff --git a/src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel b/src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel index 65ef021ebd..e5e986f70a 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel +++ b/src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel @@ -65,6 +65,7 @@ go_library( "@com_github_sirupsen_logrus//:logrus", "@com_github_vishvananda_netlink//:netlink", "@com_github_vishvananda_netns//:netns", + "@io_k8s_api//batch/v1:batch", "@io_k8s_api//coordination/v1:coordination", "@io_k8s_api//core/v1:core", "@io_k8s_apimachinery//pkg/api/errors", @@ -125,6 +126,7 @@ go_test( "//src/compute-plane-services/nvsnap/internal/objectstore", "@com_github_gorilla_mux//:mux", "@com_github_sirupsen_logrus//:logrus", + "@io_k8s_api//batch/v1:batch", "@io_k8s_api//core/v1:core", "@io_k8s_api//storage/v1:storage", "@io_k8s_apimachinery//pkg/api/resource", diff --git a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go index 2c594c93c3..01ae2f705a 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go +++ b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go @@ -15,6 +15,7 @@ import ( "time" "github.com/sirupsen/logrus" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -29,18 +30,20 @@ import ( // ModelVolumeController is the agent half of // docs/proposals/helm-shared-model-volume.md. // -// - Writers (any node): when the download init named on the pod exits -// 0, the writer claim is labelled complete and, on block storage, the -// read-only claim is minted in the writer's namespace. +// - Download Jobs (any node): when the Job for an identity succeeds, the +// claim is labelled complete and, on block storage, the read-only +// claim is minted in the Job's namespace. The Job's exit is what +// released the volume; NVMesh refuses a read-only attach while a +// running pod holds it read-write. // - Pending readers on this node (block storage): once the identity is // complete, the read-only claim is minted in the reader's namespace, // attached to this node through a mount-holder, bind-mounted read-only // onto the reader's hostPath landing (under the Bidirectional overlays -// root, so kubelet's mount sees it), and the pod is un-pended. The -// writer touched the marker inside the volume, so the reader's wait -// init sees it as soon as the bind lands. +// root, so kubelet's mount sees it), and the pod is un-pended. The Job +// touched the marker inside the volume, so the reader's wait init sees +// it as soon as the bind lands. // -// Every agent watches writers; marking and minting are idempotent, so the +// Every agent watches Jobs; marking and minting are idempotent, so the // race between agents is harmless. Only the agent on the reader's node // binds for it. type ModelVolumeController struct { @@ -73,16 +76,23 @@ func (c *ModelVolumeController) Run(ctx context.Context) error { c.init() factory := informers.NewSharedInformerFactoryWithOptions(c.Kube, 30*time.Second, informers.WithTweakListOptions(func(o *metav1.ListOptions) { o.LabelSelector = modelvolume.IdentityLabel })) - informer := factory.Core().V1().Pods().Informer() - if _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + pods := factory.Core().V1().Pods().Informer() + if _, err := pods.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj any) { c.handle(ctx, obj) }, UpdateFunc: func(_, obj any) { c.handle(ctx, obj) }, }); err != nil { - return fmt.Errorf("AddEventHandler: %w", err) + return fmt.Errorf("AddEventHandler pods: %w", err) + } + jobs := factory.Batch().V1().Jobs().Informer() + if _, err := jobs.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { c.handleJob(ctx, obj) }, + UpdateFunc: func(_, obj any) { c.handleJob(ctx, obj) }, + }); err != nil { + return fmt.Errorf("AddEventHandler jobs: %w", err) } factory.Start(ctx.Done()) - if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) { - return fmt.Errorf("model volume informer did not sync") + if !cache.WaitForCacheSync(ctx.Done(), pods.HasSynced, jobs.HasSynced) { + return fmt.Errorf("model volume informers did not sync") } c.log().WithFields(logrus.Fields{"node": c.NodeName, "mode": c.Provisioner.Cfg.Mode, "host_root": c.HostRoot}).Info("model volume controller started") <-ctx.Done() @@ -111,7 +121,7 @@ func (c *ModelVolumeController) log() logrus.FieldLogger { return logrus.NewEntry(logrus.New()).WithField("subsys", "modelvolume") } -// handle dispatches one pod event; exported for tests as Handle. +// handle dispatches one pod event: pending readers on this node. func (c *ModelVolumeController) handle(ctx context.Context, obj any) { pod, ok := obj.(*corev1.Pod) if !ok || pod == nil { @@ -119,34 +129,40 @@ func (c *ModelVolumeController) handle(ctx context.Context, obj any) { } c.init() uri := pod.Annotations[modelvolume.IdentityAnnotation] - if uri == "" { + if uri == "" || pod.Labels[modelvolume.RoleLabel] != "reader" { return } - switch pod.Labels[modelvolume.RoleLabel] { - case "writer": - c.handleWriter(ctx, pod, uri) - case "reader": - if pod.Labels[modelvolume.PendingLabel] == "true" && pod.Spec.NodeName == c.NodeName { - c.handlePendingReader(ctx, pod, uri) - } + if pod.Labels[modelvolume.PendingLabel] == "true" && pod.Spec.NodeName == c.NodeName { + c.handlePendingReader(ctx, pod, uri) } } // Handle is the test entry point for one pod event. func (c *ModelVolumeController) Handle(ctx context.Context, pod *corev1.Pod) { c.handle(ctx, pod) } -func (c *ModelVolumeController) handleWriter(ctx context.Context, pod *corev1.Pod, uri string) { - initName := pod.Annotations[modelvolume.DownloadInitAnnotation] - if initName == "" || !initExitedZero(pod, initName) { +// HandleJob is the test entry point for one Job event. +func (c *ModelVolumeController) HandleJob(ctx context.Context, job *batchv1.Job) { + c.handleJob(ctx, job) +} + +// handleJob completes the identity when its download Job succeeded. +func (c *ModelVolumeController) handleJob(ctx context.Context, obj any) { + job, ok := obj.(*batchv1.Job) + if !ok || job == nil { + return + } + c.init() + uri := job.Annotations[modelvolume.IdentityAnnotation] + if uri == "" || job.Status.Succeeded == 0 { return } - log := c.log().WithFields(logrus.Fields{"pod": pod.Namespace + "/" + pod.Name, "model": uri}) - if err := c.Provisioner.MarkComplete(ctx, uri, pod.Namespace); err != nil { + log := c.log().WithFields(logrus.Fields{"job": job.Namespace + "/" + job.Name, "model": uri}) + if err := c.Provisioner.MarkComplete(ctx, uri, job.Namespace); err != nil { log.WithError(err).Warn("model volume: mark complete failed") return } if c.Provisioner.Cfg.Mode == modelvolume.ModeBlock && c.Minter != nil { - if err := c.Minter.MintReadOnly(ctx, pod.Namespace, modelvolume.ClaimName(uri), modelvolume.ReadOnlyPVName(uri, pod.Namespace), modelvolume.ReadOnlyClaimName(uri), pod.Namespace, modelvolume.Key(uri)); err != nil { + if err := c.Minter.MintReadOnly(ctx, job.Namespace, modelvolume.ClaimName(uri), modelvolume.ReadOnlyPVName(uri, job.Namespace), modelvolume.ReadOnlyClaimName(uri), job.Namespace, modelvolume.Key(uri)); err != nil { log.WithError(err).Warn("model volume: mint read-only claim failed") return } @@ -154,16 +170,6 @@ func (c *ModelVolumeController) handleWriter(ctx context.Context, pod *corev1.Po log.Info("model volume: download complete; readers may attach") } -func initExitedZero(pod *corev1.Pod, name string) bool { - for i := range pod.Status.InitContainerStatuses { - s := &pod.Status.InitContainerStatuses[i] - if s.Name == name { - return s.State.Terminated != nil && s.State.Terminated.ExitCode == 0 - } - } - return false -} - func (c *ModelVolumeController) handlePendingReader(ctx context.Context, pod *corev1.Pod, uri string) { log := c.log().WithFields(logrus.Fields{"pod": pod.Namespace + "/" + pod.Name, "model": uri, "node": c.NodeName}) st, err := c.Provisioner.Lookup(ctx, uri) diff --git a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go index 16a2e0ad3a..ff4a56ea4c 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go +++ b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/sirupsen/logrus" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -61,17 +62,13 @@ func mvController(t *testing.T, kc *fake.Clientset, p *modelvolume.Provisioner, return c, &att, &bnd } -func writerPod(exit *int32) *corev1.Pod { - p := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "w-0", Namespace: "sr-fn", - Labels: map[string]string{modelvolume.IdentityLabel: modelvolume.Key(mvURI), modelvolume.RoleLabel: "writer"}, - Annotations: map[string]string{modelvolume.IdentityAnnotation: mvURI, modelvolume.DownloadInitAnnotation: "download-ngc-model"}}, - Spec: corev1.PodSpec{NodeName: "node-a"}, +func downloadJob(succeeded int32) *batchv1.Job { + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: modelvolume.JobName(mvURI), Namespace: "sr-fn", + Labels: map[string]string{modelvolume.IdentityLabel: modelvolume.Key(mvURI)}, + Annotations: map[string]string{modelvolume.IdentityAnnotation: mvURI}}, + Status: batchv1.JobStatus{Succeeded: succeeded}, } - if exit != nil { - p.Status.InitContainerStatuses = []corev1.ContainerStatus{{Name: "download-ngc-model", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: *exit}}}} - } - return p } func readerPod(ns, node string) *corev1.Pod { @@ -83,25 +80,19 @@ func readerPod(ns, node string) *corev1.Pod { } } -func TestModelVolumeController_WriterCompletionMintsReadOnly(t *testing.T) { +func TestModelVolumeController_JobCompletionMintsReadOnly(t *testing.T) { kc, p := writerFixture(t) c, _, _ := mvController(t, kc, p, "node-a") ctx := context.Background() - c.Handle(ctx, writerPod(nil)) // init still running - if st, _ := p.Lookup(ctx, mvURI); st.Complete { - t.Fatal("no completion before the download init exits") - } - one := int32(1) - c.Handle(ctx, writerPod(&one)) // init failed + c.HandleJob(ctx, downloadJob(0)) // still running if st, _ := p.Lookup(ctx, mvURI); st.Complete { - t.Fatal("a failed download must not complete the volume") + t.Fatal("no completion before the Job succeeds") } - zero := int32(0) - c.Handle(ctx, writerPod(&zero)) + c.HandleJob(ctx, downloadJob(1)) st, _ := p.Lookup(ctx, mvURI) if !st.Complete { - t.Fatal("exit 0 must mark the claim complete") + t.Fatal("a succeeded Job must mark the claim complete") } ro, err := kc.CoreV1().PersistentVolumeClaims("sr-fn").Get(ctx, modelvolume.ReadOnlyClaimName(mvURI), metav1.GetOptions{}) if err != nil { @@ -119,9 +110,9 @@ func TestModelVolumeController_WriterCompletionMintsReadOnly(t *testing.T) { t.Error("the writer's PV must be retained; it is the artifact") } if _, err := kc.CoreV1().PersistentVolumeClaims("sr-fn").Get(ctx, modelvolume.ClaimName(mvURI), metav1.GetOptions{}); err != nil { - t.Error("the writer claim stays: the writer is still running on it") + t.Error("the download claim stays: it is the artifact") } - c.Handle(ctx, writerPod(&zero)) // idempotent + c.HandleJob(ctx, downloadJob(1)) // idempotent } func TestModelVolumeController_PendingReaderBoundOnItsNode(t *testing.T) { @@ -137,9 +128,8 @@ func TestModelVolumeController_PendingReaderBoundOnItsNode(t *testing.T) { if len(*attached) != 0 || len(*bound) != 0 { t.Fatal("nothing may be attached before the download completes") } - zero := int32(0) cw, _, _ := mvController(t, kc, p, "node-a") - cw.Handle(ctx, writerPod(&zero)) + cw.HandleJob(ctx, downloadJob(1)) // Readers on other nodes are not this agent's business, even before // anything is bound here. diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/BUILD.bazel b/src/compute-plane-services/nvsnap/internal/modelvolume/BUILD.bazel index d17f0bdd07..bae44a5f7c 100644 --- a/src/compute-plane-services/nvsnap/internal/modelvolume/BUILD.bazel +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/BUILD.bazel @@ -6,6 +6,7 @@ go_library( importpath = "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/modelvolume", visibility = ["//src/compute-plane-services/nvsnap:__subpackages__"], deps = [ + "@io_k8s_api//batch/v1:batch", "@io_k8s_api//core/v1:core", "@io_k8s_apimachinery//pkg/api/errors", "@io_k8s_apimachinery//pkg/api/resource", diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go index 7d76c3666e..b5e4030c5e 100644 --- a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go @@ -38,6 +38,7 @@ import ( "encoding/hex" "fmt" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -204,3 +205,81 @@ func (p *Provisioner) MarkComplete(ctx context.Context, uri, ns string) error { } return nil } + +// JobName is the download Job for a model URI. +func JobName(uri string) string { return "nvsnap-model-dl-" + Key(uri) } + +// DownloadStep is the container that fetches the model into the claim, +// derived by the webhook from the chart's own download init (or from the +// engine image plus `hf download`), already wrapped to touch the marker. +type DownloadStep struct { + Container corev1.Container + ImagePullSecrets []corev1.LocalObjectReference + Tolerations []corev1.Toleration + NodeSelector map[string]string + // VolumeName is the name the container mounts the claim under. + VolumeName string +} + +// EnsureDownloadJob creates the one download Job for uri in ns, writing +// into claim. Create is atomic, so N concurrent admissions produce one +// Job and need no election. On NVMesh the Job's exit is what releases the +// volume for read-only attaches elsewhere; a download inside a serving +// pod would hold it forever. +func (p *Provisioner) EnsureDownloadJob(ctx context.Context, uri, ns, claim string, step DownloadStep) (string, error) { + name := JobName(uri) + if _, err := p.Kube.BatchV1().Jobs(ns).Get(ctx, name, metav1.GetOptions{}); err == nil { + return name, nil + } else if !apierrors.IsNotFound(err) { + return "", fmt.Errorf("get job %s/%s: %w", ns, name, err) + } + backoff := int32(6) + labels := map[string]string{"app.kubernetes.io/managed-by": managedBy, IdentityLabel: Key(uri)} + c := step.Container + c.Name = "download" + c.VolumeMounts = []corev1.VolumeMount{{Name: step.VolumeName, MountPath: mountPathOf(step)}} + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels, Annotations: map[string]string{IdentityAnnotation: uri}}, + Spec: batchv1.JobSpec{ + BackoffLimit: &backoff, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + AutomountServiceAccountToken: new(bool), + ImagePullSecrets: step.ImagePullSecrets, + Tolerations: step.Tolerations, + NodeSelector: step.NodeSelector, + Containers: []corev1.Container{c}, + Volumes: []corev1.Volume{{Name: step.VolumeName, VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: claim}}}}, + }, + }, + }, + } + if _, err := p.Kube.BatchV1().Jobs(ns).Create(ctx, job, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { + return "", fmt.Errorf("create job %s/%s: %w", ns, name, err) + } + return name, nil +} + +func mountPathOf(step DownloadStep) string { + for _, vm := range step.Container.VolumeMounts { + if vm.Name == step.VolumeName { + return vm.MountPath + } + } + return "/models" +} + +// JobSucceeded reports whether the download Job for uri in ns finished. +func (p *Provisioner) JobSucceeded(ctx context.Context, uri, ns string) (bool, error) { + job, err := p.Kube.BatchV1().Jobs(ns).Get(ctx, JobName(uri), metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return job.Status.Succeeded > 0, nil +} diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go index 7d64b3787d..97f96192ca 100644 --- a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go @@ -70,3 +70,40 @@ func TestProvisioner_LookupAndComplete(t *testing.T) { t.Errorf("names: %s %s", ClaimName(uri), ReadOnlyClaimName(uri)) } } + +func TestProvisioner_DownloadJobIdempotent(t *testing.T) { + ctx := context.Background() + kc := fake.NewSimpleClientset() + p := &Provisioner{Kube: kc, Cfg: Config{Mode: ModeBlock, StorageClass: "sc", Size: resource.MustParse("1Gi")}} + step := DownloadStep{ + Container: corev1.Container{Image: "vllm/vllm-openai", Command: []string{"/bin/sh", "-c"}, Args: []string{"hf download x && touch /m/.nvsnap-complete"}, VolumeMounts: []corev1.VolumeMount{{Name: "models", MountPath: "/m"}}}, + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "pull"}}, + Tolerations: []corev1.Toleration{{Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists}}, + VolumeName: "models", + } + name, err := p.EnsureDownloadJob(ctx, uri, "fn", ClaimName(uri), step) + if err != nil || name != JobName(uri) { + t.Fatalf("%v %q", err, name) + } + job, err := kc.BatchV1().Jobs("fn").Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + ps := job.Spec.Template.Spec + if ps.RestartPolicy != corev1.RestartPolicyOnFailure || ps.ImagePullSecrets[0].Name != "pull" || len(ps.Tolerations) != 1 || ps.Volumes[0].PersistentVolumeClaim.ClaimName != ClaimName(uri) || ps.Containers[0].VolumeMounts[0].MountPath != "/m" || job.Annotations[IdentityAnnotation] != uri { + t.Errorf("job spec: %+v", ps) + } + if _, err := p.EnsureDownloadJob(ctx, uri, "fn", ClaimName(uri), step); err != nil { + t.Errorf("second EnsureDownloadJob must be a no-op: %v", err) + } + if ok, _ := p.JobSucceeded(ctx, uri, "fn"); ok { + t.Error("job has not succeeded yet") + } + job.Status.Succeeded = 1 + if _, err := kc.BatchV1().Jobs("fn").UpdateStatus(ctx, job, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if ok, _ := p.JobSucceeded(ctx, uri, "fn"); !ok { + t.Error("succeeded job must report true") + } +} diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go index 13cfc974ca..865acd18c7 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go @@ -5,14 +5,13 @@ package webhook import ( "context" - "crypto/sha256" - "encoding/hex" "fmt" "path" "strings" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/checkpointstore" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/election" @@ -45,7 +44,7 @@ const ( // pod is not a downloader or the feature is off, and Mutate continues // with the older paths. func (m *Mutator) modelVolumePatches(ctx context.Context, pod *corev1.Pod) ([]PatchOp, error) { - if m.ModelVolume == nil || m.Elector == nil { + if m.ModelVolume == nil { return nil, nil } if gpuRequest(pod) == 0 { @@ -74,62 +73,92 @@ func (m *Mutator) modelVolumePatches(ctx context.Context, pod *corev1.Pod) ([]Pa if err != nil { return nil, err } - role := election.RoleFollower if !st.Complete { - r, _, err := m.Elector.Elect(ctx, leaseHash(uri), pod) + // The download step is a Job, created once per identity; create is + // atomic so concurrent admissions converge without an election. + // The claim lives where the Job runs, the pod's namespace. + claim, err := m.ModelVolume.EnsureWriterClaim(ctx, uri, pod.Namespace) if err != nil { return nil, err } - role = r - } - switch { - case role == election.RoleLeader: - claim, err := m.ModelVolume.EnsureWriterClaim(ctx, uri, pod.Namespace) + step, ok := m.downloadStep(pod, main, land, res.Identity) + if !ok { + log.Info("model volume: no download step can be derived (engine downloads a non-HF model); leaving pod alone") + return nil, nil + } + job, err := m.ModelVolume.EnsureDownloadJob(ctx, uri, pod.Namespace, claim, step) if err != nil { return nil, err } - patches = append(patches, mp.label(modelvolume.RoleLabel, "writer")...) - if land.Downloader == modelid.DownloaderInit { - patches = append(patches, mp.annotation(modelvolume.DownloadInitAnnotation, land.InitContainer)...) - } else if res.Identity.Scheme == "hf" { - patches = append(patches, mp.annotation(modelvolume.DownloadInitAnnotation, injectedDownloadInit)...) - } - patches = append(patches, m.substituteLandingVolume(pod, main, land, claim)...) - patches = append(patches, m.downloadStepPatches(pod, main, land, res.Identity, true)...) - patches = append(patches, m.modelCacheEnvPatches(pod, main, land, uri)...) - log.WithField("claim", claim).Info("model volume: writer; download lands in the shared volume") - case m.ModelVolume.Cfg.Mode == modelvolume.ModeRWX: - // Readers share the claim in the writer's namespace when it is - // ours, else get one minted in theirs (EnsureClaim, cross - // namespace, is the same volume on a distributed filesystem). + log.WithFields(logrus.Fields{"claim": claim, "job": job}).Info("model volume: download job ensured") + } + patches = append(patches, mp.label(modelvolume.RoleLabel, "reader")...) + switch m.ModelVolume.Cfg.Mode { + case modelvolume.ModeRWX: claim, err := m.ModelVolume.EnsureWriterClaim(ctx, uri, pod.Namespace) if err != nil { return nil, err } - patches = append(patches, mp.label(modelvolume.RoleLabel, "reader")...) patches = append(patches, m.substituteLandingVolume(pod, main, land, claim)...) - patches = append(patches, m.downloadStepPatches(pod, main, land, res.Identity, false)...) - patches = append(patches, m.modelCacheEnvPatches(pod, main, land, uri)...) log.WithFields(logrus.Fields{"claim": claim, "complete": st.Complete}).Info("model volume: reader on shared filesystem; waits for the marker") default: - // Block mode reader: the emptyDir stays; the agent binds the - // completed read-only volume over it and drops the marker. - patches = append(patches, mp.label(modelvolume.RoleLabel, "reader")...) + // Block mode: hostPath landing; the agent binds the completed + // read-only volume over it and the marker inside appears. patches = append(patches, mp.label(modelvolume.PendingLabel, "true")...) patches = append(patches, mp.annotation(modelvolume.LandingAnnotation, landingMount(land))...) patches = append(patches, m.hostPathLanding(pod, main, land, uri)...) - patches = append(patches, m.downloadStepPatches(pod, main, land, res.Identity, false)...) - patches = append(patches, m.modelCacheEnvPatches(pod, main, land, uri)...) log.WithField("complete", st.Complete).Info("model volume: reader on block storage; agent binds the volume after completion") } + patches = append(patches, m.downloadStepPatches(pod, main, land, res.Identity, false)...) + patches = append(patches, m.modelCacheEnvPatches(pod, main, land, uri)...) return patches, nil } -// leaseHash keys the writer election by model identity, in the 64-hex form -// the Lease naming expects. -func leaseHash(uri string) string { - sum := sha256.Sum256([]byte("model:" + uri)) - return hex.EncodeToString(sum[:]) +// downloadStep derives the Job's container from the pod: the chart's own +// download init wrapped to touch the marker, or `hf download` on the +// engine image when the engine fetches the model itself. +func (m *Mutator) downloadStep(pod *corev1.Pod, main *corev1.Container, land modelid.Landing, id modelid.Identity) (modelvolume.DownloadStep, bool) { + step := modelvolume.DownloadStep{ImagePullSecrets: pod.Spec.ImagePullSecrets, Tolerations: pod.Spec.Tolerations, NodeSelector: pod.Spec.NodeSelector} + if land.Downloader == modelid.DownloaderInit { + for i := range pod.Spec.InitContainers { + init := pod.Spec.InitContainers[i] + if init.Name != land.InitContainer { + continue + } + mount := initMountFor(&init, land) + orig := shellJoin(append(append([]string{}, init.Command...), init.Args...)) + c := *init.DeepCopy() + c.Command = []string{"/bin/sh", "-c"} + c.Args = []string{writerScript(orig, path.Join(mount, modelvolume.MarkerFile))} + c.VolumeMounts = []corev1.VolumeMount{{Name: landVolumeName(land), MountPath: mount}} + step.Container = c + step.VolumeName = landVolumeName(land) + return step, true + } + return step, false + } + if id.Scheme != "hf" { + return step, false + } + step.VolumeName = landVolumeName(land) + step.Container = corev1.Container{ + Image: main.Image, + Command: []string{"/bin/sh", "-c"}, + Args: []string{writerScript(hfDownloadCommand(id), path.Join(land.Path, modelvolume.MarkerFile))}, + Env: append([]corev1.EnvVar{{Name: "HF_HOME", Value: land.Path}}, tokenEnv(main)...), + VolumeMounts: []corev1.VolumeMount{{Name: step.VolumeName, MountPath: land.Path}}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("4Gi")}, + }, + } + return step, true +} + +func landVolumeName(land modelid.Landing) string { + if land.VolumeName != "" { + return land.VolumeName + } + return modelVolumeName } func gpuRequest(pod *corev1.Pod) int64 { diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go index e3204a6818..124c643634 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go @@ -114,7 +114,7 @@ func atoi(s string) int { return n } -func TestModelVolume_WriterBlock_NGCInit(t *testing.T) { +func TestModelVolume_FirstPodBlock_NGCInit_CreatesJob(t *testing.T) { kc := fake.NewSimpleClientset() m, el := mvMutator(t, modelvolume.ModeBlock, election.RoleLeader, kc) pod := ngcFunctionPod() @@ -124,32 +124,35 @@ func TestModelVolume_WriterBlock_NGCInit(t *testing.T) { } v := viewMV(pod, patches) uri := "ngc://org/team/nemotron3-ultra-genrm:bf16-fixed" - if v.annotations[modelvolume.DownloadInitAnnotation] != "download-ngc-model" { - t.Errorf("writer must name its download init for the agent, got %q", v.annotations[modelvolume.DownloadInitAnnotation]) + if el.called != 0 { + t.Error("the download step is a Job; no election runs") + } + if v.annotations[modelvolume.IdentityAnnotation] != uri || v.labels[modelvolume.RoleLabel] != "reader" || v.labels[modelvolume.PendingLabel] != "true" { + t.Errorf("every pod is a reader: ann=%v labels=%v", v.annotations, v.labels) } - if el.called != 1 || v.annotations[modelvolume.IdentityAnnotation] != uri || v.labels[modelvolume.RoleLabel] != "writer" { - t.Errorf("writer stamp: elected=%d ann=%v labels=%v", el.called, v.annotations, v.labels) + pvc, err := kc.CoreV1().PersistentVolumeClaims("sr-fn").Get(context.Background(), modelvolume.ClaimName(uri), metav1.GetOptions{}) + if err != nil || pvc.Spec.AccessModes[0] != corev1.ReadWriteOnce { + t.Errorf("download claim must exist RWO in the pod namespace: %v", err) } - vol, ok := v.volumes["ngc-models"] - if !ok || vol.PersistentVolumeClaim == nil || vol.PersistentVolumeClaim.ClaimName != modelvolume.ClaimName(uri) { - t.Errorf("landing emptyDir must be replaced by the writer claim, got %+v", vol) + job, err := kc.BatchV1().Jobs("sr-fn").Get(context.Background(), modelvolume.JobName(uri), metav1.GetOptions{}) + if err != nil { + t.Fatalf("download Job must be created: %v", err) } - if pvc, err := kc.CoreV1().PersistentVolumeClaims("sr-fn").Get(context.Background(), modelvolume.ClaimName(uri), metav1.GetOptions{}); err != nil || pvc.Spec.AccessModes[0] != corev1.ReadWriteOnce { - t.Errorf("writer claim must exist RWO in the pod namespace: %v", err) + jc := job.Spec.Template.Spec.Containers[0] + script := jc.Args[0] + if jc.Image != "nvcr.io/org/ultra:vllm" || !strings.Contains(script, "ngc registry model download-version") || !strings.Contains(script, "touch /config/models/.nvsnap-complete") { + t.Errorf("job must run the chart's own download then touch the marker:\n%s", script) } - if len(v.roMounts) != 1 || !strings.Contains(v.roMounts[0], "/spec/containers/0/volumeMounts/1/") { - t.Errorf("engine's model mount must become read-only, got %v", v.roMounts) + if len(jc.Env) != 2 || jc.VolumeMounts[0].MountPath != "/config/models" || job.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName != modelvolume.ClaimName(uri) { + t.Errorf("job must carry the init's env and mount the claim at the init's path: env=%v mounts=%v", jc.Env, jc.VolumeMounts) } s := v.initScripts["download-ngc-model"] - if !strings.Contains(s, "ngc registry model download-version") || !strings.Contains(s, "touch /config/models/.nvsnap-complete") || !strings.Contains(s, "already complete") { - t.Errorf("writer init must run the original download then touch the marker:\n%s", s) + if !strings.Contains(s, "while [ ! -f /config/models/.nvsnap-complete ]") { + t.Errorf("the pod's own init becomes a wait:\n%s", s) } - if v.env["TORCHINDUCTOR_CACHE_DIR"] != "/opt/nvsnap/cache/torchinductor" || v.env["HF_HOME"] != "" || v.env["NIM_CACHE_PATH"] != "" { + if v.env["TORCHINDUCTOR_CACHE_DIR"] != "/opt/nvsnap/cache/torchinductor" || v.env["HF_HOME"] != "" { t.Errorf("Block mode: compile caches to the local cachedir, model env untouched: %v", v.env) } - if _, ok := v.volumes[cacheDirVolumeName]; !ok { - t.Error("Block mode must add the local cachedir emptyDir for the compile caches") - } } func TestModelVolume_ReaderBlock_PendingBind(t *testing.T) { @@ -186,9 +189,6 @@ func TestModelVolume_ReaderBlock_PendingBind(t *testing.T) { t.Fatal("no pod is ever gated on the model volume path") } } - if pvcs, _ := kc.CoreV1().PersistentVolumeClaims("").List(context.Background(), metav1.ListOptions{}); len(pvcs.Items) != 0 { - t.Error("a Block reader must not create claims") - } } func TestModelVolume_ReaderRWX_SharesClaim(t *testing.T) { @@ -216,7 +216,7 @@ func TestModelVolume_ReaderRWX_SharesClaim(t *testing.T) { } } -func TestModelVolume_EngineDownload_InjectedInit(t *testing.T) { +func TestModelVolume_EngineDownload_JobRunsHF(t *testing.T) { kc := fake.NewSimpleClientset() m, _ := mvMutator(t, modelvolume.ModeRWX, election.RoleLeader, kc) pod := stockVLLMPod() @@ -225,36 +225,38 @@ func TestModelVolume_EngineDownload_InjectedInit(t *testing.T) { t.Fatal(err) } v := viewMV(pod, patches) - if len(v.newInits) != 1 || v.newInits[0].Name != "nvsnap-model-download" { - t.Fatalf("engine-download writer needs an injected download init, got %v", v.newInits) - } - init := v.newInits[0] - s := init.Args[0] - if !strings.Contains(s, "hf download Qwen/Qwen2.5-32B-Instruct") || !strings.Contains(s, "huggingface-cli download Qwen/Qwen2.5-32B-Instruct") || !strings.Contains(s, "touch /root/.cache/huggingface/.nvsnap-complete") { - t.Errorf("injected init script:\n%s", s) + uri := "hf://Qwen/Qwen2.5-32B-Instruct" + job, err := kc.BatchV1().Jobs("fn").Get(context.Background(), modelvolume.JobName(uri), metav1.GetOptions{}) + if err != nil { + t.Fatalf("engine-download needs a download Job: %v", err) } - if init.Image != pod.Spec.Containers[0].Image || len(init.VolumeMounts) != 1 || init.VolumeMounts[0].MountPath != "/root/.cache/huggingface" { - t.Errorf("init must reuse the engine image and mount the model volume at HF_HOME: %+v", init) + jc := job.Spec.Template.Spec.Containers[0] + s := jc.Args[0] + if jc.Image != pod.Spec.Containers[0].Image || !strings.Contains(s, "hf download Qwen/Qwen2.5-32B-Instruct") || !strings.Contains(s, "touch /root/.cache/huggingface/.nvsnap-complete") || jc.VolumeMounts[0].MountPath != "/root/.cache/huggingface" { + t.Errorf("job must run hf download on the engine image into HF_HOME: image=%s mounts=%v\n%s", jc.Image, jc.VolumeMounts, s) } var sawToken bool - for _, e := range init.Env { + for _, e := range jc.Env { if e.Name == "HF_TOKEN" && e.ValueFrom != nil { sawToken = true } } if !sawToken { - t.Error("registry credentials must be forwarded to the download init") + t.Error("registry credentials must be forwarded to the download Job") + } + if len(v.newInits) != 1 || v.newInits[0].Name != "nvsnap-model-download" || !strings.Contains(v.newInits[0].Args[0], "while [ ! -f") { + t.Errorf("the pod gets a wait init, got %v", v.newInits) } if v.env["HF_HUB_OFFLINE"] != "1" { t.Error("engine must start offline and read the volume") } vol, ok := v.volumes[modelVolumeName] if !ok || vol.PersistentVolumeClaim == nil { - t.Errorf("rootfs landing gets a new claim volume: %+v", vol) + t.Errorf("RWX mode: rootfs landing gets the claim volume: %+v", vol) } } -func TestModelVolume_CompleteSkipsElection(t *testing.T) { +func TestModelVolume_CompleteCreatesNoJob(t *testing.T) { kc := fake.NewSimpleClientset() m, el := mvMutator(t, modelvolume.ModeRWX, election.RoleLeader, kc) uri := "hf://Qwen/Qwen2.5-32B-Instruct" @@ -270,7 +272,10 @@ func TestModelVolume_CompleteSkipsElection(t *testing.T) { } v := viewMV(stockVLLMPod(), patches) if el.called != 0 || v.labels[modelvolume.RoleLabel] != "reader" { - t.Errorf("complete volume: no election, reader role; elected=%d labels=%v", el.called, v.labels) + t.Errorf("complete volume: reader role; labels=%v", v.labels) + } + if _, err := kc.BatchV1().Jobs("fn").Get(context.Background(), modelvolume.JobName(uri), metav1.GetOptions{}); err == nil { + t.Error("a complete volume needs no download Job") } } From c736635aa32fadd00d7a48558c400315cc248b39 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 08:38:52 -0700 Subject: [PATCH 07/16] fix(nvsnap): remove the download Job pod after success so the volume detaches A Succeeded pod keeps its volumes attached. On NVMesh that let the read-only attach succeed only on the Job's own node (dev1 2026-09-26); every other node failed until the pod was deleted. The Job now carries ttlSecondsAfterFinished=30; completion state lives on the claim label. MarkComplete tolerates the update race between agents by re-reading. Co-Authored-By: Balaji Ganesan --- .../docs/proposals/helm-shared-model-volume.md | 5 ++++- .../nvsnap/internal/modelvolume/modelvolume.go | 15 ++++++++++++++- .../internal/modelvolume/modelvolume_test.go | 3 +++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md index 1f5d0506e9..85ba51acf4 100644 --- a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md +++ b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md @@ -71,7 +71,10 @@ not. 2026-09-26: `NVMesh Attach Failed` on the read-only PV while the writer pod held the primary). The download step has to exit and release the volume before readers attach, so it cannot live inside a pod that goes - on to serve. A Job also decouples the download from the workload's + on to serve. The Job's pod must also be removed after success + (`ttlSecondsAfterFinished`): a Succeeded pod keeps its volumes attached, + and on dev1 the read-only attach worked on the Job's node but failed on + every other node until that pod was deleted. A Job also decouples the download from the workload's scheduling: it runs on any node with the image, and the workload pods of a multi-node group or a gang all schedule as plain readers. diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go index b5e4030c5e..25324b885d 100644 --- a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go @@ -201,6 +201,13 @@ func (p *Provisioner) MarkComplete(ctx context.Context, uri, ns string) error { } pvc.Labels[CompleteLabel] = "true" if _, err := p.Kube.CoreV1().PersistentVolumeClaims(ns).Update(ctx, pvc, metav1.UpdateOptions{}); err != nil { + if apierrors.IsConflict(err) { + // Every agent marks completion; whoever lost the race re-reads. + again, gerr := p.Kube.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name, metav1.GetOptions{}) + if gerr == nil && again.Labels[CompleteLabel] == "true" { + return nil + } + } return fmt.Errorf("label claim %s/%s complete: %w", ns, name, err) } return nil @@ -234,6 +241,11 @@ func (p *Provisioner) EnsureDownloadJob(ctx context.Context, uri, ns, claim stri return "", fmt.Errorf("get job %s/%s: %w", ns, name, err) } backoff := int32(6) + // A Succeeded pod keeps its volumes attached; on NVMesh that blocks the + // read-only attach on every other node (dev1 2026-09-26). The Job and + // its pod go away shortly after success; completion state lives on the + // claim label, not on the Job. + ttl := int32(30) labels := map[string]string{"app.kubernetes.io/managed-by": managedBy, IdentityLabel: Key(uri)} c := step.Container c.Name = "download" @@ -241,7 +253,8 @@ func (p *Provisioner) EnsureDownloadJob(ctx context.Context, uri, ns, claim stri job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels, Annotations: map[string]string{IdentityAnnotation: uri}}, Spec: batchv1.JobSpec{ - BackoffLimit: &backoff, + BackoffLimit: &backoff, + TTLSecondsAfterFinished: &ttl, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{ diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go index 97f96192ca..82399b46b6 100644 --- a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go @@ -89,6 +89,9 @@ func TestProvisioner_DownloadJobIdempotent(t *testing.T) { if err != nil { t.Fatal(err) } + if job.Spec.TTLSecondsAfterFinished == nil || *job.Spec.TTLSecondsAfterFinished > 60 { + t.Error("the Job must remove its pod soon after success; a lingering Succeeded pod keeps the volume attached") + } ps := job.Spec.Template.Spec if ps.RestartPolicy != corev1.RestartPolicyOnFailure || ps.ImagePullSecrets[0].Name != "pull" || len(ps.Tolerations) != 1 || ps.Volumes[0].PersistentVolumeClaim.ClaimName != ClaimName(uri) || ps.Containers[0].VolumeMounts[0].MountPath != "/m" || job.Annotations[IdentityAnnotation] != uri { t.Errorf("job spec: %+v", ps) From aff9894a63843b46132dbf365db0db661e35f768 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 08:49:14 -0700 Subject: [PATCH 08/16] fix(nvsnap): release the download claim on completion and bind under a dedicated host root Two findings from the second dev1 run of the model volume on NVMesh. The retained volume must have no read-write attachment before any node can attach it read-only: with the Job's claim still bound, the read-only attach succeeded only on the Job's own node and failed everywhere else. On block storage MarkComplete now labels the retained PV with identity and completion, sets Retain, and deletes the download claim, the same release the L2 promote performs; Lookup reads completion from the PV; readers' read-only claims are minted from that PV, and only after no VolumeAttachment references it (Detached). RWX mode keeps the shared claim and labels it as before. The bind root moves out of the overlays root: the overlay sweeper removes entries it does not own and deleted the model binds under it. Completed model volumes are bound under a dedicated Bidirectional hostPath (agent.hostPaths.nvsnapModels, default /var/lib/containerd/nvsnap-models), mounted at the same path in the agent and on the host. Tests: block completion releases the claim and labels the PV, RWX labels the claim, the reader is not served while a VolumeAttachment exists and is served once it is gone (mutation-checked: dropping the detach gate turns the test red), the read-only PV carries the reader namespace's handle. Co-Authored-By: Balaji Ganesan --- .../nvsnap/cmd/agent/main.go | 2 + .../nvsnap/templates/agent-daemonset.yaml | 16 ++++ .../nvsnap/deploy/helm/nvsnap/values.yaml | 3 + .../nvsnap/internal/agent/agent.go | 14 ++- .../internal/agent/modelvolume_controller.go | 28 ++++-- .../agent/modelvolume_controller_test.go | 66 ++++++++++---- .../internal/agent/webhook_integration.go | 3 +- .../checkpointstore/promoter_shared.go | 11 ++- .../internal/modelvolume/modelvolume.go | 91 +++++++++++++++++-- .../internal/modelvolume/modelvolume_test.go | 49 +++++++++- .../nvsnap/internal/webhook/model_volume.go | 2 +- .../internal/webhook/model_volume_test.go | 2 +- 12 files changed, 236 insertions(+), 51 deletions(-) diff --git a/src/compute-plane-services/nvsnap/cmd/agent/main.go b/src/compute-plane-services/nvsnap/cmd/agent/main.go index ec588b30b2..926f4af308 100644 --- a/src/compute-plane-services/nvsnap/cmd/agent/main.go +++ b/src/compute-plane-services/nvsnap/cmd/agent/main.go @@ -178,6 +178,8 @@ func main() { // (docs/proposals/helm-shared-model-volume.md). Needs L2. flag.BoolVar(&config.ModelVolume.Enabled, "model-volume", false, "Download each model once per cluster into a shared volume and attach it to every other pod that names it (needs L2)") + flag.StringVar(&config.ModelVolume.HostRoot, "model-volume-host-root", "", + "Host directory (mounted Bidirectional into the agent at the same path) where completed model volumes are bound for readers on block storage (default /var/lib/containerd/nvsnap-models)") flag.DurationVar(&config.ModelVolume.WaitDeadline, "model-volume-wait-deadline", 0, "How long a reader waits for the writer's download before downloading itself (default 1h)") diff --git a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-daemonset.yaml b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-daemonset.yaml index 9855664caa..a1b33e9cdb 100644 --- a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-daemonset.yaml +++ b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-daemonset.yaml @@ -116,6 +116,7 @@ spec: # write-once model volume (values: agent.modelVolume) - --model-volume - --model-volume-wait-deadline={{ .Values.agent.modelVolume.waitDeadline | default "1h" }} + - --model-volume-host-root={{ .Values.agent.hostPaths.nvsnapModels | default "/var/lib/containerd/nvsnap-models" }} {{- end }} {{- if and .Values.agent.election .Values.agent.election.enabled }} # one-downloader election (values: agent.election) @@ -322,6 +323,15 @@ spec: - name: nvsnap-overlays mountPath: {{ .Values.agent.hostPaths.nvsnapOverlays | default "/var/lib/containerd/nvsnap-overlays" }} mountPropagation: Bidirectional + {{- if and .Values.agent.modelVolume .Values.agent.modelVolume.enabled }} + # Completed model volumes are bound here for readers on block + # storage; same path in the agent and on the host, Bidirectional + # so the bind reaches kubelet. Not under the overlays root: its + # sweeper removes entries it does not own. + - name: nvsnap-models + mountPath: {{ .Values.agent.hostPaths.nvsnapModels | default "/var/lib/containerd/nvsnap-models" }} + mountPropagation: Bidirectional + {{- end }} {{- if .Values.webhook.enabled }} - name: webhook-tls mountPath: /etc/nvsnap/webhook @@ -350,6 +360,12 @@ spec: hostPath: path: {{ .Values.agent.hostPaths.checkpoints }} type: DirectoryOrCreate + {{- if and .Values.agent.modelVolume .Values.agent.modelVolume.enabled }} + - name: nvsnap-models + hostPath: + path: {{ .Values.agent.hostPaths.nvsnapModels | default "/var/lib/containerd/nvsnap-models" }} + type: DirectoryOrCreate + {{- end }} {{- if .Values.agent.podCacheDir }} - name: cachedir-env configMap: diff --git a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml index 0b18ecbf60..17347a615c 100644 --- a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml +++ b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml @@ -255,6 +255,9 @@ agent: checkpoints: /var/lib/containerd/nvsnap-checkpoints containerdSock: /run/containerd/containerd.sock containerdStorage: /var/lib/containerd + # Bind root for completed model volumes (agent.modelVolume); Bidirectional + # in the agent at the same path. + nvsnapModels: /var/lib/containerd/nvsnap-models # Cache, staging and overlays sit under the containerd root on purpose. # They are the bulk writers -- a 70B cachedir capture is ~132 GB -- and on # a typical GPU node /var/lib is the boot volume (network-backed, ~125 diff --git a/src/compute-plane-services/nvsnap/internal/agent/agent.go b/src/compute-plane-services/nvsnap/internal/agent/agent.go index ee08038ad1..d4bf99055b 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/agent.go +++ b/src/compute-plane-services/nvsnap/internal/agent/agent.go @@ -234,6 +234,10 @@ type ModelVolumeConfig struct { // WaitDeadline bounds a reader's wait for the marker before it downloads // itself. Zero means one hour. WaitDeadline time.Duration + // HostRoot is the host directory, mounted Bidirectional into the agent + // at the same path, where completed model volumes are bound for + // readers on block storage. Default /var/lib/containerd/nvsnap-models. + HostRoot string } // L2BackendConfig is the per-capture PVC L2 backend (nvsnap#63). See @@ -668,7 +672,7 @@ func (a *Agent) Run(ctx context.Context) error { Provisioner: a.modelVolume, Minter: a.modelMinter, NodeName: a.config.NodeName, - HostRoot: filepath.Join(a.config.OverlayRoot, "models"), + HostRoot: a.modelHostRoot(), HolderImage: a.config.L2.WriterImage, HolderPullSecrets: l2PullSecrets(a.config.L2), Log: a.log.WithField("subsys", "modelvolume"), @@ -1253,3 +1257,11 @@ func (a *Agent) readCheckpointFileHandler(w http.ResponseWriter, r *http.Request // receivers can pull one large pages-*.img via parallel ranges. http.ServeContent(w, r, info.Name(), info.ModTime(), f) } + +// modelHostRoot is the bind root for completed model volumes. +func (a *Agent) modelHostRoot() string { + if a.config.ModelVolume.HostRoot != "" { + return a.config.ModelVolume.HostRoot + } + return "/var/lib/containerd/nvsnap-models" +} diff --git a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go index 01ae2f705a..4e393cf0bf 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go +++ b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go @@ -54,7 +54,8 @@ type ModelVolumeController struct { // NodeName is this agent's node; readers elsewhere are ignored. NodeName string // HostRoot is where model volumes are bound for readers: /. - // Must be under the agent's Bidirectional overlays mount. + // Its own Bidirectional hostPath (not the overlays root, whose sweeper + // removes entries it does not own). HostRoot string // HolderNamespaceImage is the image for mount-holder pods (the agent image). HolderImage string @@ -161,13 +162,7 @@ func (c *ModelVolumeController) handleJob(ctx context.Context, obj any) { log.WithError(err).Warn("model volume: mark complete failed") return } - if c.Provisioner.Cfg.Mode == modelvolume.ModeBlock && c.Minter != nil { - if err := c.Minter.MintReadOnly(ctx, job.Namespace, modelvolume.ClaimName(uri), modelvolume.ReadOnlyPVName(uri, job.Namespace), modelvolume.ReadOnlyClaimName(uri), job.Namespace, modelvolume.Key(uri)); err != nil { - log.WithError(err).Warn("model volume: mint read-only claim failed") - return - } - } - log.Info("model volume: download complete; readers may attach") + log.Info("model volume: download complete; readers may attach once the volume detaches") } func (c *ModelVolumeController) handlePendingReader(ctx context.Context, pod *corev1.Pod, uri string) { @@ -186,7 +181,22 @@ func (c *ModelVolumeController) handlePendingReader(ctx context.Context, pod *co c.mu.Unlock() if !already { if c.Minter != nil { - if err := c.Minter.MintReadOnly(ctx, st.ClaimNamespace, modelvolume.ClaimName(uri), modelvolume.ReadOnlyPVName(uri, pod.Namespace), modelvolume.ReadOnlyClaimName(uri), pod.Namespace, modelvolume.Key(uri)); err != nil { + if st.PrimaryPV == "" { + log.Warn("model volume: complete but no primary volume recorded") + return + } + // The read-only attach is refused while the download's + // read-write attachment still exists; wait for the detach. + detached, err := c.Provisioner.Detached(ctx, st.PrimaryPV) + if err != nil { + log.WithError(err).Warn("model volume: detach check failed") + return + } + if !detached { + log.WithField("pv", st.PrimaryPV).Info("model volume: primary still attached; retrying after detach") + return + } + if err := c.Minter.MintReadOnlyFromPV(ctx, st.PrimaryPV, modelvolume.ReadOnlyPVName(uri, pod.Namespace), modelvolume.ReadOnlyClaimName(uri), pod.Namespace, modelvolume.Key(uri)); err != nil { log.WithError(err).Warn("model volume: mint read-only claim in reader namespace failed") return } diff --git a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go index ff4a56ea4c..dfb9a07449 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go +++ b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go @@ -11,6 +11,7 @@ import ( "github.com/sirupsen/logrus" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -91,28 +92,17 @@ func TestModelVolumeController_JobCompletionMintsReadOnly(t *testing.T) { } c.HandleJob(ctx, downloadJob(1)) st, _ := p.Lookup(ctx, mvURI) - if !st.Complete { - t.Fatal("a succeeded Job must mark the claim complete") + if !st.Complete || st.PrimaryPV != "pvc-abc" { + t.Fatalf("a succeeded Job must complete the identity on the retained PV: %+v", st) } - ro, err := kc.CoreV1().PersistentVolumeClaims("sr-fn").Get(ctx, modelvolume.ReadOnlyClaimName(mvURI), metav1.GetOptions{}) - if err != nil { - t.Fatalf("read-only claim must be minted in the writer namespace: %v", err) - } - pv, err := kc.CoreV1().PersistentVolumes().Get(ctx, ro.Spec.VolumeName, metav1.GetOptions{}) - if err != nil { - t.Fatal(err) - } - if pv.Spec.CSI.VolumeHandle != "cluster:csi-abc:vol:sr-fn" || !pv.Spec.CSI.ReadOnly || pv.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain { - t.Errorf("read-only PV: %+v", pv.Spec) + if _, err := kc.CoreV1().PersistentVolumeClaims("sr-fn").Get(ctx, modelvolume.ClaimName(mvURI), metav1.GetOptions{}); err == nil { + t.Error("the download claim must be released so the volume detaches") } primary, _ := kc.CoreV1().PersistentVolumes().Get(ctx, "pvc-abc", metav1.GetOptions{}) - if primary.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain { - t.Error("the writer's PV must be retained; it is the artifact") - } - if _, err := kc.CoreV1().PersistentVolumeClaims("sr-fn").Get(ctx, modelvolume.ClaimName(mvURI), metav1.GetOptions{}); err != nil { - t.Error("the download claim stays: it is the artifact") + if primary.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain || primary.Labels[modelvolume.CompleteLabel] != "true" { + t.Error("the primary PV must be retained and labelled complete; it is the artifact") } - c.HandleJob(ctx, downloadJob(1)) // idempotent + c.HandleJob(ctx, downloadJob(1)) // idempotent after release } func TestModelVolumeController_PendingReaderBoundOnItsNode(t *testing.T) { @@ -142,8 +132,13 @@ func TestModelVolumeController_PendingReaderBoundOnItsNode(t *testing.T) { if len(*attached) != 1 || (*attached)[0] != "other-ns/"+modelvolume.ReadOnlyClaimName(mvURI) { t.Errorf("reader's read-only claim in its own namespace must be attached, got %v", *attached) } - if _, err := kc.CoreV1().PersistentVolumeClaims("other-ns").Get(ctx, modelvolume.ReadOnlyClaimName(mvURI), metav1.GetOptions{}); err != nil { - t.Errorf("read-only claim must be minted in the reader namespace: %v", err) + ro, err := kc.CoreV1().PersistentVolumeClaims("other-ns").Get(ctx, modelvolume.ReadOnlyClaimName(mvURI), metav1.GetOptions{}) + if err != nil { + t.Fatalf("read-only claim must be minted in the reader namespace from the retained PV: %v", err) + } + roPV, _ := kc.CoreV1().PersistentVolumes().Get(ctx, ro.Spec.VolumeName, metav1.GetOptions{}) + if roPV.Spec.CSI.VolumeHandle != "cluster:csi-abc:vol:other-ns" || !roPV.Spec.CSI.ReadOnly { + t.Errorf("read-only PV must carry the reader namespace's handle: %+v", roPV.Spec.CSI) } wantDst := filepath.Join(c.HostRoot, modelvolume.Key(mvURI)) if len(*bound) != 1 || (*bound)[0][1] != wantDst || (*bound)[0][0] == "" { @@ -159,3 +154,34 @@ func TestModelVolumeController_PendingReaderBoundOnItsNode(t *testing.T) { t.Error("the bind is per identity per node, not per pod") } } + +// While the download's read-write attachment still exists, no read-only +// claim is minted and nothing is bound: NVMesh would refuse the attach. +func TestModelVolumeController_WaitsForPrimaryDetach(t *testing.T) { + kc, p := writerFixture(t) + ctx := context.Background() + pvName := "pvc-abc" + va := &storagev1.VolumeAttachment{ObjectMeta: metav1.ObjectMeta{Name: "csi-1"}, Spec: storagev1.VolumeAttachmentSpec{ + Attacher: "nvmesh-csi.excelero.com", NodeName: "node-a", Source: storagev1.VolumeAttachmentSource{PersistentVolumeName: &pvName}}} + if _, err := kc.StorageV1().VolumeAttachments().Create(ctx, va, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + cw, _, _ := mvController(t, kc, p, "node-a") + cw.HandleJob(ctx, downloadJob(1)) + c, attached, bound := mvController(t, kc, p, "node-b") + reader := readerPod("other-ns", "node-b") + if _, err := kc.CoreV1().Pods("other-ns").Create(ctx, reader, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + c.Handle(ctx, reader) + if len(*attached) != 0 || len(*bound) != 0 { + t.Fatal("nothing may be attached while the primary is still attached read-write") + } + if err := kc.StorageV1().VolumeAttachments().Delete(ctx, "csi-1", metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + c.Handle(ctx, reader) + if len(*attached) != 1 || len(*bound) != 1 { + t.Errorf("after detach the reader must be served: attached=%v bound=%v", *attached, *bound) + } +} diff --git a/src/compute-plane-services/nvsnap/internal/agent/webhook_integration.go b/src/compute-plane-services/nvsnap/internal/agent/webhook_integration.go index 2a78cb9295..5123b90a85 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/webhook_integration.go +++ b/src/compute-plane-services/nvsnap/internal/agent/webhook_integration.go @@ -21,7 +21,6 @@ import ( "context" "errors" "fmt" - "path/filepath" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/checkpointstore" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/rootfsonly" @@ -159,7 +158,7 @@ func (a *Agent) startWebhook(ctx context.Context, cfg WebhookConfig, backend che // Write-once model volume for Helm functions; nil when off. ModelVolume: a.modelVolume, ModelWaitDeadline: a.config.ModelVolume.WaitDeadline, - ModelHostRoot: filepath.Join(a.config.OverlayRoot, "models"), + ModelHostRoot: a.modelHostRoot(), Composer: &rootfsonly.HashInputComposer{ CUDADriverMajor: a.config.RootfsCapture.CUDADriverMajor, }, diff --git a/src/compute-plane-services/nvsnap/internal/checkpointstore/promoter_shared.go b/src/compute-plane-services/nvsnap/internal/checkpointstore/promoter_shared.go index bf144eef44..7b38572e87 100644 --- a/src/compute-plane-services/nvsnap/internal/checkpointstore/promoter_shared.go +++ b/src/compute-plane-services/nvsnap/internal/checkpointstore/promoter_shared.go @@ -553,9 +553,16 @@ func (p *SharedVolumePromoter) MintReadOnly(ctx context.Context, writerNS, write if writer.Spec.VolumeName == "" { return fmt.Errorf("writer claim %s/%s has no bound PV yet", writerNS, writerClaim) } - primary, err := p.KubeClient.CoreV1().PersistentVolumes().Get(ctx, writer.Spec.VolumeName, metav1.GetOptions{}) + return p.MintReadOnlyFromPV(ctx, writer.Spec.VolumeName, roPVName, roClaim, ns, labelKey) +} + +// MintReadOnlyFromPV is MintReadOnly for a retained primary PV whose claim +// has already been released (the model volume after its download Job). +func (p *SharedVolumePromoter) MintReadOnlyFromPV(ctx context.Context, primaryPV, roPVName, roClaim, ns, labelKey string) error { + p.applyDefaults() + primary, err := p.KubeClient.CoreV1().PersistentVolumes().Get(ctx, primaryPV, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("get primary PV %s: %w", writer.Spec.VolumeName, err) + return fmt.Errorf("get primary PV %s: %w", primaryPV, err) } if primary.Spec.CSI == nil || primary.Spec.CSI.VolumeHandle == "" { return fmt.Errorf("primary PV %s has no CSI volumeHandle", primary.Name) diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go index 25324b885d..d3a5088daa 100644 --- a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go @@ -115,8 +115,11 @@ type State struct { Exists bool // Complete: the download finished; readers may attach. Complete bool - // ClaimNamespace is where the writer claim lives. + // ClaimNamespace is where the writer claim lives (RWX, or in flight). ClaimNamespace string + // PrimaryPV is the retained volume holding the model (Block mode, + // complete); read-only claims are minted from it. + PrimaryPV string } // Provisioner creates and inspects model volumes. @@ -125,9 +128,10 @@ type Provisioner struct { Cfg Config } -// EnsureWriterClaim creates the claim the writer downloads into, in ns. +// EnsureWriterClaim creates the claim the download Job writes into, in ns. // Idempotent. RWX mode creates a ReadWriteMany claim readers share; Block -// mode a ReadWriteOnce claim that becomes the read-only artifact. +// mode a ReadWriteOnce claim that is released after the download, leaving +// the retained PV as the artifact. func (p *Provisioner) EnsureWriterClaim(ctx context.Context, uri, ns string) (string, error) { name := ClaimName(uri) if _, err := p.Kube.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name, metav1.GetOptions{}); err == nil { @@ -161,14 +165,30 @@ func (p *Provisioner) EnsureWriterClaim(ctx context.Context, uri, ns string) (st return name, nil } -// Lookup finds the writer claim for uri anywhere on the cluster and reports -// whether its download completed. +// Lookup reports the identity's state. Block mode: a retained PV labelled +// complete is the artifact (the writer claim is released after the +// download so the volume detaches); otherwise an in-flight writer claim. +// RWX mode: the shared claim carries the label. func (p *Provisioner) Lookup(ctx context.Context, uri string) (State, error) { + st := State{} + if p.Cfg.Mode == ModeBlock { + pvs, err := p.Kube.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{LabelSelector: IdentityLabel + "=" + Key(uri) + "," + CompleteLabel + "=true"}) + if err != nil { + return State{}, fmt.Errorf("list volumes for %s: %w", uri, err) + } + for i := range pvs.Items { + pv := &pvs.Items[i] + if pv.Labels["nvsnap.io/role"] == "reader-shared" || pv.Spec.CSI == nil || pv.Spec.CSI.ReadOnly { + continue + } + st.Exists, st.Complete, st.PrimaryPV = true, true, pv.Name + return st, nil + } + } list, err := p.Kube.CoreV1().PersistentVolumeClaims("").List(ctx, metav1.ListOptions{LabelSelector: IdentityLabel + "=" + Key(uri)}) if err != nil { return State{}, fmt.Errorf("list claims for %s: %w", uri, err) } - st := State{} for i := range list.Items { c := &list.Items[i] if c.Name != ClaimName(uri) { @@ -178,21 +198,58 @@ func (p *Provisioner) Lookup(ctx context.Context, uri string) (State, error) { st.ClaimNamespace = c.Namespace if c.Labels[CompleteLabel] == "true" { st.Complete = true + st.PrimaryPV = c.Spec.VolumeName return st, nil } } return st, nil } -// MarkComplete labels the writer claim complete. The agent calls it when -// the writer's download step exits 0 (Block mode); in RWX mode the marker -// file is authoritative and this label is informational. +// MarkComplete records that the download finished. RWX mode labels the +// shared claim. Block mode labels the retained PV and deletes the writer +// claim: a claim still bound keeps the volume attached read-write to the +// Job's node, and NVMesh refuses read-only attaches elsewhere until that +// attachment is gone (dev1 2026-09-26). Idempotent. func (p *Provisioner) MarkComplete(ctx context.Context, uri, ns string) error { name := ClaimName(uri) pvc, err := p.Kube.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name, metav1.GetOptions{}) if err != nil { + if apierrors.IsNotFound(err) && p.Cfg.Mode == ModeBlock { + if st, lerr := p.Lookup(ctx, uri); lerr == nil && st.Complete { + return nil // released already + } + } return fmt.Errorf("get claim %s/%s: %w", ns, name, err) } + if p.Cfg.Mode == ModeBlock { + if pvc.Spec.VolumeName == "" { + return fmt.Errorf("claim %s/%s has no bound volume", ns, name) + } + pv, err := p.Kube.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get volume %s: %w", pvc.Spec.VolumeName, err) + } + if pv.Labels[CompleteLabel] != "true" { + if pv.Labels == nil { + pv.Labels = map[string]string{} + } + pv.Labels["app.kubernetes.io/managed-by"] = managedBy + pv.Labels[IdentityLabel] = Key(uri) + pv.Labels[CompleteLabel] = "true" + if pv.Annotations == nil { + pv.Annotations = map[string]string{} + } + pv.Annotations[IdentityAnnotation] = uri + pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain + if _, err := p.Kube.CoreV1().PersistentVolumes().Update(ctx, pv, metav1.UpdateOptions{}); err != nil && !apierrors.IsConflict(err) { + return fmt.Errorf("label volume %s complete: %w", pv.Name, err) + } + } + if err := p.Kube.CoreV1().PersistentVolumeClaims(ns).Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("release writer claim %s/%s: %w", ns, name, err) + } + return nil + } if pvc.Labels[CompleteLabel] == "true" { return nil } @@ -202,7 +259,6 @@ func (p *Provisioner) MarkComplete(ctx context.Context, uri, ns string) error { pvc.Labels[CompleteLabel] = "true" if _, err := p.Kube.CoreV1().PersistentVolumeClaims(ns).Update(ctx, pvc, metav1.UpdateOptions{}); err != nil { if apierrors.IsConflict(err) { - // Every agent marks completion; whoever lost the race re-reads. again, gerr := p.Kube.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name, metav1.GetOptions{}) if gerr == nil && again.Labels[CompleteLabel] == "true" { return nil @@ -213,6 +269,21 @@ func (p *Provisioner) MarkComplete(ctx context.Context, uri, ns string) error { return nil } +// Detached reports whether no VolumeAttachment references pv: the moment +// a Block-mode volume may be attached read-only on any node. +func (p *Provisioner) Detached(ctx context.Context, pv string) (bool, error) { + vas, err := p.Kube.StorageV1().VolumeAttachments().List(ctx, metav1.ListOptions{}) + if err != nil { + return false, fmt.Errorf("list VolumeAttachments: %w", err) + } + for i := range vas.Items { + if src := vas.Items[i].Spec.Source.PersistentVolumeName; src != nil && *src == pv { + return false, nil + } + } + return true, nil +} + // JobName is the download Job for a model URI. func JobName(uri string) string { return "nvsnap-model-dl-" + Key(uri) } diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go index 82399b46b6..fac22d5f8e 100644 --- a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume_test.go @@ -40,9 +40,12 @@ func TestProvisioner_WriterClaimModes(t *testing.T) { } } -func TestProvisioner_LookupAndComplete(t *testing.T) { +func TestProvisioner_LookupAndComplete_Block(t *testing.T) { ctx := context.Background() - kc := fake.NewSimpleClientset() + pv := &corev1.PersistentVolume{ObjectMeta: metav1.ObjectMeta{Name: "pv-model"}, Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + PersistentVolumeSource: corev1.PersistentVolumeSource{CSI: &corev1.CSIPersistentVolumeSource{Driver: "nvmesh-csi.excelero.com", VolumeHandle: "c:v:fn-a"}}}} + kc := fake.NewSimpleClientset(pv) p := &Provisioner{Kube: kc, Cfg: Config{Mode: ModeBlock, StorageClass: "sc", Size: resource.MustParse("1Gi")}} if st, err := p.Lookup(ctx, uri); err != nil || st.Exists || st.Complete { t.Errorf("nothing yet: %+v %v", st, err) @@ -50,25 +53,61 @@ func TestProvisioner_LookupAndComplete(t *testing.T) { if _, err := p.EnsureWriterClaim(ctx, uri, "fn-a"); err != nil { t.Fatal(err) } + pvc, _ := kc.CoreV1().PersistentVolumeClaims("fn-a").Get(ctx, ClaimName(uri), metav1.GetOptions{}) + pvc.Spec.VolumeName = "pv-model" + if _, err := kc.CoreV1().PersistentVolumeClaims("fn-a").Update(ctx, pvc, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } if st, err := p.Lookup(ctx, uri); err != nil || !st.Exists || st.Complete || st.ClaimNamespace != "fn-a" { t.Errorf("in flight: %+v %v", st, err) } if err := p.MarkComplete(ctx, uri, "fn-a"); err != nil { t.Fatal(err) } - if st, _ := p.Lookup(ctx, uri); !st.Complete { + // Block mode: the claim is released so the volume detaches; the retained + // PV carries the identity and completion. + if _, err := kc.CoreV1().PersistentVolumeClaims("fn-a").Get(ctx, ClaimName(uri), metav1.GetOptions{}); err == nil { + t.Error("writer claim must be released after completion on block storage") + } + got, _ := kc.CoreV1().PersistentVolumes().Get(ctx, "pv-model", metav1.GetOptions{}) + if got.Labels[CompleteLabel] != "true" || got.Labels[IdentityLabel] != Key(uri) || got.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain || got.Annotations[IdentityAnnotation] != uri { + t.Errorf("primary PV must be labelled complete and retained: %+v", got.ObjectMeta) + } + st, _ := p.Lookup(ctx, uri) + if !st.Complete || st.PrimaryPV != "pv-model" { t.Errorf("after MarkComplete: %+v", st) } if err := p.MarkComplete(ctx, uri, "fn-a"); err != nil { - t.Errorf("second MarkComplete must be a no-op: %v", err) + t.Errorf("second MarkComplete (claim gone, PV complete) must be a no-op: %v", err) } - // Another identity is unaffected. if st, _ := p.Lookup(ctx, "hf://other/model"); st.Exists { t.Error("lookup must be per identity") } if len(Key(uri)) != 16 || ClaimName(uri) != "nvsnap-model-"+Key(uri) || ReadOnlyClaimName(uri) != ClaimName(uri)+"-ro" { t.Errorf("names: %s %s", ClaimName(uri), ReadOnlyClaimName(uri)) } + if d, _ := p.Detached(ctx, "pv-model"); !d { + t.Error("no VolumeAttachment means detached") + } +} + +func TestProvisioner_LookupAndComplete_RWX(t *testing.T) { + ctx := context.Background() + kc := fake.NewSimpleClientset() + p := &Provisioner{Kube: kc, Cfg: Config{Mode: ModeRWX, StorageClass: "sc", Size: resource.MustParse("1Gi")}} + if _, err := p.EnsureWriterClaim(ctx, uri, "fn-a"); err != nil { + t.Fatal(err) + } + if err := p.MarkComplete(ctx, uri, "fn-a"); err != nil { + t.Fatal(err) + } + pvc, err := kc.CoreV1().PersistentVolumeClaims("fn-a").Get(ctx, ClaimName(uri), metav1.GetOptions{}) + if err != nil || pvc.Labels[CompleteLabel] != "true" { + t.Errorf("RWX keeps the shared claim and labels it: %v %v", err, pvc.Labels) + } + if st, _ := p.Lookup(ctx, uri); !st.Complete { + t.Errorf("RWX complete: %+v", st) + } } func TestProvisioner_DownloadJobIdempotent(t *testing.T) { diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go index 865acd18c7..16a68becd4 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go @@ -224,7 +224,7 @@ func (m *Mutator) substituteLandingVolume(pod *corev1.Pod, main *corev1.Containe func (m *Mutator) hostPathLanding(pod *corev1.Pod, main *corev1.Container, land modelid.Landing, uri string) []PatchOp { root := m.ModelHostRoot if root == "" { - root = "/var/lib/containerd/nvsnap-overlays/models" + root = "/var/lib/containerd/nvsnap-models" } hp := corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: path.Join(root, modelvolume.Key(uri)), Type: hostPathType(corev1.HostPathDirectoryOrCreate)}} prop := corev1.MountPropagationHostToContainer diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go index 124c643634..d09b4f3fd4 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go @@ -168,7 +168,7 @@ func TestModelVolume_ReaderBlock_PendingBind(t *testing.T) { t.Errorf("Block reader stamp: %v %v", v.labels, v.annotations) } vol, replaced := v.volumes["ngc-models"] - if !replaced || vol.HostPath == nil || vol.HostPath.Path != "/var/lib/containerd/nvsnap-overlays/models/"+modelvolume.Key("ngc://org/team/nemotron3-ultra-genrm:bf16-fixed") || *vol.HostPath.Type != corev1.HostPathDirectoryOrCreate { + if !replaced || vol.HostPath == nil || vol.HostPath.Path != "/var/lib/containerd/nvsnap-models/"+modelvolume.Key("ngc://org/team/nemotron3-ultra-genrm:bf16-fixed") || *vol.HostPath.Type != corev1.HostPathDirectoryOrCreate { t.Errorf("Block reader lands on a hostPath under the model host root for the agent to bind into, got %+v", vol) } var propagations int From 3ea6111eda7e127ba19a966091721e2c6514f68b Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 09:09:19 -0700 Subject: [PATCH 09/16] docs(nvsnap): record the unbind and read-only PV lifecycle gaps seen on dev1 Co-Authored-By: Balaji Ganesan --- .../nvsnap/docs/proposals/helm-shared-model-volume.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md index 85ba51acf4..0b7757fb55 100644 --- a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md +++ b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md @@ -142,7 +142,9 @@ between them. Everything after that first start is a full hit. | agent down on a reader node (NVMesh) | no bind arrives | wait deadline, local download | | writer pod restarts after complete | volume immutable, unaffected | none needed | | identity changes (revision, quantization, image) | different URI or cache key | separate volume; old one ages out | -| gang scheduler | readers always schedulable (RWX bound, or emptyDir); writer PVC binds in seconds on Immediate storage classes | none needed | +| gang scheduler | readers always schedulable (RWX bound, or hostPath); download claim binds in seconds on Immediate storage classes | none needed | +| identity deleted while a read-only PV is still Terminating (NVMesh) | the read-only PV name is deterministic per identity and namespace, so a re-download of the same identity cannot mint until the old PV finalizes; the attacher's detach timed out for minutes after the volume was gone | retention deletes read-only claims and PVs before the primary, and the controller retries minting; a stale VolumeAttachment on a deleted volume needs the finalizer cleared (seen on dev1 2026-09-26) | +| last reader of an identity leaves a node (NVMesh) | the agent's bind mount keeps the volume published; kubelet cannot unmount and the attacher's detach times out (seen on dev1 2026-09-26 during cleanup) | the agent must unbind and drop its mount-holder when no pod on the node uses the identity; part of retention (follow-up) | ## What changes in the code From 48b17211dee494202cd3b1194c275f7f8f6eaa97 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 09:41:05 -0700 Subject: [PATCH 10/16] fix(nvsnap): verify model volume binds against the mount table, rebind when stale The controller kept "bound" in memory. After the volume for an identity was replaced (and after a manual unmount during cleanup on dev1) it un-pended readers without a bind and they waited on an empty hostPath. The mount table is now the truth: a reader is served only when the device mounted at the bind target belongs to the identity's primary PV (NVMesh device csi-id against the volume handle); a missing mount is bound again, a mount of a replaced volume is unbound and redone, and a bind that leaves nothing mounted does not un-pend the pod. Tests: rebinding after an unmount, replacing a stale bind, device/handle matching. Mutation-checked: dropping the stale unbind and the missing mount check each turn tests red. Co-Authored-By: Balaji Ganesan --- .../internal/agent/modelvolume_controller.go | 97 +++++++++++++++---- .../agent/modelvolume_controller_test.go | 66 ++++++++++++- 2 files changed, 144 insertions(+), 19 deletions(-) diff --git a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go index 4e393cf0bf..b221806725 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go +++ b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go @@ -63,12 +63,15 @@ type ModelVolumeController struct { Log logrus.FieldLogger // Seams for tests: attach returns the host path a claim is mounted at - // on this node; bind bind-mounts src onto dst read-only. - attach func(ctx context.Context, ns, claim string) (string, error) - bind func(src, dst string) error + // on this node; bind bind-mounts src onto dst read-only; unbind undoes + // it; mountedDevice reports the block device mounted at a path ("" when + // nothing is mounted there). + attach func(ctx context.Context, ns, claim string) (string, error) + bind func(src, dst string) error + unbind func(dst string) error + mountedDevice func(dst string) string mu sync.Mutex - bound map[string]bool // dst paths already bound holders map[string]*checkpointstore.MountHolder } @@ -101,9 +104,6 @@ func (c *ModelVolumeController) Run(ctx context.Context) error { } func (c *ModelVolumeController) init() { - if c.bound == nil { - c.bound = map[string]bool{} - } if c.holders == nil { c.holders = map[string]*checkpointstore.MountHolder{} } @@ -113,6 +113,12 @@ func (c *ModelVolumeController) init() { if c.bind == nil { c.bind = bindReadOnly } + if c.unbind == nil { + c.unbind = func(dst string) error { return syscall.Unmount(dst, syscall.MNT_DETACH) } + } + if c.mountedDevice == nil { + c.mountedDevice = mountedDeviceAt + } } func (c *ModelVolumeController) log() logrus.FieldLogger { @@ -176,15 +182,28 @@ func (c *ModelVolumeController) handlePendingReader(ctx context.Context, pod *co return // the wait init keeps waiting; the writer's completion re-triggers via its own event } dst := filepath.Join(c.HostRoot, modelvolume.Key(uri)) - c.mu.Lock() - already := c.bound[dst] - c.mu.Unlock() - if !already { + // The mount table is the truth, not memory: the agent may have + // restarted, the volume may have been replaced by a re-download of the + // same identity, or an operator may have unmounted by hand. A bind is + // current only if the device under dst belongs to this primary PV. + primary, err := c.Kube.CoreV1().PersistentVolumes().Get(ctx, st.PrimaryPV, metav1.GetOptions{}) + if err != nil { + log.WithError(err).Warn("model volume: get primary PV failed") + return + } + handle := "" + if primary.Spec.CSI != nil { + handle = primary.Spec.CSI.VolumeHandle + } + if dev := c.mountedDevice(dst); dev != "" && !deviceMatchesHandle(dev, handle) { + log.WithFields(logrus.Fields{"dst": dst, "device": dev}).Info("model volume: stale bind for a replaced volume; unbinding") + if err := c.unbind(dst); err != nil { + log.WithError(err).Warn("model volume: unbind stale bind failed") + return + } + } + if dev := c.mountedDevice(dst); dev == "" { if c.Minter != nil { - if st.PrimaryPV == "" { - log.Warn("model volume: complete but no primary volume recorded") - return - } // The read-only attach is refused while the download's // read-write attachment still exists; wait for the detach. detached, err := c.Provisioner.Detached(ctx, st.PrimaryPV) @@ -214,9 +233,10 @@ func (c *ModelVolumeController) handlePendingReader(ctx context.Context, pod *co log.WithError(err).Warn("model volume: bind failed") return } - c.mu.Lock() - c.bound[dst] = true - c.mu.Unlock() + if dev := c.mountedDevice(dst); dev == "" { + log.WithField("dst", dst).Warn("model volume: bind reported success but nothing is mounted at the target; not un-pending") + return + } log.WithFields(logrus.Fields{"src": src, "dst": dst}).Info("model volume: bound read-only volume for reader") } patch, _ := json.Marshal(map[string]any{"metadata": map[string]any{"labels": map[string]string{modelvolume.PendingLabel: "false"}}}) @@ -277,3 +297,44 @@ func bindReadOnly(src, dst string) error { } return nil } + +// mountedDeviceAt returns the source device of the mount at dst, or "" +// when dst is not a mount point. Reads the agent's own mount table; dst +// lives under the Bidirectional model root, so agent and host agree. +func mountedDeviceAt(dst string) string { + data, err := os.ReadFile("/proc/self/mountinfo") + if err != nil { + return "" + } + dst = filepath.Clean(dst) + best := "" + for _, line := range strings.Split(string(data), "\n") { + f := strings.Fields(line) + if len(f) < 10 { + continue + } + if f[4] != dst { + continue + } + // fields after the "-" separator: fstype, source, options + for i := 6; i < len(f)-2; i++ { + if f[i] == "-" { + best = f[i+2] + break + } + } + } + return best +} + +// deviceMatchesHandle reports whether a mounted device belongs to the CSI +// volume behind handle. NVMesh devices are /dev/nvmesh/ and the +// handle carries the same csi-id segment; other drivers fall back to +// "something is mounted, trust it". +func deviceMatchesHandle(device, handle string) bool { + base := filepath.Base(device) + if !strings.HasPrefix(base, "csi-") || handle == "" { + return true + } + return strings.Contains(handle, base) +} diff --git a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go index dfb9a07449..330d1257ce 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go +++ b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go @@ -54,12 +54,19 @@ func mvController(t *testing.T, kc *fake.Clientset, p *modelvolume.Provisioner, minter := &checkpointstore.SharedVolumePromoter{KubeClient: kc, StorageClass: "nvcf-sc", Transform: tx, MountOptions: []string{"ro", "norecovery", "nouuid"}, Log: logrus.New()} att := []string{} bnd := [][2]string{} + mounts := map[string]string{} // dst -> device, the fake mount table c = &ModelVolumeController{Kube: kc, Provisioner: p, Minter: minter, NodeName: node, HostRoot: filepath.Join(t.TempDir(), "models"), Log: logrus.New()} c.attach = func(_ context.Context, ns, claim string) (string, error) { att = append(att, ns+"/"+claim) return "/host/var/lib/kubelet/pods/h/volumes/kubernetes.io~csi/pv/mount", nil } - c.bind = func(src, dst string) error { bnd = append(bnd, [2]string{src, dst}); return nil } + c.bind = func(src, dst string) error { + bnd = append(bnd, [2]string{src, dst}) + mounts[dst] = "/dev/nvmesh/csi-abc" + return nil + } + c.unbind = func(dst string) error { delete(mounts, dst); return nil } + c.mountedDevice = func(dst string) string { return mounts[dst] } return c, &att, &bnd } @@ -185,3 +192,60 @@ func TestModelVolumeController_WaitsForPrimaryDetach(t *testing.T) { t.Errorf("after detach the reader must be served: attached=%v bound=%v", *attached, *bound) } } + +// Memory is not the truth: with nothing mounted at the target (agent +// restarted, operator unmounted) the reader is bound again, and a bind that +// belongs to a replaced volume of the same identity is redone. +func TestModelVolumeController_RebindsWhenMountIsMissingOrStale(t *testing.T) { + kc, p := writerFixture(t) + ctx := context.Background() + cw, _, _ := mvController(t, kc, p, "node-a") + cw.HandleJob(ctx, downloadJob(1)) + c, _, bound := mvController(t, kc, p, "node-b") + reader := readerPod("third-ns", "node-b") + if _, err := kc.CoreV1().Pods("third-ns").Create(ctx, reader, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + c.Handle(ctx, reader) + if len(*bound) != 1 { + t.Fatalf("first bind expected, got %v", *bound) + } + // Someone unmounted it: the next reader event binds again. + _ = c.unbind((*bound)[0][1]) + c.Handle(ctx, reader) + if len(*bound) != 2 { + t.Errorf("a missing mount must be bound again, got %d binds", len(*bound)) + } + // The identity was re-downloaded onto a new volume: the old bind is + // stale and must be replaced. + c.mountedDevice = func(string) string { return "/dev/nvmesh/csi-OLD" } + unbound := 0 + c.unbind = func(string) error { unbound++; c.mountedDevice = func(string) string { return "" }; return nil } + c.Handle(ctx, reader) + if unbound != 1 || len(*bound) != 3 { + t.Errorf("stale bind must be unbound and redone: unbound=%d binds=%d", unbound, len(*bound)) + } + if !deviceMatchesHandle("/dev/nvmesh/csi-abc", "cluster:csi-abc:vol:ns") || deviceMatchesHandle("/dev/nvmesh/csi-old", "cluster:csi-abc:vol:ns") || !deviceMatchesHandle("/dev/md127", "anything") { + t.Error("deviceMatchesHandle") + } +} + +// A bind that leaves nothing mounted must not un-pend the reader; the +// wait init would otherwise be released against an empty directory. +func TestModelVolumeController_NoUnpendWithoutAMount(t *testing.T) { + kc, p := writerFixture(t) + ctx := context.Background() + cw, _, _ := mvController(t, kc, p, "node-a") + cw.HandleJob(ctx, downloadJob(1)) + c, _, _ := mvController(t, kc, p, "node-b") + c.bind = func(string, string) error { return nil } // reports success, mounts nothing + reader := readerPod("other-ns", "node-b") + if _, err := kc.CoreV1().Pods("other-ns").Create(ctx, reader, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + c.Handle(ctx, reader) + got, _ := kc.CoreV1().Pods("other-ns").Get(ctx, "r-1", metav1.GetOptions{}) + if got.Labels[modelvolume.PendingLabel] != "true" { + t.Error("reader must stay pending when nothing is mounted at the bind target") + } +} From 7b4b180d0906106731dd248840058e3a6c09a4c4 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 10:02:18 -0700 Subject: [PATCH 11/16] docs(nvsnap): record the model volume e2e results from dev1 Co-Authored-By: Balaji Ganesan --- .../proposals/helm-shared-model-volume.md | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md index 0b7757fb55..7920a4e6ab 100644 --- a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md +++ b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md @@ -174,3 +174,39 @@ Removed for Helm: `schedulingGates`, promote-to-ROX of the whole tree, 5. Retire the gate; e2e on dev1 in all matrix rows that dev1 can host (NVMesh; DFS stands in with an NFS class), each measured cold, first deploy with two pods per instance, redeploy, second namespace. + +## Results, dev1 2026-09-26 (NVMesh, block mode) + +Stock `vllm-workers` chart, Qwen2.5-32B-Instruct TP=4, replicas=2 on two +nodes, no nvsnap markers in the chart, agent v0.2.76-mv5. + +``` +first deploy (nothing on the cluster) + t+0 both pods admitted as readers (hostPath landing, wait init); one Job created + t+354s Job succeeded: 65 GB via `hf download` into the RWO claim; claim released + t+400s both readers un-pended (read-only PV minted per namespace, attached via + mount-holder, bound under /var/lib/containerd/nvsnap-models/) + t+591s both Ready; 0 downloads in either pod; serve " Paris. Correct!" +uninstall + reinstall + t+12s both un-pended (identity complete: no Job) + t+136s both Ready +earlier run, identity already complete on the cluster + t+205s both Ready, wait init 10-15 s, engine reads the volume directly +``` + +Cold start of the same pod on the same node: 325 s. The reinstall number +is the engine's own load and compile from a read-only NVMesh mount with no +prewarm; compile caches on block storage are still the follow-up (step 4). + +Findings that changed the design during these runs: + +- NVMesh refuses a read-only attach on any node while the volume is + attached read-write anywhere, including a Succeeded Job pod that still + exists. Hence the download Job, its TTL, releasing the claim on + completion, and the detach check before minting. +- The overlays root is swept by the L1 overlay GC; binds live under their + own Bidirectional hostPath. +- Memory is not a mount table: binds are verified against the mounted + device and the volume handle and redone when missing or stale. +- The agent's binds pin the volume on the node; unbinding when the last + reader leaves is part of retention (open). From 1dd2567a7611aeecccf4904559a936afb516f95c Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 14:26:20 -0700 Subject: [PATCH 12/16] fix(nvsnap): make the model volume fit NVCF function namespaces Three gaps against the shape of a real NVCF Helm function (an NGC init download with a secrets file and a scripts ConfigMap, in a namespace with Kyverno enforced): - the download Job carries every volume the chart's init mounted, with only the landing volume redirected to the claim; before, the Job had only the landing mount and the download could not read its key - block readers default to a `pvc` reader mode: the pod references the read-only claim in its namespace, the webhook mints it at admission when the volume is complete, and any agent mints it after the Job when the primary detaches; no hostPath, so disallow-host-path passes. The former bind-in path is `readerMode: hostPath` in the storage profile, for gang-scheduled charts - Job pods get default requests and limits, a seccomp profile, dropped capabilities and no privilege escalation when the init set none Co-Authored-By: Balaji Ganesan --- .../nvsnap/deploy/helm/nvsnap/values.yaml | 7 +- .../proposals/helm-shared-model-volume.md | 33 +++-- .../nvsnap/internal/agent/l2_integration.go | 3 + .../internal/agent/modelvolume_controller.go | 40 +++++- .../agent/modelvolume_controller_test.go | 55 ++++++++- .../internal/agent/webhook_integration.go | 10 ++ .../checkpointstore/storage_profile.go | 4 + .../internal/modelvolume/modelvolume.go | 69 +++++++++-- .../nvsnap/internal/webhook/BUILD.bazel | 1 + .../nvsnap/internal/webhook/model_volume.go | 43 +++++-- .../internal/webhook/model_volume_test.go | 116 +++++++++++++++++- .../nvsnap/internal/webhook/mutate.go | 16 ++- 12 files changed, 355 insertions(+), 42 deletions(-) diff --git a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml index 17347a615c..d90da51df1 100644 --- a/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml +++ b/src/compute-plane-services/nvsnap/deploy/helm/nvsnap/values.yaml @@ -119,8 +119,11 @@ agent: # once per cluster into a per-identity volume and every other pod that # names it attaches that volume; compile caches are shared the same way. # Mode comes from the storage profile: block on NVMesh (the L2 class), - # rwx when the profile declares a distributed-filesystem class. Needs - # agent.l2. Off until qualified. + # rwx when the profile declares a distributed-filesystem class. Block + # readers reference a read-only claim by default (profile + # modelVolume.readerMode: pvc); readerMode: hostPath schedules readers + # at once and binds the volume in, for gang-scheduled charts where + # policy allows hostPath. Needs agent.l2. Off until qualified. modelVolume: enabled: false # How long a reader waits for the writer's download before it downloads diff --git a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md index 7920a4e6ab..78d524c5a7 100644 --- a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md +++ b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md @@ -87,15 +87,27 @@ not. namespaces (`EnsureClaim`, done) and new versions of the function all attach the same volume. There is no capture copy of the model anymore. -4. Readers never block scheduling (webhook + agent). On a distributed - filesystem the RWX claim exists and is bound at admission, so the pod - schedules. On NVMesh, before the download is complete, a reader cannot - reference a bindable claim, so it gets an emptyDir at the landing path - plus `nvsnap-wait-model`; once complete, the agent on the reader's node - attaches the read-only volume (mount-holder, existing) and bind-mounts - it over the emptyDir, then drops the marker the wait init is polling. - After completion, NVMesh readers reference the read-only claim directly. - No pod ever needs network access to nvsnap; function namespaces block it. +4. Readers reach the volume without help from inside the pod (webhook + + agent). On a distributed filesystem the RWX claim exists and is bound + at admission, so the pod schedules. On NVMesh the storage profile picks + one of two reader modes (`modelVolume.readerMode`): + - `pvc` (default). The reader references the read-only claim + `nvsnap-model--ro` in its own namespace. Complete already: the + webhook mints the claim at admission and the pod binds at once. Not + yet: the pod stays Pending on volume binding; when the Job succeeds + and the primary is detached, any agent mints the claim and kubelet + starts the pod. No hostPath, so it passes Kyverno's + `disallow-host-path` in NVCF function namespaces (enforced there). + A pod Pending on a claim holds a gang scheduler, so this mode is for + Deployments, StatefulSets and LWS, which is every NVCF chart today. + - `hostPath`. The reader gets a hostPath landing under the agent's + model root and schedules at once; the agent on its node attaches the + read-only volume (mount-holder) and bind-mounts it over the landing, + and the marker the wait init polls appears. For gang-scheduled + workloads (Grove, kai-scheduler) where policy allows hostPath. + In both modes the init waits for the marker and falls back to its own + download at the deadline. No pod ever needs network access to nvsnap; + function namespaces block it. 5. Compile caches (webhook env + agent). All caches are redirected to a cache location keyed by image digest plus identity plus role-neutral @@ -142,7 +154,8 @@ between them. Everything after that first start is a full hit. | agent down on a reader node (NVMesh) | no bind arrives | wait deadline, local download | | writer pod restarts after complete | volume immutable, unaffected | none needed | | identity changes (revision, quantization, image) | different URI or cache key | separate volume; old one ages out | -| gang scheduler | readers always schedulable (RWX bound, or hostPath); download claim binds in seconds on Immediate storage classes | none needed | +| gang scheduler | readers always schedulable in RWX or hostPath mode; in `pvc` mode readers pend on binding until the download completes, so gang-scheduled charts use `hostPath` | profile `readerMode: hostPath` | +| function namespace with Kyverno enforced (`disallow-host-path`, requests and limits, no SA token) | `pvc` mode uses no hostPath; the download Job carries every mount the chart's init had (registry key secret, script ConfigMap), default requests and limits, seccomp, dropped capabilities and no token | none needed | | identity deleted while a read-only PV is still Terminating (NVMesh) | the read-only PV name is deterministic per identity and namespace, so a re-download of the same identity cannot mint until the old PV finalizes; the attacher's detach timed out for minutes after the volume was gone | retention deletes read-only claims and PVs before the primary, and the controller retries minting; a stale VolumeAttachment on a deleted volume needs the finalizer cleared (seen on dev1 2026-09-26) | | last reader of an identity leaves a node (NVMesh) | the agent's bind mount keeps the volume published; kubelet cannot unmount and the attacher's detach times out (seen on dev1 2026-09-26 during cleanup) | the agent must unbind and drop its mount-holder when no pod on the node uses the identity; part of retention (follow-up) | diff --git a/src/compute-plane-services/nvsnap/internal/agent/l2_integration.go b/src/compute-plane-services/nvsnap/internal/agent/l2_integration.go index 19b63bdd80..5435fb77c3 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/l2_integration.go +++ b/src/compute-plane-services/nvsnap/internal/agent/l2_integration.go @@ -371,6 +371,9 @@ func buildModelVolume(kc kubernetes.Interface, profile *checkpointstore.StorageP if mvp.StorageClass != "" { cfg.StorageClass = mvp.StorageClass } + if mvp.ReaderMode != "" { + cfg.Reader = modelvolume.ReaderMode(mvp.ReaderMode) + } if mvp.Size != "" { q, err := resource.ParseQuantity(mvp.Size) if err != nil { diff --git a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go index b221806725..6ab4b731dc 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go +++ b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller.go @@ -139,7 +139,13 @@ func (c *ModelVolumeController) handle(ctx context.Context, obj any) { if uri == "" || pod.Labels[modelvolume.RoleLabel] != "reader" { return } - if pod.Labels[modelvolume.PendingLabel] == "true" && pod.Spec.NodeName == c.NodeName { + if pod.Labels[modelvolume.PendingLabel] != "true" { + return + } + // hostPath readers are served by the agent on their node (it does the + // bind); PVC readers may be unscheduled, so every agent serves them and + // the idempotent mint converges. + if c.Provisioner.Cfg.ReaderMode() == modelvolume.ReaderPVC || pod.Spec.NodeName == c.NodeName { c.handlePendingReader(ctx, pod, uri) } } @@ -181,6 +187,10 @@ func (c *ModelVolumeController) handlePendingReader(ctx context.Context, pod *co if !st.Complete { return // the wait init keeps waiting; the writer's completion re-triggers via its own event } + if c.Provisioner.Cfg.ReaderMode() == modelvolume.ReaderPVC { + c.servePVCReader(ctx, pod, uri, st, log) + return + } dst := filepath.Join(c.HostRoot, modelvolume.Key(uri)) // The mount table is the truth, not memory: the agent may have // restarted, the volume may have been replaced by a re-download of the @@ -338,3 +348,31 @@ func deviceMatchesHandle(device, handle string) bool { } return strings.Contains(handle, base) } + +// servePVCReader mints the read-only claim the pending reader already +// references, once the primary is detached, and un-pends it. Kubelet +// binds the claim and starts the pod; no hostPath and no agent bind. +func (c *ModelVolumeController) servePVCReader(ctx context.Context, pod *corev1.Pod, uri string, st modelvolume.State, log logrus.FieldLogger) { + if c.Minter == nil || st.PrimaryPV == "" { + return + } + detached, err := c.Provisioner.Detached(ctx, st.PrimaryPV) + if err != nil { + log.WithError(err).Warn("model volume: detach check failed") + return + } + if !detached { + log.WithField("pv", st.PrimaryPV).Info("model volume: primary still attached; retrying after detach") + return + } + if err := c.Minter.MintReadOnlyFromPV(ctx, st.PrimaryPV, modelvolume.ReadOnlyPVName(uri, pod.Namespace), modelvolume.ReadOnlyClaimName(uri), pod.Namespace, modelvolume.Key(uri)); err != nil { + log.WithError(err).Warn("model volume: mint read-only claim in reader namespace failed") + return + } + patch, _ := json.Marshal(map[string]any{"metadata": map[string]any{"labels": map[string]string{modelvolume.PendingLabel: "false"}}}) + if _, err := c.Kube.CoreV1().Pods(pod.Namespace).Patch(ctx, pod.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + log.WithError(err).Warn("model volume: un-pend reader failed") + return + } + log.WithField("claim", modelvolume.ReadOnlyClaimName(uri)).Info("model volume: read-only claim minted for reader") +} diff --git a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go index 330d1257ce..63bf0cdad4 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go +++ b/src/compute-plane-services/nvsnap/internal/agent/modelvolume_controller_test.go @@ -36,7 +36,7 @@ func writerFixture(t *testing.T) (*fake.Clientset, *modelvolume.Provisioner) { }, } kc := fake.NewSimpleClientset(pv) - p := &modelvolume.Provisioner{Kube: kc, Cfg: modelvolume.Config{Mode: modelvolume.ModeBlock, StorageClass: sc, Size: resource.MustParse("512Gi")}} + p := &modelvolume.Provisioner{Kube: kc, Cfg: modelvolume.Config{Mode: modelvolume.ModeBlock, StorageClass: sc, Size: resource.MustParse("512Gi"), Reader: modelvolume.ReaderHostPath}} if _, err := p.EnsureWriterClaim(context.Background(), mvURI, "sr-fn"); err != nil { t.Fatal(err) } @@ -249,3 +249,56 @@ func TestModelVolumeController_NoUnpendWithoutAMount(t *testing.T) { t.Error("reader must stay pending when nothing is mounted at the bind target") } } + +// PVC reader mode (the default): the reader references the read-only +// claim; the agent mints it in the reader's namespace once the primary is +// detached and un-pends the pod. No bind, no hostPath, any agent serves +// it because the pod may still be unscheduled. +func TestModelVolumeController_PVCReaderMintedWithoutBind(t *testing.T) { + kc, p := writerFixture(t) + p.Cfg.Reader = "" + ctx := context.Background() + c, attached, bound := mvController(t, kc, p, "node-b") + reader := readerPod("other-ns", "") + if _, err := kc.CoreV1().Pods("other-ns").Create(ctx, reader, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + c.Handle(ctx, reader) + if _, err := kc.CoreV1().PersistentVolumeClaims("other-ns").Get(ctx, modelvolume.ReadOnlyClaimName(mvURI), metav1.GetOptions{}); err == nil { + t.Fatal("no read-only claim before the download completes") + } + pvAbc := "pvc-abc" + va := &storagev1.VolumeAttachment{ObjectMeta: metav1.ObjectMeta{Name: "va-1"}, Spec: storagev1.VolumeAttachmentSpec{Attacher: "nvmesh-csi.excelero.com", NodeName: "node-a", Source: storagev1.VolumeAttachmentSource{PersistentVolumeName: &pvAbc}}} + if _, err := kc.StorageV1().VolumeAttachments().Create(ctx, va, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + cw, _, _ := mvController(t, kc, p, "node-a") + cw.HandleJob(ctx, downloadJob(1)) + c.Handle(ctx, reader) + if _, err := kc.CoreV1().PersistentVolumeClaims("other-ns").Get(ctx, modelvolume.ReadOnlyClaimName(mvURI), metav1.GetOptions{}); err == nil { + t.Fatal("no read-only claim while the primary is attached read-write") + } + if err := kc.StorageV1().VolumeAttachments().Delete(ctx, "va-1", metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + c.Handle(ctx, reader) + ro, err := kc.CoreV1().PersistentVolumeClaims("other-ns").Get(ctx, modelvolume.ReadOnlyClaimName(mvURI), metav1.GetOptions{}) + if err != nil { + t.Fatalf("read-only claim must be minted in the reader namespace: %v", err) + } + roPV, _ := kc.CoreV1().PersistentVolumes().Get(ctx, ro.Spec.VolumeName, metav1.GetOptions{}) + if roPV.Spec.CSI.VolumeHandle != "cluster:csi-abc:vol:other-ns" || !roPV.Spec.CSI.ReadOnly { + t.Errorf("read-only PV must carry the reader namespace's handle: %+v", roPV.Spec.CSI) + } + if len(*attached) != 0 || len(*bound) != 0 { + t.Errorf("PVC mode never attaches a holder or binds: attached=%v bound=%v", *attached, *bound) + } + got, _ := kc.CoreV1().Pods("other-ns").Get(ctx, "r-1", metav1.GetOptions{}) + if got.Labels[modelvolume.PendingLabel] != "false" { + t.Errorf("reader must be un-pended, labels %v", got.Labels) + } + c.Handle(ctx, reader) // idempotent + if len(*attached) != 0 { + t.Error("nothing to attach on the second pass either") + } +} diff --git a/src/compute-plane-services/nvsnap/internal/agent/webhook_integration.go b/src/compute-plane-services/nvsnap/internal/agent/webhook_integration.go index 5123b90a85..4a887018b5 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/webhook_integration.go +++ b/src/compute-plane-services/nvsnap/internal/agent/webhook_integration.go @@ -159,6 +159,7 @@ func (a *Agent) startWebhook(ctx context.Context, cfg WebhookConfig, backend che ModelVolume: a.modelVolume, ModelWaitDeadline: a.config.ModelVolume.WaitDeadline, ModelHostRoot: a.modelHostRoot(), + ReadOnlyMinter: a.modelReadOnlyMinter(), Composer: &rootfsonly.HashInputComposer{ CUDADriverMajor: a.config.RootfsCapture.CUDADriverMajor, }, @@ -215,3 +216,12 @@ func (a *Agent) startWebhook(ctx context.Context, cfg WebhookConfig, backend che }() return nil } + +// modelReadOnlyMinter is the webhook's minter for completed Block-mode +// volumes; nil (a nil interface) when there is none. +func (a *Agent) modelReadOnlyMinter() webhook.ReadOnlyMinter { + if a.modelMinter == nil { + return nil + } + return a.modelMinter +} diff --git a/src/compute-plane-services/nvsnap/internal/checkpointstore/storage_profile.go b/src/compute-plane-services/nvsnap/internal/checkpointstore/storage_profile.go index 71b74a5464..92447585e7 100644 --- a/src/compute-plane-services/nvsnap/internal/checkpointstore/storage_profile.go +++ b/src/compute-plane-services/nvsnap/internal/checkpointstore/storage_profile.go @@ -99,6 +99,10 @@ type ModelVolumeProfile struct { // Size requested per volume (a ceiling; the model size is unknown at // admission). Empty means "512Gi". Size string `json:"size,omitempty"` + // ReaderMode for block volumes: "pvc" (default; policy-friendly, pods + // wait on volume binding) or "hostPath" (schedules at once, agent binds; + // needed under gang schedulers, requires hostPath allowed by policy). + ReaderMode string `json:"readerMode,omitempty"` } // DefaultPrewarmParallelism is the reader count when a profile does not set diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go index d3a5088daa..bfc266e055 100644 --- a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go @@ -80,6 +80,20 @@ const ( managedBy = "nvsnap" ) +// ReaderMode is how a Block-mode reader reaches the finished volume. +type ReaderMode string + +// Reader modes. PVC: the pod references the read-only claim in its own +// namespace and stays Pending on volume binding until the agent mints it; +// works under Kyverno's disallow-host-path and needs no bind, but a pod +// pending on a claim holds a gang scheduler. HostPath: the pod schedules +// at once on a hostPath landing and the agent binds the volume in; needs +// hostPath allowed by policy. Default PVC. +const ( + ReaderPVC ReaderMode = "pvc" + ReaderHostPath ReaderMode = "hostPath" +) + // Config comes from the storage profile. type Config struct { Mode Mode @@ -88,6 +102,16 @@ type Config struct { // admission, so this is a ceiling. Thin-provisioned classes make it // cheap. Size resource.Quantity + // Reader selects the Block-mode reader mode; empty means ReaderPVC. + Reader ReaderMode +} + +// ReaderMode returns the configured reader mode with its default. +func (c Config) ReaderMode() ReaderMode { + if c.Reader == "" { + return ReaderPVC + } + return c.Reader } // Key is the short stable token for a model URI, used in object names @@ -297,6 +321,9 @@ type DownloadStep struct { NodeSelector map[string]string // VolumeName is the name the container mounts the claim under. VolumeName string + // Volumes are the other volumes the container mounts (secrets with + // registry keys, ConfigMaps with scripts), copied from the pod. + Volumes []corev1.Volume } // EnsureDownloadJob creates the one download Job for uri in ns, writing @@ -320,7 +347,34 @@ func (p *Provisioner) EnsureDownloadJob(ctx context.Context, uri, ns, claim stri labels := map[string]string{"app.kubernetes.io/managed-by": managedBy, IdentityLabel: Key(uri)} c := step.Container c.Name = "download" - c.VolumeMounts = []corev1.VolumeMount{{Name: step.VolumeName, MountPath: mountPathOf(step)}} + // The container keeps every mount the chart's init had (registry keys + // under /var/secrets, scripts from a ConfigMap); only the landing + // volume is redirected to the claim. + volumes := []corev1.Volume{{Name: step.VolumeName, VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: claim}}}} + for i := range step.Volumes { + if step.Volumes[i].Name != step.VolumeName { + volumes = append(volumes, step.Volumes[i]) + } + } + // Function namespaces enforce Kyverno baselines: requests and limits + // on every container, no service account token, a seccomp profile and + // no added capabilities. Resources are set only when the init had none. + if c.Resources.Limits == nil && c.Resources.Requests == nil { + c.Resources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("4Gi")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8"), corev1.ResourceMemory: resource.MustParse("16Gi")}, + } + } + if c.SecurityContext == nil { + c.SecurityContext = &corev1.SecurityContext{} + } + if c.SecurityContext.AllowPrivilegeEscalation == nil { + c.SecurityContext.AllowPrivilegeEscalation = new(bool) + } + if c.SecurityContext.Capabilities == nil { + c.SecurityContext.Capabilities = &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}} + } job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels, Annotations: map[string]string{IdentityAnnotation: uri}}, Spec: batchv1.JobSpec{ @@ -331,12 +385,12 @@ func (p *Provisioner) EnsureDownloadJob(ctx context.Context, uri, ns, claim stri Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, AutomountServiceAccountToken: new(bool), + SecurityContext: &corev1.PodSecurityContext{SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, ImagePullSecrets: step.ImagePullSecrets, Tolerations: step.Tolerations, NodeSelector: step.NodeSelector, Containers: []corev1.Container{c}, - Volumes: []corev1.Volume{{Name: step.VolumeName, VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: claim}}}}, + Volumes: volumes, }, }, }, @@ -347,15 +401,6 @@ func (p *Provisioner) EnsureDownloadJob(ctx context.Context, uri, ns, claim stri return name, nil } -func mountPathOf(step DownloadStep) string { - for _, vm := range step.Container.VolumeMounts { - if vm.Name == step.VolumeName { - return vm.MountPath - } - } - return "/models" -} - // JobSucceeded reports whether the download Job for uri in ns finished. func (p *Provisioner) JobSucceeded(ctx context.Context, uri, ns string) (bool, error) { job, err := p.Kube.BatchV1().Jobs(ns).Get(ctx, JobName(uri), metav1.GetOptions{}) diff --git a/src/compute-plane-services/nvsnap/internal/webhook/BUILD.bazel b/src/compute-plane-services/nvsnap/internal/webhook/BUILD.bazel index 8e5668d0cf..ae639bc6ae 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/BUILD.bazel +++ b/src/compute-plane-services/nvsnap/internal/webhook/BUILD.bazel @@ -59,6 +59,7 @@ go_test( "//src/compute-plane-services/nvsnap/internal/election", "//src/compute-plane-services/nvsnap/internal/modelvolume", "//src/compute-plane-services/nvsnap/internal/rootfsonly", + "@com_github_sirupsen_logrus//:logrus", "@io_k8s_api//admission/v1:admission", "@io_k8s_api//core/v1:core", "@io_k8s_apimachinery//pkg/api/resource", diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go index 16a68becd4..8178bcd167 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go @@ -93,21 +93,36 @@ func (m *Mutator) modelVolumePatches(ctx context.Context, pod *corev1.Pod) ([]Pa log.WithFields(logrus.Fields{"claim": claim, "job": job}).Info("model volume: download job ensured") } patches = append(patches, mp.label(modelvolume.RoleLabel, "reader")...) - switch m.ModelVolume.Cfg.Mode { - case modelvolume.ModeRWX: + switch { + case m.ModelVolume.Cfg.Mode == modelvolume.ModeRWX: claim, err := m.ModelVolume.EnsureWriterClaim(ctx, uri, pod.Namespace) if err != nil { return nil, err } patches = append(patches, m.substituteLandingVolume(pod, main, land, claim)...) log.WithFields(logrus.Fields{"claim": claim, "complete": st.Complete}).Info("model volume: reader on shared filesystem; waits for the marker") - default: - // Block mode: hostPath landing; the agent binds the completed - // read-only volume over it and the marker inside appears. + case m.ModelVolume.Cfg.ReaderMode() == modelvolume.ReaderHostPath: + // Block mode, hostPath landing: schedules at once; the agent binds + // the completed read-only volume over it and the marker appears. patches = append(patches, mp.label(modelvolume.PendingLabel, "true")...) patches = append(patches, mp.annotation(modelvolume.LandingAnnotation, landingMount(land))...) patches = append(patches, m.hostPathLanding(pod, main, land, uri)...) log.WithField("complete", st.Complete).Info("model volume: reader on block storage; agent binds the volume after completion") + default: + // Block mode, PVC reader: the pod references the read-only claim in + // its namespace. Complete already: mint it now so the pod binds at + // once. Not yet: the pod stays Pending on volume binding until the + // agent mints the claim after the download. No hostPath, no bind. + if st.Complete && m.ReadOnlyMinter != nil { + if err := m.ReadOnlyMinter.MintReadOnlyFromPV(ctx, st.PrimaryPV, modelvolume.ReadOnlyPVName(uri, pod.Namespace), modelvolume.ReadOnlyClaimName(uri), pod.Namespace, modelvolume.Key(uri)); err != nil { + return nil, fmt.Errorf("mint read-only claim: %w", err) + } + } + if !st.Complete { + patches = append(patches, mp.label(modelvolume.PendingLabel, "true")...) + } + patches = append(patches, m.substituteLandingVolume(pod, main, land, modelvolume.ReadOnlyClaimName(uri))...) + log.WithField("complete", st.Complete).Info("model volume: reader on block storage references the read-only claim") } patches = append(patches, m.downloadStepPatches(pod, main, land, res.Identity, false)...) patches = append(patches, m.modelCacheEnvPatches(pod, main, land, uri)...) @@ -130,9 +145,23 @@ func (m *Mutator) downloadStep(pod *corev1.Pod, main *corev1.Container, land mod c := *init.DeepCopy() c.Command = []string{"/bin/sh", "-c"} c.Args = []string{writerScript(orig, path.Join(mount, modelvolume.MarkerFile))} - c.VolumeMounts = []corev1.VolumeMount{{Name: landVolumeName(land), MountPath: mount}} - step.Container = c + // Keep every mount the init had; the landing one is redirected + // to the claim by name, the rest (secrets, scripts) come along. step.VolumeName = landVolumeName(land) + if land.VolumeName == "" { + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{Name: step.VolumeName, MountPath: mount}) + } + mounted := map[string]bool{} + for _, vm := range c.VolumeMounts { + mounted[vm.Name] = true + } + for i := range pod.Spec.Volumes { + v := &pod.Spec.Volumes[i] + if mounted[v.Name] && v.Name != step.VolumeName { + step.Volumes = append(step.Volumes, *v.DeepCopy()) + } + } + step.Container = c return step, true } return step, false diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go index d09b4f3fd4..8fc27fed1f 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go @@ -8,25 +8,36 @@ import ( "strings" "testing" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/checkpointstore" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/election" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/modelvolume" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/rootfsonly" ) func mvMutator(t *testing.T, mode modelvolume.Mode, role election.Role, kc *fake.Clientset) (*Mutator, *fakeElector) { + t.Helper() + m, el := mvMutatorReader(t, mode, modelvolume.ReaderHostPath, role, kc) + return m, el +} + +func mvMutatorReader(t *testing.T, mode modelvolume.Mode, reader modelvolume.ReaderMode, role election.Role, kc *fake.Clientset) (*Mutator, *fakeElector) { t.Helper() el := &fakeElector{role: role} + tx, _ := checkpointstore.LookupVolumeHandleTransform("nvmesh") return &Mutator{ - Backend: newBackend(t), - CacheDir: "/opt/nvsnap", - Composer: &rootfsonly.HashInputComposer{CUDADriverMajor: 580}, - Elector: el, - ModelVolume: &modelvolume.Provisioner{Kube: kc, Cfg: modelvolume.Config{Mode: mode, StorageClass: "sc", Size: resource.MustParse("512Gi")}}, + Backend: newBackend(t), + CacheDir: "/opt/nvsnap", + Composer: &rootfsonly.HashInputComposer{CUDADriverMajor: 580}, + Elector: el, + ModelVolume: &modelvolume.Provisioner{Kube: kc, Cfg: modelvolume.Config{Mode: mode, StorageClass: "sc", Size: resource.MustParse("512Gi"), Reader: reader}}, + ReadOnlyMinter: &checkpointstore.SharedVolumePromoter{KubeClient: kc, StorageClass: "sc", Transform: tx, Log: logrus.New()}, }, el } @@ -41,7 +52,7 @@ func ngcFunctionPod() *corev1.Pod { Name: "download-ngc-model", Image: "nvcr.io/org/ultra:vllm", Command: []string{"/bin/bash", "-c"}, Args: []string{"set -euo pipefail\nngc registry model download-version --dest \"${NGC_MODEL_MOUNT}\" \"${NGC_MODEL_NAME}\"\n"}, Env: []corev1.EnvVar{{Name: "NGC_MODEL_NAME", Value: "org/team/nemotron3-ultra-genrm:bf16-fixed"}, {Name: "NGC_MODEL_MOUNT", Value: "/config/models"}}, - VolumeMounts: []corev1.VolumeMount{{Name: "ngc-models", MountPath: "/config/models"}}, + VolumeMounts: []corev1.VolumeMount{{Name: "ngc-models", MountPath: "/config/models"}, {Name: "secrets", MountPath: "/var/secrets", ReadOnly: true}, {Name: "scripts", MountPath: "/opt/kimi-k3"}}, }}, Containers: []corev1.Container{{ Name: "kimi-k3", Image: "nvcr.io/org/ultra:vllm", Command: []string{"/bin/bash", "/opt/kimi-k3/start.sh"}, @@ -52,6 +63,8 @@ func ngcFunctionPod() *corev1.Pod { Volumes: []corev1.Volume{ {Name: "dshm", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "ngc-models", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "secrets", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "function-secrets"}}}, + {Name: "scripts", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "kimi-k3-scripts"}}}}, }, }, } @@ -146,6 +159,24 @@ func TestModelVolume_FirstPodBlock_NGCInit_CreatesJob(t *testing.T) { if len(jc.Env) != 2 || jc.VolumeMounts[0].MountPath != "/config/models" || job.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName != modelvolume.ClaimName(uri) { t.Errorf("job must carry the init's env and mount the claim at the init's path: env=%v mounts=%v", jc.Env, jc.VolumeMounts) } + // The NGC key lives in a secret file and the init's helper scripts in + // a ConfigMap; the Job must mount both or the download cannot run. + jobVols := map[string]corev1.Volume{} + for _, vol := range job.Spec.Template.Spec.Volumes { + jobVols[vol.Name] = vol + } + if len(jc.VolumeMounts) != 3 || jobVols["secrets"].Secret == nil || jobVols["secrets"].Secret.SecretName != "function-secrets" || jobVols["scripts"].ConfigMap == nil { + t.Errorf("job must carry every volume the init mounted, only the landing one redirected: mounts=%v volumes=%v", jc.VolumeMounts, jobVols) + } + if _, dshm := jobVols["dshm"]; dshm || len(jobVols) != 3 { + t.Errorf("volumes the init does not mount stay behind: %v", jobVols) + } + ps := job.Spec.Template.Spec + if jc.Resources.Requests.Cpu().IsZero() || jc.Resources.Limits.Memory().IsZero() || ps.SecurityContext == nil || ps.SecurityContext.SeccompProfile == nil || + jc.SecurityContext == nil || jc.SecurityContext.AllowPrivilegeEscalation == nil || *jc.SecurityContext.AllowPrivilegeEscalation || jc.SecurityContext.Capabilities == nil || + ps.AutomountServiceAccountToken == nil || *ps.AutomountServiceAccountToken { + t.Errorf("job pod must satisfy the function-namespace baselines (requests/limits, seccomp, no escalation, dropped caps, no SA token): %+v %+v", jc.Resources, ps.SecurityContext) + } s := v.initScripts["download-ngc-model"] if !strings.Contains(s, "while [ ! -f /config/models/.nvsnap-complete ]") { t.Errorf("the pod's own init becomes a wait:\n%s", s) @@ -191,6 +222,79 @@ func TestModelVolume_ReaderBlock_PendingBind(t *testing.T) { } } +// Default reader mode on block storage: no hostPath (Kyverno's +// disallow-host-path rejects it in function namespaces); the pod +// references the read-only claim in its namespace and waits on binding. +func TestModelVolume_ReaderBlockPVC_ReferencesReadOnlyClaim(t *testing.T) { + kc := fake.NewSimpleClientset() + m, _ := mvMutatorReader(t, modelvolume.ModeBlock, "", election.RoleFollower, kc) + pod := ngcFunctionPod() + patches, err := m.Mutate(context.Background(), pod) + if err != nil { + t.Fatal(err) + } + v := viewMV(pod, patches) + uri := "ngc://org/team/nemotron3-ultra-genrm:bf16-fixed" + if v.labels[modelvolume.RoleLabel] != "reader" || v.labels[modelvolume.PendingLabel] != "true" { + t.Errorf("PVC reader is pending until the agent mints its claim: %v", v.labels) + } + vol, replaced := v.volumes["ngc-models"] + if !replaced || vol.PersistentVolumeClaim == nil || vol.PersistentVolumeClaim.ClaimName != modelvolume.ReadOnlyClaimName(uri) { + t.Errorf("landing volume becomes the read-only claim, got %+v", vol) + } + for _, p := range patches { + if strings.HasSuffix(p.Path, "/mountPropagation") { + t.Error("no mount propagation in PVC mode") + } + if vv, ok := p.Value.(corev1.Volume); ok && vv.HostPath != nil { + t.Error("no hostPath in PVC mode") + } + } + if _, err := kc.CoreV1().PersistentVolumeClaims("sr-fn").Get(context.Background(), modelvolume.ReadOnlyClaimName(uri), metav1.GetOptions{}); err == nil { + t.Error("the read-only claim is minted by the agent after completion, not at admission of an incomplete volume") + } + if s := v.initScripts["download-ngc-model"]; !strings.Contains(s, "while [ ! -f /config/models/.nvsnap-complete ]") { + t.Errorf("reader init still waits for the marker on the bound claim:\n%s", s) + } +} + +// Volume already complete: the webhook mints the read-only claim in the +// pod namespace itself so the pod binds immediately, and it is not pending. +func TestModelVolume_ReaderBlockPVC_CompleteMintsAtAdmission(t *testing.T) { + uri := "ngc://org/team/nemotron3-ultra-genrm:bf16-fixed" + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "pvc-done", Labels: map[string]string{modelvolume.IdentityLabel: modelvolume.Key(uri), modelvolume.CompleteLabel: "true", "app.kubernetes.io/managed-by": "nvsnap"}}, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("512Gi")}, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + PersistentVolumeSource: corev1.PersistentVolumeSource{CSI: &corev1.CSIPersistentVolumeSource{Driver: "nvmesh-csi.excelero.com", VolumeHandle: "cluster:csi-done:vol:sr-fn"}}, + }, + } + kc := fake.NewSimpleClientset(pv) + m, _ := mvMutatorReader(t, modelvolume.ModeBlock, modelvolume.ReaderPVC, election.RoleFollower, kc) + pod := ngcFunctionPod() + pod.Namespace = "sr-other" + patches, err := m.Mutate(context.Background(), pod) + if err != nil { + t.Fatal(err) + } + v := viewMV(pod, patches) + if _, pending := v.labels[modelvolume.PendingLabel]; pending { + t.Errorf("complete volume: the reader is not pending, labels %v", v.labels) + } + ro, err := kc.CoreV1().PersistentVolumeClaims("sr-other").Get(context.Background(), modelvolume.ReadOnlyClaimName(uri), metav1.GetOptions{}) + if err != nil { + t.Fatalf("read-only claim must be minted in the pod namespace at admission: %v", err) + } + roPV, _ := kc.CoreV1().PersistentVolumes().Get(context.Background(), ro.Spec.VolumeName, metav1.GetOptions{}) + if roPV.Spec.CSI.VolumeHandle != "cluster:csi-done:vol:sr-other" { + t.Errorf("handle rewritten to the reader namespace: %+v", roPV.Spec.CSI) + } + if _, err := kc.BatchV1().Jobs("sr-other").Get(context.Background(), modelvolume.JobName(uri), metav1.GetOptions{}); err == nil { + t.Error("no download Job for a complete volume") + } +} + func TestModelVolume_ReaderRWX_SharesClaim(t *testing.T) { kc := fake.NewSimpleClientset() m, _ := mvMutator(t, modelvolume.ModeRWX, election.RoleFollower, kc) diff --git a/src/compute-plane-services/nvsnap/internal/webhook/mutate.go b/src/compute-plane-services/nvsnap/internal/webhook/mutate.go index 018904854e..92625e06cf 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/mutate.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/mutate.go @@ -268,10 +268,14 @@ type Mutator struct { ModelVolume *modelvolume.Provisioner Groups modelid.GroupResolver ModelWaitDeadline time.Duration - // ModelHostRoot is the host directory (under the agent's Bidirectional - // overlays root) where Block-mode readers get their hostPath and the - // agent binds completed model volumes: /. + // ModelHostRoot is the host directory where hostPath-mode readers get + // their landing and the agent binds completed model volumes: + // /. ModelHostRoot string + // ReadOnlyMinter mints the read-only claim for a completed Block-mode + // volume in the admitted pod's namespace (PVC reader mode); nil in RWX + // mode. + ReadOnlyMinter ReadOnlyMinter // L2WaitImage is the nvsnap-l2-wait init-container image ref // (nvsnap#147). When non-empty, tryL2Mount prepends a @@ -1119,3 +1123,9 @@ func (m *Mutator) logger() logrus.FieldLogger { } return logrus.NewEntry(logrus.New()).WithField("subsys", "webhook.mutate") } + +// ReadOnlyMinter exposes a completed model volume as a read-only claim in +// a namespace (checkpointstore.SharedVolumePromoter implements it). +type ReadOnlyMinter interface { + MintReadOnlyFromPV(ctx context.Context, primaryPV, roPVName, roClaim, ns, labelKey string) error +} From b0c319dfa0b038f480a952e36f5d9d4692840bf9 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 14:34:54 -0700 Subject: [PATCH 13/16] fix(nvsnap): harden the injected model init and the download Job alike The engine-download Job container had requests without limits and the injected reader init had neither, so both failed require-pod-requests-limits where Kyverno is enforced (seen on dev1). One helper now gives every container the model volume creates the same baseline: requests and limits unless the chart sized it, no privilege escalation, dropped capabilities. Mount propagation on the injected init is set only in hostPath reader mode, the one case where the host binds a volume in after the pod started. Co-Authored-By: Balaji Ganesan --- .../internal/modelvolume/modelvolume.go | 48 ++++++++++++------- .../nvsnap/internal/webhook/model_volume.go | 24 +++++++--- .../internal/webhook/model_volume_test.go | 34 +++++++++++++ 3 files changed, 81 insertions(+), 25 deletions(-) diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go index bfc266e055..8cf5fdbf75 100644 --- a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go @@ -357,24 +357,7 @@ func (p *Provisioner) EnsureDownloadJob(ctx context.Context, uri, ns, claim stri volumes = append(volumes, step.Volumes[i]) } } - // Function namespaces enforce Kyverno baselines: requests and limits - // on every container, no service account token, a seccomp profile and - // no added capabilities. Resources are set only when the init had none. - if c.Resources.Limits == nil && c.Resources.Requests == nil { - c.Resources = corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("4Gi")}, - Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8"), corev1.ResourceMemory: resource.MustParse("16Gi")}, - } - } - if c.SecurityContext == nil { - c.SecurityContext = &corev1.SecurityContext{} - } - if c.SecurityContext.AllowPrivilegeEscalation == nil { - c.SecurityContext.AllowPrivilegeEscalation = new(bool) - } - if c.SecurityContext.Capabilities == nil { - c.SecurityContext.Capabilities = &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}} - } + Harden(&c, DownloadResources) job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels, Annotations: map[string]string{IdentityAnnotation: uri}}, Spec: batchv1.JobSpec{ @@ -412,3 +395,32 @@ func (p *Provisioner) JobSucceeded(ctx context.Context, uri, ns string) (bool, e } return job.Status.Succeeded > 0, nil } + +// DownloadResources are the defaults for a container that downloads a +// model: enough CPU and memory for a parallel fetch, bounded for policy. +var DownloadResources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("4Gi")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8"), corev1.ResourceMemory: resource.MustParse("16Gi")}, +} + +// Harden gives a container the fields function-namespace baselines +// require (Kyverno on NVCF clusters: requests and limits on every +// container, no privilege escalation, dropped capabilities). Fields the +// chart already set are kept; resources are defaulted only when both +// requests and limits are absent, so a chart's own sizing wins. +func Harden(c *corev1.Container, resources corev1.ResourceRequirements) { + if c.Resources.Limits == nil && c.Resources.Requests == nil { + c.Resources = *resources.DeepCopy() + } else if c.Resources.Limits == nil { + c.Resources.Limits = resources.Limits.DeepCopy() + } + if c.SecurityContext == nil { + c.SecurityContext = &corev1.SecurityContext{} + } + if c.SecurityContext.AllowPrivilegeEscalation == nil { + c.SecurityContext.AllowPrivilegeEscalation = new(bool) + } + if c.SecurityContext.Capabilities == nil { + c.SecurityContext.Capabilities = &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}} + } +} diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go index 8178bcd167..12ca2e6cdd 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go @@ -11,7 +11,6 @@ import ( "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/checkpointstore" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvsnap/internal/election" @@ -176,9 +175,6 @@ func (m *Mutator) downloadStep(pod *corev1.Pod, main *corev1.Container, land mod Args: []string{writerScript(hfDownloadCommand(id), path.Join(land.Path, modelvolume.MarkerFile))}, Env: append([]corev1.EnvVar{{Name: "HF_HOME", Value: land.Path}}, tokenEnv(main)...), VolumeMounts: []corev1.VolumeMount{{Name: step.VolumeName, MountPath: land.Path}}, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("4Gi")}, - }, } return step, true } @@ -346,14 +342,22 @@ func (m *Mutator) downloadStepPatches(pod *corev1.Pod, main *corev1.Container, l Args: []string{script}, Env: append([]corev1.EnvVar{{Name: "HF_HOME", Value: land.Path}}, tokenEnv(main)...), } - prop := corev1.MountPropagationHostToContainer + // The init mostly waits; the fallback download is the one case that + // needs real resources, and policy needs limits either way. + modelvolume.Harden(&init, modelvolume.DownloadResources) + // Only a hostPath landing receives a bind from the host after start. + var prop *corev1.MountPropagationMode + if m.hostPathReaders() { + p := corev1.MountPropagationHostToContainer + prop = &p + } for _, vm := range main.VolumeMounts { if vm.Name == land.VolumeName || vm.Name == modelVolumeName { - init.VolumeMounts = append(init.VolumeMounts, corev1.VolumeMount{Name: vm.Name, MountPath: vm.MountPath, MountPropagation: &prop}) + init.VolumeMounts = append(init.VolumeMounts, corev1.VolumeMount{Name: vm.Name, MountPath: vm.MountPath, MountPropagation: prop}) } } if len(init.VolumeMounts) == 0 { - init.VolumeMounts = []corev1.VolumeMount{{Name: modelVolumeName, MountPath: land.Path, MountPropagation: &prop}} + init.VolumeMounts = []corev1.VolumeMount{{Name: modelVolumeName, MountPath: land.Path, MountPropagation: prop}} } patches = append(patches, PatchOp{Op: "add", Path: "/spec/initContainers/0", Value: init}) if id.Scheme == "hf" { @@ -499,3 +503,9 @@ func hfDownloadCommand(id modelid.Identity) string { } return fmt.Sprintf("if command -v hf >/dev/null 2>&1; then hf download %[1]s; else huggingface-cli download %[1]s; fi", args) } + +// hostPathReaders reports whether Block-mode readers land on a hostPath +// the agent binds into (as opposed to referencing the read-only claim). +func (m *Mutator) hostPathReaders() bool { + return m.ModelVolume != nil && m.ModelVolume.Cfg.Mode != modelvolume.ModeRWX && m.ModelVolume.Cfg.ReaderMode() == modelvolume.ReaderHostPath +} diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go index 8fc27fed1f..218a8a21aa 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go @@ -260,6 +260,40 @@ func TestModelVolume_ReaderBlockPVC_ReferencesReadOnlyClaim(t *testing.T) { // Volume already complete: the webhook mints the read-only claim in the // pod namespace itself so the pod binds immediately, and it is not pending. +// Engine-download chart in PVC mode: the injected init carries no +// mount propagation (nothing arrives from the host) and satisfies the +// baselines; the Job container gets requests and limits. +func TestModelVolume_ReaderBlockPVC_InjectedInitHardened(t *testing.T) { + kc := fake.NewSimpleClientset() + m, _ := mvMutatorReader(t, modelvolume.ModeBlock, modelvolume.ReaderPVC, election.RoleFollower, kc) + pod := stockVLLMPod() + patches, err := m.Mutate(context.Background(), pod) + if err != nil { + t.Fatal(err) + } + v := viewMV(pod, patches) + if len(v.newInits) != 1 { + t.Fatalf("one injected init expected, got %d", len(v.newInits)) + } + init := v.newInits[0] + for _, vm := range init.VolumeMounts { + if vm.MountPropagation != nil { + t.Errorf("no mount propagation in PVC mode: %+v", vm) + } + } + if init.Resources.Limits.Memory().IsZero() || init.SecurityContext == nil || init.SecurityContext.Capabilities == nil { + t.Errorf("injected init must carry limits and a security context: %+v %+v", init.Resources, init.SecurityContext) + } + uri := "hf://Qwen/Qwen2.5-32B-Instruct" + job, err := kc.BatchV1().Jobs(pod.Namespace).Get(context.Background(), modelvolume.JobName(uri), metav1.GetOptions{}) + if err != nil { + t.Fatalf("download Job: %v", err) + } + if jc := job.Spec.Template.Spec.Containers[0]; jc.Resources.Limits.Cpu().IsZero() || jc.Resources.Requests.Memory().IsZero() { + t.Errorf("engine-download Job container needs requests and limits: %+v", jc.Resources) + } +} + func TestModelVolume_ReaderBlockPVC_CompleteMintsAtAdmission(t *testing.T) { uri := "ngc://org/team/nemotron3-ultra-genrm:bf16-fixed" pv := &corev1.PersistentVolume{ From ec96abd09ecf5dfa2062c6c51952ce7b8e4cbd18 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 14:48:37 -0700 Subject: [PATCH 14/16] fix(nvsnap): inherit the chart's user posture on model download containers The injected init and the download Job still failed require-run-as-non-root and the by-name CAP_NET_RAW check on dev1. The download runs the engine's image and must own what it writes, so both containers now inherit runAsNonRoot, runAsUser, runAsGroup and seccomp from the chart's main container, and the Job pod inherits the pod security context (fsGroup). NET_RAW is dropped by name next to ALL. Co-Authored-By: Balaji Ganesan --- .../internal/modelvolume/modelvolume.go | 54 +++++++++++++++---- .../nvsnap/internal/webhook/model_volume.go | 7 ++- .../internal/webhook/model_volume_test.go | 40 ++++++++++++++ 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go index 8cf5fdbf75..4c69faffb1 100644 --- a/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go +++ b/src/compute-plane-services/nvsnap/internal/modelvolume/modelvolume.go @@ -324,6 +324,12 @@ type DownloadStep struct { // Volumes are the other volumes the container mounts (secrets with // registry keys, ConfigMaps with scripts), copied from the pod. Volumes []corev1.Volume + // PodSecurityContext is the source pod's, so the Job writes the volume + // with the same user, groups and fsGroup the readers will use. + PodSecurityContext *corev1.PodSecurityContext + // MainSecurityContext is the source pod's engine container posture, + // inherited by the download container. + MainSecurityContext *corev1.SecurityContext } // EnsureDownloadJob creates the one download Job for uri in ns, writing @@ -357,7 +363,14 @@ func (p *Provisioner) EnsureDownloadJob(ctx context.Context, uri, ns, claim stri volumes = append(volumes, step.Volumes[i]) } } - Harden(&c, DownloadResources) + Harden(&c, DownloadResources, step.MainSecurityContext) + psc := &corev1.PodSecurityContext{} + if step.PodSecurityContext != nil { + psc = step.PodSecurityContext.DeepCopy() + } + if psc.SeccompProfile == nil { + psc.SeccompProfile = &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault} + } job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels, Annotations: map[string]string{IdentityAnnotation: uri}}, Spec: batchv1.JobSpec{ @@ -368,7 +381,7 @@ func (p *Provisioner) EnsureDownloadJob(ctx context.Context, uri, ns, claim stri Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, AutomountServiceAccountToken: new(bool), - SecurityContext: &corev1.PodSecurityContext{SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, + SecurityContext: psc, ImagePullSecrets: step.ImagePullSecrets, Tolerations: step.Tolerations, NodeSelector: step.NodeSelector, @@ -405,10 +418,15 @@ var DownloadResources = corev1.ResourceRequirements{ // Harden gives a container the fields function-namespace baselines // require (Kyverno on NVCF clusters: requests and limits on every -// container, no privilege escalation, dropped capabilities). Fields the -// chart already set are kept; resources are defaulted only when both -// requests and limits are absent, so a chart's own sizing wins. -func Harden(c *corev1.Container, resources corev1.ResourceRequirements) { +// container, no privilege escalation, dropped capabilities with NET_RAW +// named because one rule checks for it by name). Fields the chart +// already set are kept; resources are defaulted only when both requests +// and limits are absent, so a chart's own sizing wins. The user posture +// (runAsNonRoot, runAsUser, runAsGroup, seccomp) is inherited from the +// chart's main container when given: the download runs the same image +// as the engine and must own what it writes, so it runs as the same +// user the engine will read as. +func Harden(c *corev1.Container, resources corev1.ResourceRequirements, from *corev1.SecurityContext) { if c.Resources.Limits == nil && c.Resources.Requests == nil { c.Resources = *resources.DeepCopy() } else if c.Resources.Limits == nil { @@ -417,10 +435,26 @@ func Harden(c *corev1.Container, resources corev1.ResourceRequirements) { if c.SecurityContext == nil { c.SecurityContext = &corev1.SecurityContext{} } - if c.SecurityContext.AllowPrivilegeEscalation == nil { - c.SecurityContext.AllowPrivilegeEscalation = new(bool) + sc := c.SecurityContext + if sc.AllowPrivilegeEscalation == nil { + sc.AllowPrivilegeEscalation = new(bool) + } + if sc.Capabilities == nil { + sc.Capabilities = &corev1.Capabilities{Drop: []corev1.Capability{"ALL", "NET_RAW"}} + } + if from == nil { + return + } + if sc.RunAsNonRoot == nil && from.RunAsNonRoot != nil { + sc.RunAsNonRoot = from.RunAsNonRoot + } + if sc.RunAsUser == nil && from.RunAsUser != nil { + sc.RunAsUser = from.RunAsUser + } + if sc.RunAsGroup == nil && from.RunAsGroup != nil { + sc.RunAsGroup = from.RunAsGroup } - if c.SecurityContext.Capabilities == nil { - c.SecurityContext.Capabilities = &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}} + if sc.SeccompProfile == nil && from.SeccompProfile != nil { + sc.SeccompProfile = from.SeccompProfile.DeepCopy() } } diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go index 12ca2e6cdd..08ce54f28b 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume.go @@ -132,7 +132,10 @@ func (m *Mutator) modelVolumePatches(ctx context.Context, pod *corev1.Pod) ([]Pa // download init wrapped to touch the marker, or `hf download` on the // engine image when the engine fetches the model itself. func (m *Mutator) downloadStep(pod *corev1.Pod, main *corev1.Container, land modelid.Landing, id modelid.Identity) (modelvolume.DownloadStep, bool) { - step := modelvolume.DownloadStep{ImagePullSecrets: pod.Spec.ImagePullSecrets, Tolerations: pod.Spec.Tolerations, NodeSelector: pod.Spec.NodeSelector} + step := modelvolume.DownloadStep{ + ImagePullSecrets: pod.Spec.ImagePullSecrets, Tolerations: pod.Spec.Tolerations, NodeSelector: pod.Spec.NodeSelector, + PodSecurityContext: pod.Spec.SecurityContext, MainSecurityContext: main.SecurityContext, + } if land.Downloader == modelid.DownloaderInit { for i := range pod.Spec.InitContainers { init := pod.Spec.InitContainers[i] @@ -344,7 +347,7 @@ func (m *Mutator) downloadStepPatches(pod *corev1.Pod, main *corev1.Container, l } // The init mostly waits; the fallback download is the one case that // needs real resources, and policy needs limits either way. - modelvolume.Harden(&init, modelvolume.DownloadResources) + modelvolume.Harden(&init, modelvolume.DownloadResources, main.SecurityContext) // Only a hostPath landing receives a bind from the host after start. var prop *corev1.MountPropagationMode if m.hostPathReaders() { diff --git a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go index 218a8a21aa..af8edb6ba9 100644 --- a/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go +++ b/src/compute-plane-services/nvsnap/internal/webhook/model_volume_test.go @@ -294,6 +294,46 @@ func TestModelVolume_ReaderBlockPVC_InjectedInitHardened(t *testing.T) { } } +// The chart runs its engine as a fixed non-root user with an fsGroup. +// The download Job and the injected init inherit that posture: the +// volume is written by the user that reads it, and runAsNonRoot holds. +func TestModelVolume_InheritsChartUserPosture(t *testing.T) { + kc := fake.NewSimpleClientset() + m, _ := mvMutatorReader(t, modelvolume.ModeBlock, modelvolume.ReaderPVC, election.RoleFollower, kc) + pod := stockVLLMPod() + uid, gid, nonRoot := int64(1000), int64(2000), true + pod.Spec.SecurityContext = &corev1.PodSecurityContext{FSGroup: &gid} + pod.Spec.Containers[0].SecurityContext = &corev1.SecurityContext{RunAsNonRoot: &nonRoot, RunAsUser: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}} + patches, err := m.Mutate(context.Background(), pod) + if err != nil { + t.Fatal(err) + } + v := viewMV(pod, patches) + isc := v.newInits[0].SecurityContext + if isc.RunAsNonRoot == nil || !*isc.RunAsNonRoot || isc.RunAsUser == nil || *isc.RunAsUser != uid || isc.SeccompProfile == nil { + t.Errorf("injected init inherits the engine's user posture: %+v", isc) + } + job, err := kc.BatchV1().Jobs(pod.Namespace).Get(context.Background(), modelvolume.JobName("hf://Qwen/Qwen2.5-32B-Instruct"), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + ps := job.Spec.Template.Spec + jsc := ps.Containers[0].SecurityContext + if jsc.RunAsNonRoot == nil || !*jsc.RunAsNonRoot || jsc.RunAsUser == nil || *jsc.RunAsUser != uid { + t.Errorf("Job container inherits the engine's user posture: %+v", jsc) + } + if ps.SecurityContext == nil || ps.SecurityContext.FSGroup == nil || *ps.SecurityContext.FSGroup != gid || ps.SecurityContext.SeccompProfile == nil { + t.Errorf("Job pod inherits the pod security context plus seccomp: %+v", ps.SecurityContext) + } + var drops []corev1.Capability + if jsc.Capabilities != nil { + drops = jsc.Capabilities.Drop + } + if len(drops) != 2 || drops[1] != "NET_RAW" { + t.Errorf("NET_RAW is dropped by name: %v", drops) + } +} + func TestModelVolume_ReaderBlockPVC_CompleteMintsAtAdmission(t *testing.T) { uri := "ngc://org/team/nemotron3-ultra-genrm:bf16-fixed" pv := &corev1.PersistentVolume{ From 3831eb96d97ba243105c31e2ae69549263c2533e Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 14:49:13 -0700 Subject: [PATCH 15/16] docs(nvsnap): record the PVC reader mode results from dev1 Co-Authored-By: Balaji Ganesan --- .../proposals/helm-shared-model-volume.md | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md index 78d524c5a7..7b0f8e7d4f 100644 --- a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md +++ b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md @@ -207,6 +207,28 @@ earlier run, identity already complete on the cluster t+205s both Ready, wait init 10-15 s, engine reads the volume directly ``` +PVC reader mode (the default since the function-namespace fixes), same +chart, Qwen2.5-14B-Instruct TP=4, replicas=2, agent v0.2.76-mv6: + +``` +first deploy (fresh identity) + t+0 both pods admitted as readers referencing nvsnap-model--ro; Pending on + "persistentvolumeclaim not found"; one Job created (2 CPU / 4 Gi requests, + no SA token, seccomp, dropped capabilities) + t+135s Job succeeded: 28 GB via `hf download` + t+195s read-only PV and claim minted in the pod namespace after the primary + detached; both readers un-pended and scheduled; init found the marker + t+344s both Ready; 0 downloads; /root/.cache/huggingface is the NVMesh volume + mounted ro,norecovery,nouuid; serve " Paris. The capital" +uninstall + reinstall + t+118s both Ready (claim minted at admission, no pending phase) +``` + +No hostPath and no agent bind in this mode. Kyverno on dev1 is Audit, so +the remaining warnings were the stock chart's own containers plus, until +v0.2.76-mv8, the injected init (no resources, capabilities not dropped, +no runAsNonRoot); the enforced rejection itself is not testable on dev1. + Cold start of the same pod on the same node: 325 s. The reinstall number is the engine's own load and compile from a read-only NVMesh mount with no prewarm; compile caches on block storage are still the follow-up (step 4). From 9db38149562f7ad5d6b2c04e492f3ab3084be47f Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Sat, 26 Sep 2026 19:18:34 -0700 Subject: [PATCH 16/16] docs(nvsnap): record the real NVCF Helm function run on dev1 Co-Authored-By: Balaji Ganesan --- .../proposals/helm-shared-model-volume.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md index 7b0f8e7d4f..f22a329739 100644 --- a/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md +++ b/src/compute-plane-services/nvsnap/docs/proposals/helm-shared-model-volume.md @@ -224,6 +224,36 @@ uninstall + reinstall t+118s both Ready (claim minted at admission, no pending phase) ``` +Real NVCF Helm function, 2026-09-27 (dev1 through the staging control +plane, QA org; stock `kimi-k3` 0.2.3 chart copied from the prod org with +values only: init-container NGC download into an emptyDir, two +StatefulSet replicas, Qwen2.5-0.5B-Instruct hosted on prod NGC, agent +v0.2.76-mv8, PVC reader mode). No chart change and no nvsnap marker. + +``` +first deploy (fresh identity, function namespace sr-848a0bc4-...) + t+0 both mini-service pods admitted as readers referencing the ro claim; one Job; + NVCA's own webhook mutated the Job pod too, so /var/secrets/secrets.json with + the NGC key was present and the chart's download script ran unchanged + t+150s Job succeeded (NGC CLI download); claim released + t+152s ro claim minted in the function namespace; both readers scheduled; all three + inits (two NVCA cert inits, the wrapped download-ngc-model) exited 0 + engine crashed on my override (TP=4 does not divide the model's 14 heads); NVCF + marked the function ERROR and deleted the namespace; the primary volume stayed +redeploy (fixed override; identity already complete on the cluster) + t+51s namespace sr-4d3e0de5-... created; both readers admitted complete=true, ro claim + minted at admission (no Job) + t+153s both Ready; init logs "model complete, skipping download"; 0 NGC downloads; + /config/models is the NVMesh volume (ro,norecovery,nouuid); chat completion + answered; function ACTIVE +``` + +Kyverno on dev1 is Audit; the audit failures on the function pods belong to +the chart (IPC_LOCK add, no capability drop, root) and to NVCA's cert inits, +not to the containers nvsnap creates. The namespace deletion left the first +run's read-only PV Released: retention must also cover PVs whose claim +namespace is gone. + No hostPath and no agent bind in this mode. Kyverno on dev1 is Audit, so the remaining warnings were the stock chart's own containers plus, until v0.2.76-mv8, the injected init (no resources, capabilities not dropped,