From 49472b4c03418414df963b30fd3226c9cd4a50f4 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Wed, 4 Nov 2026 19:28:56 -0800 Subject: [PATCH 1/3] Configure the controllers' Reconcilers through Setup's options Each controller's Setup took a logger positionally and built its Reconciler's options itself, so configuring a Reconciler any other way meant adding another positional argument to Setup. Setup now takes the same ReconcilerOptions NewReconciler does, and passes them through. Signed-off-by: Nic Cope --- cmd/cluster-agent/main.go | 8 ++++++-- internal/controller/poolobservation/reconciler.go | 7 ++++--- internal/controller/replicaobservation/reconciler.go | 7 ++++--- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/cmd/cluster-agent/main.go b/cmd/cluster-agent/main.go index 3e5f060..18eb691 100644 --- a/cmd/cluster-agent/main.go +++ b/cmd/cluster-agent/main.go @@ -119,8 +119,12 @@ func main() { _, err = claims.AddEventHandler(allocation.ForgetDeleted(record)) kctx.FatalIfErrorf(err, "cannot add ResourceClaim event handler") - kctx.FatalIfErrorf(poolobservation.Setup(mgr, record, log, flags.ObservationDelay), "cannot set up PoolObservation controller") - kctx.FatalIfErrorf(replicaobservation.Setup(mgr, record, log), "cannot set up ReplicaObservation controller") + kctx.FatalIfErrorf(poolobservation.Setup(mgr, record, flags.ObservationDelay, + poolobservation.WithLogger(log.With("controller", "poolobservation")), + ), "cannot set up PoolObservation controller") + kctx.FatalIfErrorf(replicaobservation.Setup(mgr, record, + replicaobservation.WithLogger(log.With("controller", "replicaobservation")), + ), "cannot set up ReplicaObservation controller") kctx.FatalIfErrorf(mgr.Start(ctx), "cannot run manager") } diff --git a/internal/controller/poolobservation/reconciler.go b/internal/controller/poolobservation/reconciler.go index 5850cbc..3e078d3 100644 --- a/internal/controller/poolobservation/reconciler.go +++ b/internal/controller/poolobservation/reconciler.go @@ -112,8 +112,9 @@ func NewReconciler(c client.Client, record *allocation.Record, opts ...Reconcile // Setup adds a PoolObservation controller to the manager. The controller // observes every pool whenever an object it reads changes, but waits for the // supplied delay first, so that a burst of changes produces one observation -// of each pool. It reads ResourceClaims through the supplied Record. -func Setup(mgr ctrl.Manager, record *allocation.Record, log *slog.Logger, delay time.Duration) error { +// of each pool. It reads ResourceClaims through the supplied Record, and +// configures its Reconciler with the supplied options. +func Setup(mgr ctrl.Manager, record *allocation.Record, delay time.Duration, opts ...ReconcilerOption) error { c := mgr.GetClient() enqueue := EnqueueAllAfter(c, delay) err := ctrl.NewControllerManagedBy(mgr). @@ -125,7 +126,7 @@ func Setup(mgr ctrl.Manager, record *allocation.Record, log *slog.Logger, delay Watches(&resourcev1.ResourceClaim{}, enqueue). Watches(&corev1.Pod{}, enqueue). Watches(&corev1.Event{}, enqueue). - Complete(NewReconciler(c, record, WithLogger(log.With("controller", "poolobservation")))) + Complete(NewReconciler(c, record, opts...)) if err != nil { return fmt.Errorf("cannot create controller: %w", err) } diff --git a/internal/controller/replicaobservation/reconciler.go b/internal/controller/replicaobservation/reconciler.go index dfdb7bc..0fdab08 100644 --- a/internal/controller/replicaobservation/reconciler.go +++ b/internal/controller/replicaobservation/reconciler.go @@ -111,8 +111,9 @@ func NewReconciler(c client.Client, record *allocation.Record, opts ...Reconcile } // Setup adds a ReplicaObservation controller to the manager. It reads -// ResourceClaims through the supplied Record. -func Setup(mgr ctrl.Manager, record *allocation.Record, log *slog.Logger) error { +// ResourceClaims through the supplied Record, and configures its Reconciler +// with the supplied options. +func Setup(mgr ctrl.Manager, record *allocation.Record, opts ...ReconcilerOption) error { c := mgr.GetClient() err := ctrl.NewControllerManagedBy(mgr). Named("replicaobservation"). @@ -126,7 +127,7 @@ func Setup(mgr ctrl.Manager, record *allocation.Record, log *slog.Logger) error // but the claim cache can lag the pod cache. Watching claims catches // an allocation the pod's binding raced ahead of. Watches(&resourcev1.ResourceClaim{}, handler.EnqueueRequestsFromMapFunc(ClaimRequests(c))). - Complete(NewReconciler(c, record, WithLogger(log.With("controller", "replicaobservation")))) + Complete(NewReconciler(c, record, opts...)) if err != nil { return fmt.Errorf("cannot create controller: %w", err) } From 0f96d5591929ed572f28105cb0be5651b1dafdfb Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Wed, 4 Nov 2026 19:32:09 -0800 Subject: [PATCH 2/3] Tolerate the AMD GPU and CPU pool taints when counting schedulable nodes The agent counted a node as schedulable only if every NoSchedule or NoExecute taint on it was nvidia.com/gpu with NoSchedule. Modelplane's engine pods also tolerate amd.com/gpu and modelplane.ai/cpu, so the agent would report AMD GPU and CPU pools as having no schedulable nodes. To avoid that Modelplane only asks the agent to observe NVIDIA pools, and falls back to whole-node scheduling on the others (modelplaneai/modelplane#601). The agent now tolerates the taints its --tolerated-taints flag names, which default to those three. A Modelplane that installs it can observe every pool. Signed-off-by: Nic Cope --- README.md | 2 +- apis/v1alpha1/poolobservation_types.go | 3 +- .../cluster-agent/templates/deployment.yaml | 1 + cluster/charts/cluster-agent/values.yaml | 8 + cmd/cluster-agent/main.go | 9 + .../controller/poolobservation/reconciler.go | 10 +- internal/pool/pool.go | 101 ++++++---- internal/pool/pool_test.go | 174 ++++++++++++++---- 8 files changed, 237 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index d70c327..a7eb871 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ status: total: 1 ready: 1 schedulable: 1 # Ready, not cordoned, and not tainted against - # Modelplane's GPU pods. + # Modelplane's pods. unschedulablePods: 1 # Pinned pods the scheduler can't place. devices: - deviceClassName: gpu.nvidia.com diff --git a/apis/v1alpha1/poolobservation_types.go b/apis/v1alpha1/poolobservation_types.go index 80951a0..cbc9b72 100644 --- a/apis/v1alpha1/poolobservation_types.go +++ b/apis/v1alpha1/poolobservation_types.go @@ -140,8 +140,7 @@ type NodeCounts struct { Ready int32 `json:"ready"` // Schedulable counts the ready nodes that aren't cordoned, and carry no - // NoSchedule or NoExecute taint but nvidia.com/gpu with NoSchedule, which - // Modelplane's GPU pods tolerate. + // NoSchedule or NoExecute taint that Modelplane's pods don't tolerate. // // +kubebuilder:validation:Minimum=0 Schedulable int32 `json:"schedulable"` diff --git a/cluster/charts/cluster-agent/templates/deployment.yaml b/cluster/charts/cluster-agent/templates/deployment.yaml index 76563f7..c4109b3 100644 --- a/cluster/charts/cluster-agent/templates/deployment.yaml +++ b/cluster/charts/cluster-agent/templates/deployment.yaml @@ -34,6 +34,7 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} args: - --observation-delay={{ .Values.observationDelay }} + - --tolerated-taints={{ join "," .Values.toleratedTaints }} {{- if .Values.debug }} - --debug {{- end }} diff --git a/cluster/charts/cluster-agent/values.yaml b/cluster/charts/cluster-agent/values.yaml index 41e8359..f36ff20 100644 --- a/cluster/charts/cluster-agent/values.yaml +++ b/cluster/charts/cluster-agent/values.yaml @@ -13,6 +13,14 @@ debug: false # Changes made while waiting are observed together. observationDelay: 10s +# Taints Modelplane's pods tolerate, as key[=value]:effect. A taint without a +# value matches any value. A node with any other NoSchedule or NoExecute taint +# doesn't count as schedulable. +toleratedTaints: +- nvidia.com/gpu:NoSchedule +- amd.com/gpu:NoSchedule +- modelplane.ai/cpu:NoSchedule + # The agent caches the nodes, pods and ResourceSlices it observes, so its # memory grows with the cluster. It uses about 50Mi on a cluster with nothing # to observe. There's no limit by default. diff --git a/cmd/cluster-agent/main.go b/cmd/cluster-agent/main.go index 18eb691..3851dda 100644 --- a/cmd/cluster-agent/main.go +++ b/cmd/cluster-agent/main.go @@ -54,6 +54,7 @@ type cli struct { MetricsBindAddress string `default:":8080" help:"The address the metrics endpoint binds to."` HealthProbeBindAddress string `default:":8081" help:"The address the health probe endpoint binds to."` LeaderElectionNamespace string `help:"The namespace of the Lease that elects one agent to run. Defaults to the agent's own namespace."` + ToleratedTaints []string `default:"nvidia.com/gpu:NoSchedule,amd.com/gpu:NoSchedule,modelplane.ai/cpu:NoSchedule" help:"Taints Modelplane's pods tolerate, as key[=value]:effect. A taint without a value matches any value. A node with any other NoSchedule or NoExecute taint doesn't count as schedulable."` } func main() { @@ -64,6 +65,13 @@ func main() { kong.UsageOnError(), ) + tolerations := make([]corev1.Toleration, len(flags.ToleratedTaints)) + for i, s := range flags.ToleratedTaints { + t, err := pool.ParseToleration(s) + kctx.FatalIfErrorf(err, "cannot parse --tolerated-taints") + tolerations[i] = t + } + level := slog.LevelInfo if flags.Debug { level = slog.LevelDebug @@ -120,6 +128,7 @@ func main() { kctx.FatalIfErrorf(err, "cannot add ResourceClaim event handler") kctx.FatalIfErrorf(poolobservation.Setup(mgr, record, flags.ObservationDelay, + poolobservation.WithPoolObserver(pool.NewObserver(pool.WithTolerations(tolerations))), poolobservation.WithLogger(log.With("controller", "poolobservation")), ), "cannot set up PoolObservation controller") kctx.FatalIfErrorf(replicaobservation.Setup(mgr, record, diff --git a/internal/controller/poolobservation/reconciler.go b/internal/controller/poolobservation/reconciler.go index 3e078d3..8057eb7 100644 --- a/internal/controller/poolobservation/reconciler.go +++ b/internal/controller/poolobservation/reconciler.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package poolobservation reports the state of a GPU pool in the status of a +// Package poolobservation reports the state of a pool in the status of a // PoolObservation. package poolobservation @@ -57,7 +57,7 @@ const ( FieldOwner = "cluster-agent.modelplane.ai" ) -// A PoolObserver observes a GPU pool. It must not modify its inputs, which +// A PoolObserver observes a pool. It must not modify its inputs, which // the Reconciler reads straight from its cache. type PoolObserver interface { ObservePool(ctx context.Context, name string, in pool.Inputs) (pool.Observation, error) @@ -71,7 +71,7 @@ func (fn PoolObserverFn) ObservePool(ctx context.Context, name string, in pool.I return fn(ctx, name, in) } -// A Reconciler reports the state of a GPU pool in the status of a +// A Reconciler reports the state of a pool in the status of a // PoolObservation. type Reconciler struct { client client.Client @@ -89,7 +89,7 @@ func WithLogger(l *slog.Logger) ReconcilerOption { } // WithPoolObserver sets how the Reconciler observes pools. It uses a -// pool.Observer by default. +// pool.Observer that tolerates no taints by default. func WithPoolObserver(o PoolObserver) ReconcilerOption { return func(r *Reconciler) { r.observer = o } } @@ -156,7 +156,7 @@ func EnqueueAllAfter(c client.Reader, delay time.Duration) handler.Funcs { } } -// Reconcile observes a GPU pool and reports it in the status of the requested +// Reconcile observes a pool and reports it in the status of the requested // PoolObservation. func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { log := r.log.With("request", req) diff --git a/internal/pool/pool.go b/internal/pool/pool.go index e5b7381..d054609 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package pool observes a GPU pool: the nodes Modelplane labels with the +// Package pool observes a pool: the nodes Modelplane labels with the // pool's name, the DRA devices they publish, and the pods waiting for them. package pool @@ -27,10 +27,12 @@ import ( "strings" "time" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" resourcev1 "k8s.io/api/resource/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/dynamic-resource-allocation/cel" "github.com/modelplaneai/cluster-agent/internal/dra" @@ -40,10 +42,6 @@ import ( // Modelplane pins a pod to a pool with a node selector on it. const LabelPool = "modelplane.ai/pool" -// TaintGPU is the taint Modelplane puts on the GPU pools it provisions. Its GPU -// pods tolerate it with effect NoSchedule. -const TaintGPU = "nvidia.com/gpu" - // Cluster autoscaler pod event reasons. See // https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-events-are-emitted-by-ca const ( @@ -111,8 +109,8 @@ type Nodes struct { // Cordoned counts the nodes marked unschedulable, ready or not. Cordoned int32 - // Schedulable counts the nodes Modelplane's GPU pods can schedule onto. - // See IsSchedulable. + // Schedulable counts the nodes Modelplane's pods can schedule onto. See + // IsSchedulable. Schedulable int32 } @@ -179,24 +177,39 @@ type Membership struct { // Nodes is the set of the pool's nodes. Nodes map[string]bool - // Schedulable is the set of the pool's nodes that Modelplane's GPU pods - // can schedule onto. See IsSchedulable. + // Schedulable is the set of the pool's nodes that Modelplane's pods can + // schedule onto. See IsSchedulable. Schedulable map[string]bool } // An Observer observes pools. type Observer struct { - cel *cel.Cache + cel *cel.Cache + tolerations []corev1.Toleration +} + +// An ObserverOption configures an Observer. +type ObserverOption func(*Observer) + +// WithTolerations sets the tolerations Modelplane's pods carry. The Observer +// counts a node as schedulable only if they tolerate its taints. It tolerates +// no taints by default. +func WithTolerations(t []corev1.Toleration) ObserverOption { + return func(o *Observer) { o.tolerations = t } } // NewObserver returns an Observer. -func NewObserver() *Observer { +func NewObserver(opts ...ObserverOption) *Observer { // Enable every CEL feature so the Observer can compile any expression // the API server accepted. - return &Observer{cel: cel.NewCache(celCacheSize, cel.Features{ + o := &Observer{cel: cel.NewCache(celCacheSize, cel.Features{ EnableConsumableCapacity: true, EnableListTypeAttributes: true, })} + for _, fn := range opts { + fn(o) + } + return o } // ObservePool returns what's true of the named pool. A pool's nodes are those @@ -208,7 +221,7 @@ func (o *Observer) ObservePool(ctx context.Context, name string, in Inputs) (Obs nodes = append(nodes, n) } } - m := Members(nodes) + m := Members(nodes, o.tolerations) devices := dra.Devices(in.Slices) byClass, err := o.DevicesByClass(ctx, devices, in.Classes, Allocated(in.Claims), m) @@ -216,7 +229,7 @@ func (o *Observer) ObservePool(ctx context.Context, name string, in Inputs) (Obs return Observation{}, err } return Observation{ - Nodes: CountNodes(nodes), + Nodes: CountNodes(nodes, o.tolerations), Devices: byClass, Free: Free(byClass, m), LastAllocation: LastAllocation(in.Claims, in.AllocatedAt, devices, m), @@ -227,20 +240,21 @@ func (o *Observer) ObservePool(ctx context.Context, name string, in Inputs) (Obs } // Members returns the supplied nodes as a pool's members, and which of them -// are schedulable. -func Members(nodes []corev1.Node) Membership { +// are schedulable by pods with the supplied tolerations. +func Members(nodes []corev1.Node, ts []corev1.Toleration) Membership { m := Membership{Nodes: map[string]bool{}, Schedulable: map[string]bool{}} for _, n := range nodes { m.Nodes[n.Name] = true - if IsSchedulable(n) { + if IsSchedulable(n, ts) { m.Schedulable[n.Name] = true } } return m } -// CountNodes counts a pool's nodes. -func CountNodes(nodes []corev1.Node) Nodes { +// CountNodes counts a pool's nodes, counting a node as schedulable if pods with +// the supplied tolerations can schedule onto it. +func CountNodes(nodes []corev1.Node, ts []corev1.Toleration) Nodes { c := Nodes{} for _, n := range nodes { c.Total++ @@ -250,7 +264,7 @@ func CountNodes(nodes []corev1.Node) Nodes { if IsReady(n) { c.Ready++ } - if IsSchedulable(n) { + if IsSchedulable(n, ts) { c.Schedulable++ } } @@ -453,28 +467,51 @@ func IsReady(n corev1.Node) bool { return false } -// IsSchedulable returns true if Modelplane's GPU pods can schedule onto the -// node: it's ready, it isn't cordoned, and every NoSchedule or NoExecute taint -// it carries is one they tolerate. -func IsSchedulable(n corev1.Node) bool { +// IsSchedulable returns true if pods with the supplied tolerations can +// schedule onto the node: it's ready, it isn't cordoned, and every NoSchedule +// or NoExecute taint it carries is one they tolerate. +func IsSchedulable(n corev1.Node, ts []corev1.Toleration) bool { if !IsReady(n) || n.Spec.Unschedulable { return false } - for _, t := range n.Spec.Taints { - if t.Effect == corev1.TaintEffectPreferNoSchedule { + for _, taint := range n.Spec.Taints { + if taint.Effect == corev1.TaintEffectPreferNoSchedule { continue } - // compose-model-replica gives Modelplane's GPU pods this one - // toleration (_GPU_TOLERATION in - // functions/compose-model-replica/function/backends/base.py). - if t.Key == TaintGPU && t.Effect == corev1.TaintEffectNoSchedule { - continue + // ToleratesTaint only logs when comparing a Lt or Gt toleration's + // value, which it won't do with comparison operators disabled. + if !slices.ContainsFunc(ts, func(t corev1.Toleration) bool { return t.ToleratesTaint(logr.Discard(), &taint, false) }) { + return false } - return false } return true } +// ParseToleration parses a toleration of the taints written key[=value]:effect. +// The toleration tolerates taints with the key and effect. It tolerates only +// the value if there is one, and any value if there isn't. The effect must be +// NoSchedule or NoExecute, the effects IsSchedulable considers. +func ParseToleration(s string) (corev1.Toleration, error) { + kv, effect, ok := strings.Cut(s, ":") + if !ok { + return corev1.Toleration{}, fmt.Errorf("toleration %q has no effect, want key[=value]:effect", s) + } + t := corev1.Toleration{Key: kv, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffect(effect)} + if key, value, ok := strings.Cut(kv, "="); ok { + t = corev1.Toleration{Key: key, Operator: corev1.TolerationOpEqual, Value: value, Effect: t.Effect} + } + if errs := validation.IsQualifiedName(t.Key); len(errs) > 0 { + return corev1.Toleration{}, fmt.Errorf("toleration %q has an invalid key: %s", s, strings.Join(errs, ", ")) + } + if errs := validation.IsValidLabelValue(t.Value); len(errs) > 0 { + return corev1.Toleration{}, fmt.Errorf("toleration %q has an invalid value: %s", s, strings.Join(errs, ", ")) + } + if t.Effect != corev1.TaintEffectNoSchedule && t.Effect != corev1.TaintEffectNoExecute { + return corev1.Toleration{}, fmt.Errorf("toleration %q has an invalid effect, want NoSchedule or NoExecute", s) + } + return t, nil +} + // IsUnschedulable returns true if the scheduler couldn't find a node for the // pod. func IsUnschedulable(p corev1.Pod) bool { diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index 59b9184..14caeec 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -50,6 +50,10 @@ import ( // pool, pods of a replica and the cluster autoscaler's events, all without // managed fields. +// The snapshots' GPU node carries the nvidia.com/gpu taint, which Modelplane's +// pods tolerate. +var gpuTolerations = []corev1.Toleration{{Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}} + var equateQuantities = cmp.Comparer(func(a, b resource.Quantity) bool { return a.Cmp(b) == 0 }) // The attributes every L4 on the GPU node agrees on. @@ -209,7 +213,7 @@ func TestObservePool(t *testing.T) { for name, tc := range cases { t.Run(name, func(t *testing.T) { before := deepCopy(tc.args.in) - got, err := NewObserver().ObservePool(t.Context(), tc.args.pool, tc.args.in) + got, err := NewObserver(WithTolerations(gpuTolerations)).ObservePool(t.Context(), tc.args.pool, tc.args.in) if diff := cmp.Diff(nil, err, cmpopts.EquateErrors()); diff != "" { t.Errorf("\n%s\nObservePool(...): -want error, +got error:\n%s", tc.reason, diff) } @@ -434,55 +438,97 @@ func TestCountNodes(t *testing.T) { ready := corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}} notReady := corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionFalse}}} + type args struct { + nodes []corev1.Node + tolerations []corev1.Toleration + } + cases := map[string]struct { reason string - nodes []corev1.Node + args args want Nodes }{ "States": { reason: "Each node should count by whether it's ready and cordoned.", - nodes: []corev1.Node{ - {ObjectMeta: metav1.ObjectMeta{Name: "ready"}, Status: ready}, - {ObjectMeta: metav1.ObjectMeta{Name: "cordoned"}, Spec: corev1.NodeSpec{Unschedulable: true}, Status: ready}, - {ObjectMeta: metav1.ObjectMeta{Name: "draining"}, Spec: corev1.NodeSpec{Unschedulable: true}, Status: notReady}, - {ObjectMeta: metav1.ObjectMeta{Name: "no-condition"}}, + args: args{ + nodes: []corev1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "ready"}, Status: ready}, + {ObjectMeta: metav1.ObjectMeta{Name: "cordoned"}, Spec: corev1.NodeSpec{Unschedulable: true}, Status: ready}, + {ObjectMeta: metav1.ObjectMeta{Name: "draining"}, Spec: corev1.NodeSpec{Unschedulable: true}, Status: notReady}, + {ObjectMeta: metav1.ObjectMeta{Name: "no-condition"}}, + }, }, want: Nodes{Total: 4, Ready: 2, Cordoned: 2, Schedulable: 1}, }, "Taints": { - reason: "A ready node should be schedulable only if Modelplane's GPU pods tolerate every NoSchedule and NoExecute taint it carries.", - nodes: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{Name: "gpu-taint"}, - Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "nvidia.com/gpu", Value: "true", Effect: corev1.TaintEffectNoSchedule}}}, - Status: ready, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "prefer-no-schedule"}, - Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "DeletionCandidateOfClusterAutoscaler", Effect: corev1.TaintEffectPreferNoSchedule}}}, - Status: ready, + reason: "A ready node should be schedulable only if the tolerations tolerate every NoSchedule and NoExecute taint it carries.", + args: args{ + nodes: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{Name: "nvidia-gpu"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "nvidia.com/gpu", Value: "true", Effect: corev1.TaintEffectNoSchedule}}}, + Status: ready, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "amd-gpu"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "amd.com/gpu", Value: "present", Effect: corev1.TaintEffectNoSchedule}}}, + Status: ready, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "cpu"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "modelplane.ai/cpu", Value: "true", Effect: corev1.TaintEffectNoSchedule}}}, + Status: ready, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "cpu-other-value"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "modelplane.ai/cpu", Value: "false", Effect: corev1.TaintEffectNoSchedule}}}, + Status: ready, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "prefer-no-schedule"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "DeletionCandidateOfClusterAutoscaler", Effect: corev1.TaintEffectPreferNoSchedule}}}, + Status: ready, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "being-removed"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{ + {Key: "nvidia.com/gpu", Value: "true", Effect: corev1.TaintEffectNoSchedule}, + {Key: "ToBeDeletedByClusterAutoscaler", Effect: corev1.TaintEffectNoSchedule}, + }}, + Status: ready, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "gpu-no-execute"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "nvidia.com/gpu", Effect: corev1.TaintEffectNoExecute}}}, + Status: ready, + }, }, - { - ObjectMeta: metav1.ObjectMeta{Name: "being-removed"}, - Spec: corev1.NodeSpec{Taints: []corev1.Taint{ - {Key: "nvidia.com/gpu", Value: "true", Effect: corev1.TaintEffectNoSchedule}, - {Key: "ToBeDeletedByClusterAutoscaler", Effect: corev1.TaintEffectNoSchedule}, - }}, - Status: ready, + tolerations: []corev1.Toleration{ + {Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, + {Key: "amd.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, + {Key: "modelplane.ai/cpu", Operator: corev1.TolerationOpEqual, Value: "true", Effect: corev1.TaintEffectNoSchedule}, }, - { - ObjectMeta: metav1.ObjectMeta{Name: "gpu-no-execute"}, - Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "nvidia.com/gpu", Effect: corev1.TaintEffectNoExecute}}}, - Status: ready, + }, + want: Nodes{Total: 7, Ready: 7, Schedulable: 4}, + }, + "NoTolerations": { + reason: "A ready node with a NoSchedule taint shouldn't be schedulable without tolerations.", + args: args{ + nodes: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{Name: "nvidia-gpu"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "nvidia.com/gpu", Value: "true", Effect: corev1.TaintEffectNoSchedule}}}, + Status: ready, + }, }, }, - want: Nodes{Total: 4, Ready: 4, Schedulable: 2}, + want: Nodes{Total: 1, Ready: 1}, }, } for name, tc := range cases { t.Run(name, func(t *testing.T) { - got := CountNodes(tc.nodes) + got := CountNodes(tc.args.nodes, tc.args.tolerations) if diff := cmp.Diff(tc.want, got); diff != "" { t.Errorf("\n%s\nCountNodes(...): -want, +got:\n%s", tc.reason, diff) } @@ -490,6 +536,72 @@ func TestCountNodes(t *testing.T) { } } +func TestParseToleration(t *testing.T) { + type want struct { + t corev1.Toleration + err error + } + + cases := map[string]struct { + reason string + s string + want want + }{ + "AnyValue": { + reason: "A taint without a value should parse as a toleration of any value.", + s: "nvidia.com/gpu:NoSchedule", + want: want{t: corev1.Toleration{Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}}, + }, + "Value": { + reason: "A taint with a value should parse as a toleration of only that value.", + s: "modelplane.ai/cpu=true:NoExecute", + want: want{t: corev1.Toleration{Key: "modelplane.ai/cpu", Operator: corev1.TolerationOpEqual, Value: "true", Effect: corev1.TaintEffectNoExecute}}, + }, + "EmptyValue": { + reason: "A taint with an empty value should parse as a toleration of only the empty value.", + s: "dedicated=:NoSchedule", + want: want{t: corev1.Toleration{Key: "dedicated", Operator: corev1.TolerationOpEqual, Effect: corev1.TaintEffectNoSchedule}}, + }, + "PreferNoSchedule": { + reason: "A taint with effect PreferNoSchedule should be an error, because it never makes a node unschedulable.", + s: "nvidia.com/gpu:PreferNoSchedule", + want: want{err: cmpopts.AnyError}, + }, + "NoEffect": { + reason: "A taint without an effect should be an error.", + s: "nvidia.com/gpu", + want: want{err: cmpopts.AnyError}, + }, + "InvalidEffect": { + reason: "A taint with an effect Kubernetes doesn't define should be an error.", + s: "nvidia.com/gpu:noschedule", + want: want{err: cmpopts.AnyError}, + }, + "InvalidKey": { + reason: "A taint with a key that isn't a qualified name should be an error.", + s: ":NoSchedule", + want: want{err: cmpopts.AnyError}, + }, + "InvalidValue": { + reason: "A taint with a value that isn't a valid label value should be an error.", + s: "nvidia.com/gpu=a/b:NoSchedule", + want: want{err: cmpopts.AnyError}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + got, err := ParseToleration(tc.s) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("\n%s\nParseToleration(%q): -want error, +got error:\n%s", tc.reason, tc.s, diff) + } + if diff := cmp.Diff(tc.want.t, got); diff != "" { + t.Errorf("\n%s\nParseToleration(%q): -want, +got:\n%s", tc.reason, tc.s, diff) + } + }) + } +} + func TestExtendedResources(t *testing.T) { cases := map[string]struct { reason string From 1203c9870c01c9657a95b50b3595f82d70c075d4 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Wed, 4 Nov 2026 19:32:12 -0800 Subject: [PATCH 3/3] Regenerate CRDs Signed-off-by: Nic Cope --- .../files/crds/agent.modelplane.ai_poolobservations.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cluster/charts/cluster-agent/files/crds/agent.modelplane.ai_poolobservations.yaml b/cluster/charts/cluster-agent/files/crds/agent.modelplane.ai_poolobservations.yaml index 695f916..5243205 100644 --- a/cluster/charts/cluster-agent/files/crds/agent.modelplane.ai_poolobservations.yaml +++ b/cluster/charts/cluster-agent/files/crds/agent.modelplane.ai_poolobservations.yaml @@ -283,8 +283,7 @@ spec: schedulable: description: |- Schedulable counts the ready nodes that aren't cordoned, and carry no - NoSchedule or NoExecute taint but nvidia.com/gpu with NoSchedule, which - Modelplane's GPU pods tolerate. + NoSchedule or NoExecute taint that Modelplane's pods don't tolerate. format: int32 minimum: 0 type: integer