diff --git a/README.md b/README.md index d70c327..a7eb871 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ status: total: 1 ready: 1 schedulable: 1 # Ready, not cordoned, and not tainted against - # Modelplane's GPU pods. + # Modelplane's pods. unschedulablePods: 1 # Pinned pods the scheduler can't place. devices: - deviceClassName: gpu.nvidia.com diff --git a/apis/v1alpha1/poolobservation_types.go b/apis/v1alpha1/poolobservation_types.go index 80951a0..cbc9b72 100644 --- a/apis/v1alpha1/poolobservation_types.go +++ b/apis/v1alpha1/poolobservation_types.go @@ -140,8 +140,7 @@ type NodeCounts struct { Ready int32 `json:"ready"` // Schedulable counts the ready nodes that aren't cordoned, and carry no - // NoSchedule or NoExecute taint but nvidia.com/gpu with NoSchedule, which - // Modelplane's GPU pods tolerate. + // NoSchedule or NoExecute taint that Modelplane's pods don't tolerate. // // +kubebuilder:validation:Minimum=0 Schedulable int32 `json:"schedulable"` diff --git a/cluster/charts/cluster-agent/files/crds/agent.modelplane.ai_poolobservations.yaml b/cluster/charts/cluster-agent/files/crds/agent.modelplane.ai_poolobservations.yaml index 695f916..5243205 100644 --- a/cluster/charts/cluster-agent/files/crds/agent.modelplane.ai_poolobservations.yaml +++ b/cluster/charts/cluster-agent/files/crds/agent.modelplane.ai_poolobservations.yaml @@ -283,8 +283,7 @@ spec: schedulable: description: |- Schedulable counts the ready nodes that aren't cordoned, and carry no - NoSchedule or NoExecute taint but nvidia.com/gpu with NoSchedule, which - Modelplane's GPU pods tolerate. + NoSchedule or NoExecute taint that Modelplane's pods don't tolerate. format: int32 minimum: 0 type: integer diff --git a/cluster/charts/cluster-agent/templates/deployment.yaml b/cluster/charts/cluster-agent/templates/deployment.yaml index 76563f7..c4109b3 100644 --- a/cluster/charts/cluster-agent/templates/deployment.yaml +++ b/cluster/charts/cluster-agent/templates/deployment.yaml @@ -34,6 +34,7 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} args: - --observation-delay={{ .Values.observationDelay }} + - --tolerated-taints={{ join "," .Values.toleratedTaints }} {{- if .Values.debug }} - --debug {{- end }} diff --git a/cluster/charts/cluster-agent/values.yaml b/cluster/charts/cluster-agent/values.yaml index 41e8359..f36ff20 100644 --- a/cluster/charts/cluster-agent/values.yaml +++ b/cluster/charts/cluster-agent/values.yaml @@ -13,6 +13,14 @@ debug: false # Changes made while waiting are observed together. observationDelay: 10s +# Taints Modelplane's pods tolerate, as key[=value]:effect. A taint without a +# value matches any value. A node with any other NoSchedule or NoExecute taint +# doesn't count as schedulable. +toleratedTaints: +- nvidia.com/gpu:NoSchedule +- amd.com/gpu:NoSchedule +- modelplane.ai/cpu:NoSchedule + # The agent caches the nodes, pods and ResourceSlices it observes, so its # memory grows with the cluster. It uses about 50Mi on a cluster with nothing # to observe. There's no limit by default. diff --git a/cmd/cluster-agent/main.go b/cmd/cluster-agent/main.go index 3e5f060..3851dda 100644 --- a/cmd/cluster-agent/main.go +++ b/cmd/cluster-agent/main.go @@ -54,6 +54,7 @@ type cli struct { MetricsBindAddress string `default:":8080" help:"The address the metrics endpoint binds to."` HealthProbeBindAddress string `default:":8081" help:"The address the health probe endpoint binds to."` LeaderElectionNamespace string `help:"The namespace of the Lease that elects one agent to run. Defaults to the agent's own namespace."` + ToleratedTaints []string `default:"nvidia.com/gpu:NoSchedule,amd.com/gpu:NoSchedule,modelplane.ai/cpu:NoSchedule" help:"Taints Modelplane's pods tolerate, as key[=value]:effect. A taint without a value matches any value. A node with any other NoSchedule or NoExecute taint doesn't count as schedulable."` } func main() { @@ -64,6 +65,13 @@ func main() { kong.UsageOnError(), ) + tolerations := make([]corev1.Toleration, len(flags.ToleratedTaints)) + for i, s := range flags.ToleratedTaints { + t, err := pool.ParseToleration(s) + kctx.FatalIfErrorf(err, "cannot parse --tolerated-taints") + tolerations[i] = t + } + level := slog.LevelInfo if flags.Debug { level = slog.LevelDebug @@ -119,8 +127,13 @@ func main() { _, err = claims.AddEventHandler(allocation.ForgetDeleted(record)) kctx.FatalIfErrorf(err, "cannot add ResourceClaim event handler") - kctx.FatalIfErrorf(poolobservation.Setup(mgr, record, log, flags.ObservationDelay), "cannot set up PoolObservation controller") - kctx.FatalIfErrorf(replicaobservation.Setup(mgr, record, log), "cannot set up ReplicaObservation controller") + kctx.FatalIfErrorf(poolobservation.Setup(mgr, record, flags.ObservationDelay, + poolobservation.WithPoolObserver(pool.NewObserver(pool.WithTolerations(tolerations))), + poolobservation.WithLogger(log.With("controller", "poolobservation")), + ), "cannot set up PoolObservation controller") + kctx.FatalIfErrorf(replicaobservation.Setup(mgr, record, + replicaobservation.WithLogger(log.With("controller", "replicaobservation")), + ), "cannot set up ReplicaObservation controller") kctx.FatalIfErrorf(mgr.Start(ctx), "cannot run manager") } diff --git a/internal/controller/poolobservation/reconciler.go b/internal/controller/poolobservation/reconciler.go index 5850cbc..8057eb7 100644 --- a/internal/controller/poolobservation/reconciler.go +++ b/internal/controller/poolobservation/reconciler.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package poolobservation reports the state of a GPU pool in the status of a +// Package poolobservation reports the state of a pool in the status of a // PoolObservation. package poolobservation @@ -57,7 +57,7 @@ const ( FieldOwner = "cluster-agent.modelplane.ai" ) -// A PoolObserver observes a GPU pool. It must not modify its inputs, which +// A PoolObserver observes a pool. It must not modify its inputs, which // the Reconciler reads straight from its cache. type PoolObserver interface { ObservePool(ctx context.Context, name string, in pool.Inputs) (pool.Observation, error) @@ -71,7 +71,7 @@ func (fn PoolObserverFn) ObservePool(ctx context.Context, name string, in pool.I return fn(ctx, name, in) } -// A Reconciler reports the state of a GPU pool in the status of a +// A Reconciler reports the state of a pool in the status of a // PoolObservation. type Reconciler struct { client client.Client @@ -89,7 +89,7 @@ func WithLogger(l *slog.Logger) ReconcilerOption { } // WithPoolObserver sets how the Reconciler observes pools. It uses a -// pool.Observer by default. +// pool.Observer that tolerates no taints by default. func WithPoolObserver(o PoolObserver) ReconcilerOption { return func(r *Reconciler) { r.observer = o } } @@ -112,8 +112,9 @@ func NewReconciler(c client.Client, record *allocation.Record, opts ...Reconcile // Setup adds a PoolObservation controller to the manager. The controller // observes every pool whenever an object it reads changes, but waits for the // supplied delay first, so that a burst of changes produces one observation -// of each pool. It reads ResourceClaims through the supplied Record. -func Setup(mgr ctrl.Manager, record *allocation.Record, log *slog.Logger, delay time.Duration) error { +// of each pool. It reads ResourceClaims through the supplied Record, and +// configures its Reconciler with the supplied options. +func Setup(mgr ctrl.Manager, record *allocation.Record, delay time.Duration, opts ...ReconcilerOption) error { c := mgr.GetClient() enqueue := EnqueueAllAfter(c, delay) err := ctrl.NewControllerManagedBy(mgr). @@ -125,7 +126,7 @@ func Setup(mgr ctrl.Manager, record *allocation.Record, log *slog.Logger, delay Watches(&resourcev1.ResourceClaim{}, enqueue). Watches(&corev1.Pod{}, enqueue). Watches(&corev1.Event{}, enqueue). - Complete(NewReconciler(c, record, WithLogger(log.With("controller", "poolobservation")))) + Complete(NewReconciler(c, record, opts...)) if err != nil { return fmt.Errorf("cannot create controller: %w", err) } @@ -155,7 +156,7 @@ func EnqueueAllAfter(c client.Reader, delay time.Duration) handler.Funcs { } } -// Reconcile observes a GPU pool and reports it in the status of the requested +// Reconcile observes a pool and reports it in the status of the requested // PoolObservation. func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { log := r.log.With("request", req) diff --git a/internal/controller/replicaobservation/reconciler.go b/internal/controller/replicaobservation/reconciler.go index dfdb7bc..0fdab08 100644 --- a/internal/controller/replicaobservation/reconciler.go +++ b/internal/controller/replicaobservation/reconciler.go @@ -111,8 +111,9 @@ func NewReconciler(c client.Client, record *allocation.Record, opts ...Reconcile } // Setup adds a ReplicaObservation controller to the manager. It reads -// ResourceClaims through the supplied Record. -func Setup(mgr ctrl.Manager, record *allocation.Record, log *slog.Logger) error { +// ResourceClaims through the supplied Record, and configures its Reconciler +// with the supplied options. +func Setup(mgr ctrl.Manager, record *allocation.Record, opts ...ReconcilerOption) error { c := mgr.GetClient() err := ctrl.NewControllerManagedBy(mgr). Named("replicaobservation"). @@ -126,7 +127,7 @@ func Setup(mgr ctrl.Manager, record *allocation.Record, log *slog.Logger) error // but the claim cache can lag the pod cache. Watching claims catches // an allocation the pod's binding raced ahead of. Watches(&resourcev1.ResourceClaim{}, handler.EnqueueRequestsFromMapFunc(ClaimRequests(c))). - Complete(NewReconciler(c, record, WithLogger(log.With("controller", "replicaobservation")))) + Complete(NewReconciler(c, record, opts...)) if err != nil { return fmt.Errorf("cannot create controller: %w", err) } diff --git a/internal/pool/pool.go b/internal/pool/pool.go index e5b7381..d054609 100644 --- a/internal/pool/pool.go +++ b/internal/pool/pool.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package pool observes a GPU pool: the nodes Modelplane labels with the +// Package pool observes a pool: the nodes Modelplane labels with the // pool's name, the DRA devices they publish, and the pods waiting for them. package pool @@ -27,10 +27,12 @@ import ( "strings" "time" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" resourcev1 "k8s.io/api/resource/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/dynamic-resource-allocation/cel" "github.com/modelplaneai/cluster-agent/internal/dra" @@ -40,10 +42,6 @@ import ( // Modelplane pins a pod to a pool with a node selector on it. const LabelPool = "modelplane.ai/pool" -// TaintGPU is the taint Modelplane puts on the GPU pools it provisions. Its GPU -// pods tolerate it with effect NoSchedule. -const TaintGPU = "nvidia.com/gpu" - // Cluster autoscaler pod event reasons. See // https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-events-are-emitted-by-ca const ( @@ -111,8 +109,8 @@ type Nodes struct { // Cordoned counts the nodes marked unschedulable, ready or not. Cordoned int32 - // Schedulable counts the nodes Modelplane's GPU pods can schedule onto. - // See IsSchedulable. + // Schedulable counts the nodes Modelplane's pods can schedule onto. See + // IsSchedulable. Schedulable int32 } @@ -179,24 +177,39 @@ type Membership struct { // Nodes is the set of the pool's nodes. Nodes map[string]bool - // Schedulable is the set of the pool's nodes that Modelplane's GPU pods - // can schedule onto. See IsSchedulable. + // Schedulable is the set of the pool's nodes that Modelplane's pods can + // schedule onto. See IsSchedulable. Schedulable map[string]bool } // An Observer observes pools. type Observer struct { - cel *cel.Cache + cel *cel.Cache + tolerations []corev1.Toleration +} + +// An ObserverOption configures an Observer. +type ObserverOption func(*Observer) + +// WithTolerations sets the tolerations Modelplane's pods carry. The Observer +// counts a node as schedulable only if they tolerate its taints. It tolerates +// no taints by default. +func WithTolerations(t []corev1.Toleration) ObserverOption { + return func(o *Observer) { o.tolerations = t } } // NewObserver returns an Observer. -func NewObserver() *Observer { +func NewObserver(opts ...ObserverOption) *Observer { // Enable every CEL feature so the Observer can compile any expression // the API server accepted. - return &Observer{cel: cel.NewCache(celCacheSize, cel.Features{ + o := &Observer{cel: cel.NewCache(celCacheSize, cel.Features{ EnableConsumableCapacity: true, EnableListTypeAttributes: true, })} + for _, fn := range opts { + fn(o) + } + return o } // ObservePool returns what's true of the named pool. A pool's nodes are those @@ -208,7 +221,7 @@ func (o *Observer) ObservePool(ctx context.Context, name string, in Inputs) (Obs nodes = append(nodes, n) } } - m := Members(nodes) + m := Members(nodes, o.tolerations) devices := dra.Devices(in.Slices) byClass, err := o.DevicesByClass(ctx, devices, in.Classes, Allocated(in.Claims), m) @@ -216,7 +229,7 @@ func (o *Observer) ObservePool(ctx context.Context, name string, in Inputs) (Obs return Observation{}, err } return Observation{ - Nodes: CountNodes(nodes), + Nodes: CountNodes(nodes, o.tolerations), Devices: byClass, Free: Free(byClass, m), LastAllocation: LastAllocation(in.Claims, in.AllocatedAt, devices, m), @@ -227,20 +240,21 @@ func (o *Observer) ObservePool(ctx context.Context, name string, in Inputs) (Obs } // Members returns the supplied nodes as a pool's members, and which of them -// are schedulable. -func Members(nodes []corev1.Node) Membership { +// are schedulable by pods with the supplied tolerations. +func Members(nodes []corev1.Node, ts []corev1.Toleration) Membership { m := Membership{Nodes: map[string]bool{}, Schedulable: map[string]bool{}} for _, n := range nodes { m.Nodes[n.Name] = true - if IsSchedulable(n) { + if IsSchedulable(n, ts) { m.Schedulable[n.Name] = true } } return m } -// CountNodes counts a pool's nodes. -func CountNodes(nodes []corev1.Node) Nodes { +// CountNodes counts a pool's nodes, counting a node as schedulable if pods with +// the supplied tolerations can schedule onto it. +func CountNodes(nodes []corev1.Node, ts []corev1.Toleration) Nodes { c := Nodes{} for _, n := range nodes { c.Total++ @@ -250,7 +264,7 @@ func CountNodes(nodes []corev1.Node) Nodes { if IsReady(n) { c.Ready++ } - if IsSchedulable(n) { + if IsSchedulable(n, ts) { c.Schedulable++ } } @@ -453,28 +467,51 @@ func IsReady(n corev1.Node) bool { return false } -// IsSchedulable returns true if Modelplane's GPU pods can schedule onto the -// node: it's ready, it isn't cordoned, and every NoSchedule or NoExecute taint -// it carries is one they tolerate. -func IsSchedulable(n corev1.Node) bool { +// IsSchedulable returns true if pods with the supplied tolerations can +// schedule onto the node: it's ready, it isn't cordoned, and every NoSchedule +// or NoExecute taint it carries is one they tolerate. +func IsSchedulable(n corev1.Node, ts []corev1.Toleration) bool { if !IsReady(n) || n.Spec.Unschedulable { return false } - for _, t := range n.Spec.Taints { - if t.Effect == corev1.TaintEffectPreferNoSchedule { + for _, taint := range n.Spec.Taints { + if taint.Effect == corev1.TaintEffectPreferNoSchedule { continue } - // compose-model-replica gives Modelplane's GPU pods this one - // toleration (_GPU_TOLERATION in - // functions/compose-model-replica/function/backends/base.py). - if t.Key == TaintGPU && t.Effect == corev1.TaintEffectNoSchedule { - continue + // ToleratesTaint only logs when comparing a Lt or Gt toleration's + // value, which it won't do with comparison operators disabled. + if !slices.ContainsFunc(ts, func(t corev1.Toleration) bool { return t.ToleratesTaint(logr.Discard(), &taint, false) }) { + return false } - return false } return true } +// ParseToleration parses a toleration of the taints written key[=value]:effect. +// The toleration tolerates taints with the key and effect. It tolerates only +// the value if there is one, and any value if there isn't. The effect must be +// NoSchedule or NoExecute, the effects IsSchedulable considers. +func ParseToleration(s string) (corev1.Toleration, error) { + kv, effect, ok := strings.Cut(s, ":") + if !ok { + return corev1.Toleration{}, fmt.Errorf("toleration %q has no effect, want key[=value]:effect", s) + } + t := corev1.Toleration{Key: kv, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffect(effect)} + if key, value, ok := strings.Cut(kv, "="); ok { + t = corev1.Toleration{Key: key, Operator: corev1.TolerationOpEqual, Value: value, Effect: t.Effect} + } + if errs := validation.IsQualifiedName(t.Key); len(errs) > 0 { + return corev1.Toleration{}, fmt.Errorf("toleration %q has an invalid key: %s", s, strings.Join(errs, ", ")) + } + if errs := validation.IsValidLabelValue(t.Value); len(errs) > 0 { + return corev1.Toleration{}, fmt.Errorf("toleration %q has an invalid value: %s", s, strings.Join(errs, ", ")) + } + if t.Effect != corev1.TaintEffectNoSchedule && t.Effect != corev1.TaintEffectNoExecute { + return corev1.Toleration{}, fmt.Errorf("toleration %q has an invalid effect, want NoSchedule or NoExecute", s) + } + return t, nil +} + // IsUnschedulable returns true if the scheduler couldn't find a node for the // pod. func IsUnschedulable(p corev1.Pod) bool { diff --git a/internal/pool/pool_test.go b/internal/pool/pool_test.go index 59b9184..14caeec 100644 --- a/internal/pool/pool_test.go +++ b/internal/pool/pool_test.go @@ -50,6 +50,10 @@ import ( // pool, pods of a replica and the cluster autoscaler's events, all without // managed fields. +// The snapshots' GPU node carries the nvidia.com/gpu taint, which Modelplane's +// pods tolerate. +var gpuTolerations = []corev1.Toleration{{Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}} + var equateQuantities = cmp.Comparer(func(a, b resource.Quantity) bool { return a.Cmp(b) == 0 }) // The attributes every L4 on the GPU node agrees on. @@ -209,7 +213,7 @@ func TestObservePool(t *testing.T) { for name, tc := range cases { t.Run(name, func(t *testing.T) { before := deepCopy(tc.args.in) - got, err := NewObserver().ObservePool(t.Context(), tc.args.pool, tc.args.in) + got, err := NewObserver(WithTolerations(gpuTolerations)).ObservePool(t.Context(), tc.args.pool, tc.args.in) if diff := cmp.Diff(nil, err, cmpopts.EquateErrors()); diff != "" { t.Errorf("\n%s\nObservePool(...): -want error, +got error:\n%s", tc.reason, diff) } @@ -434,55 +438,97 @@ func TestCountNodes(t *testing.T) { ready := corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}} notReady := corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionFalse}}} + type args struct { + nodes []corev1.Node + tolerations []corev1.Toleration + } + cases := map[string]struct { reason string - nodes []corev1.Node + args args want Nodes }{ "States": { reason: "Each node should count by whether it's ready and cordoned.", - nodes: []corev1.Node{ - {ObjectMeta: metav1.ObjectMeta{Name: "ready"}, Status: ready}, - {ObjectMeta: metav1.ObjectMeta{Name: "cordoned"}, Spec: corev1.NodeSpec{Unschedulable: true}, Status: ready}, - {ObjectMeta: metav1.ObjectMeta{Name: "draining"}, Spec: corev1.NodeSpec{Unschedulable: true}, Status: notReady}, - {ObjectMeta: metav1.ObjectMeta{Name: "no-condition"}}, + args: args{ + nodes: []corev1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "ready"}, Status: ready}, + {ObjectMeta: metav1.ObjectMeta{Name: "cordoned"}, Spec: corev1.NodeSpec{Unschedulable: true}, Status: ready}, + {ObjectMeta: metav1.ObjectMeta{Name: "draining"}, Spec: corev1.NodeSpec{Unschedulable: true}, Status: notReady}, + {ObjectMeta: metav1.ObjectMeta{Name: "no-condition"}}, + }, }, want: Nodes{Total: 4, Ready: 2, Cordoned: 2, Schedulable: 1}, }, "Taints": { - reason: "A ready node should be schedulable only if Modelplane's GPU pods tolerate every NoSchedule and NoExecute taint it carries.", - nodes: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{Name: "gpu-taint"}, - Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "nvidia.com/gpu", Value: "true", Effect: corev1.TaintEffectNoSchedule}}}, - Status: ready, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "prefer-no-schedule"}, - Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "DeletionCandidateOfClusterAutoscaler", Effect: corev1.TaintEffectPreferNoSchedule}}}, - Status: ready, + reason: "A ready node should be schedulable only if the tolerations tolerate every NoSchedule and NoExecute taint it carries.", + args: args{ + nodes: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{Name: "nvidia-gpu"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "nvidia.com/gpu", Value: "true", Effect: corev1.TaintEffectNoSchedule}}}, + Status: ready, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "amd-gpu"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "amd.com/gpu", Value: "present", Effect: corev1.TaintEffectNoSchedule}}}, + Status: ready, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "cpu"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "modelplane.ai/cpu", Value: "true", Effect: corev1.TaintEffectNoSchedule}}}, + Status: ready, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "cpu-other-value"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "modelplane.ai/cpu", Value: "false", Effect: corev1.TaintEffectNoSchedule}}}, + Status: ready, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "prefer-no-schedule"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "DeletionCandidateOfClusterAutoscaler", Effect: corev1.TaintEffectPreferNoSchedule}}}, + Status: ready, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "being-removed"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{ + {Key: "nvidia.com/gpu", Value: "true", Effect: corev1.TaintEffectNoSchedule}, + {Key: "ToBeDeletedByClusterAutoscaler", Effect: corev1.TaintEffectNoSchedule}, + }}, + Status: ready, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "gpu-no-execute"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "nvidia.com/gpu", Effect: corev1.TaintEffectNoExecute}}}, + Status: ready, + }, }, - { - ObjectMeta: metav1.ObjectMeta{Name: "being-removed"}, - Spec: corev1.NodeSpec{Taints: []corev1.Taint{ - {Key: "nvidia.com/gpu", Value: "true", Effect: corev1.TaintEffectNoSchedule}, - {Key: "ToBeDeletedByClusterAutoscaler", Effect: corev1.TaintEffectNoSchedule}, - }}, - Status: ready, + tolerations: []corev1.Toleration{ + {Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, + {Key: "amd.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, + {Key: "modelplane.ai/cpu", Operator: corev1.TolerationOpEqual, Value: "true", Effect: corev1.TaintEffectNoSchedule}, }, - { - ObjectMeta: metav1.ObjectMeta{Name: "gpu-no-execute"}, - Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "nvidia.com/gpu", Effect: corev1.TaintEffectNoExecute}}}, - Status: ready, + }, + want: Nodes{Total: 7, Ready: 7, Schedulable: 4}, + }, + "NoTolerations": { + reason: "A ready node with a NoSchedule taint shouldn't be schedulable without tolerations.", + args: args{ + nodes: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{Name: "nvidia-gpu"}, + Spec: corev1.NodeSpec{Taints: []corev1.Taint{{Key: "nvidia.com/gpu", Value: "true", Effect: corev1.TaintEffectNoSchedule}}}, + Status: ready, + }, }, }, - want: Nodes{Total: 4, Ready: 4, Schedulable: 2}, + want: Nodes{Total: 1, Ready: 1}, }, } for name, tc := range cases { t.Run(name, func(t *testing.T) { - got := CountNodes(tc.nodes) + got := CountNodes(tc.args.nodes, tc.args.tolerations) if diff := cmp.Diff(tc.want, got); diff != "" { t.Errorf("\n%s\nCountNodes(...): -want, +got:\n%s", tc.reason, diff) } @@ -490,6 +536,72 @@ func TestCountNodes(t *testing.T) { } } +func TestParseToleration(t *testing.T) { + type want struct { + t corev1.Toleration + err error + } + + cases := map[string]struct { + reason string + s string + want want + }{ + "AnyValue": { + reason: "A taint without a value should parse as a toleration of any value.", + s: "nvidia.com/gpu:NoSchedule", + want: want{t: corev1.Toleration{Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}}, + }, + "Value": { + reason: "A taint with a value should parse as a toleration of only that value.", + s: "modelplane.ai/cpu=true:NoExecute", + want: want{t: corev1.Toleration{Key: "modelplane.ai/cpu", Operator: corev1.TolerationOpEqual, Value: "true", Effect: corev1.TaintEffectNoExecute}}, + }, + "EmptyValue": { + reason: "A taint with an empty value should parse as a toleration of only the empty value.", + s: "dedicated=:NoSchedule", + want: want{t: corev1.Toleration{Key: "dedicated", Operator: corev1.TolerationOpEqual, Effect: corev1.TaintEffectNoSchedule}}, + }, + "PreferNoSchedule": { + reason: "A taint with effect PreferNoSchedule should be an error, because it never makes a node unschedulable.", + s: "nvidia.com/gpu:PreferNoSchedule", + want: want{err: cmpopts.AnyError}, + }, + "NoEffect": { + reason: "A taint without an effect should be an error.", + s: "nvidia.com/gpu", + want: want{err: cmpopts.AnyError}, + }, + "InvalidEffect": { + reason: "A taint with an effect Kubernetes doesn't define should be an error.", + s: "nvidia.com/gpu:noschedule", + want: want{err: cmpopts.AnyError}, + }, + "InvalidKey": { + reason: "A taint with a key that isn't a qualified name should be an error.", + s: ":NoSchedule", + want: want{err: cmpopts.AnyError}, + }, + "InvalidValue": { + reason: "A taint with a value that isn't a valid label value should be an error.", + s: "nvidia.com/gpu=a/b:NoSchedule", + want: want{err: cmpopts.AnyError}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + got, err := ParseToleration(tc.s) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("\n%s\nParseToleration(%q): -want error, +got error:\n%s", tc.reason, tc.s, diff) + } + if diff := cmp.Diff(tc.want.t, got); diff != "" { + t.Errorf("\n%s\nParseToleration(%q): -want, +got:\n%s", tc.reason, tc.s, diff) + } + }) + } +} + func TestExtendedResources(t *testing.T) { cases := map[string]struct { reason string