Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ status:
total: 1
ready: 1
schedulable: 1 # Ready, not cordoned, and not tainted against
# Modelplane's GPU pods.
# Modelplane's pods.
unschedulablePods: 1 # Pinned pods the scheduler can't place.
devices:
- deviceClassName: gpu.nvidia.com
Expand Down
3 changes: 1 addition & 2 deletions apis/v1alpha1/poolobservation_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,7 @@ type NodeCounts struct {
Ready int32 `json:"ready"`

// Schedulable counts the ready nodes that aren't cordoned, and carry no
// NoSchedule or NoExecute taint but nvidia.com/gpu with NoSchedule, which
// Modelplane's GPU pods tolerate.
// NoSchedule or NoExecute taint that Modelplane's pods don't tolerate.
//
// +kubebuilder:validation:Minimum=0
Schedulable int32 `json:"schedulable"`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -283,8 +283,7 @@ spec:
schedulable:
description: |-
Schedulable counts the ready nodes that aren't cordoned, and carry no
NoSchedule or NoExecute taint but nvidia.com/gpu with NoSchedule, which
Modelplane's GPU pods tolerate.
NoSchedule or NoExecute taint that Modelplane's pods don't tolerate.
format: int32
minimum: 0
type: integer
Expand Down
1 change: 1 addition & 0 deletions cluster/charts/cluster-agent/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ spec:
imagePullPolicy: {{ .Values.image.pullPolicy }}
args:
- --observation-delay={{ .Values.observationDelay }}
- --tolerated-taints={{ join "," .Values.toleratedTaints }}
{{- if .Values.debug }}
- --debug
{{- end }}
Expand Down
8 changes: 8 additions & 0 deletions cluster/charts/cluster-agent/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,14 @@ debug: false
# Changes made while waiting are observed together.
observationDelay: 10s

# Taints Modelplane's pods tolerate, as key[=value]:effect. A taint without a
# value matches any value. A node with any other NoSchedule or NoExecute taint
# doesn't count as schedulable.
toleratedTaints:
- nvidia.com/gpu:NoSchedule
- amd.com/gpu:NoSchedule
- modelplane.ai/cpu:NoSchedule

# The agent caches the nodes, pods and ResourceSlices it observes, so its
# memory grows with the cluster. It uses about 50Mi on a cluster with nothing
# to observe. There's no limit by default.
Expand Down
17 changes: 15 additions & 2 deletions cmd/cluster-agent/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ type cli struct {
MetricsBindAddress string `default:":8080" help:"The address the metrics endpoint binds to."`
HealthProbeBindAddress string `default:":8081" help:"The address the health probe endpoint binds to."`
LeaderElectionNamespace string `help:"The namespace of the Lease that elects one agent to run. Defaults to the agent's own namespace."`
ToleratedTaints []string `default:"nvidia.com/gpu:NoSchedule,amd.com/gpu:NoSchedule,modelplane.ai/cpu:NoSchedule" help:"Taints Modelplane's pods tolerate, as key[=value]:effect. A taint without a value matches any value. A node with any other NoSchedule or NoExecute taint doesn't count as schedulable."`
}

func main() {
Expand All @@ -64,6 +65,13 @@ func main() {
kong.UsageOnError(),
)

tolerations := make([]corev1.Toleration, len(flags.ToleratedTaints))
for i, s := range flags.ToleratedTaints {
t, err := pool.ParseToleration(s)
kctx.FatalIfErrorf(err, "cannot parse --tolerated-taints")
tolerations[i] = t
}

level := slog.LevelInfo
if flags.Debug {
level = slog.LevelDebug
Expand Down Expand Up @@ -119,8 +127,13 @@ func main() {
_, err = claims.AddEventHandler(allocation.ForgetDeleted(record))
kctx.FatalIfErrorf(err, "cannot add ResourceClaim event handler")

kctx.FatalIfErrorf(poolobservation.Setup(mgr, record, log, flags.ObservationDelay), "cannot set up PoolObservation controller")
kctx.FatalIfErrorf(replicaobservation.Setup(mgr, record, log), "cannot set up ReplicaObservation controller")
kctx.FatalIfErrorf(poolobservation.Setup(mgr, record, flags.ObservationDelay,
poolobservation.WithPoolObserver(pool.NewObserver(pool.WithTolerations(tolerations))),
poolobservation.WithLogger(log.With("controller", "poolobservation")),
), "cannot set up PoolObservation controller")
kctx.FatalIfErrorf(replicaobservation.Setup(mgr, record,
replicaobservation.WithLogger(log.With("controller", "replicaobservation")),
), "cannot set up ReplicaObservation controller")

kctx.FatalIfErrorf(mgr.Start(ctx), "cannot run manager")
}
Expand Down
17 changes: 9 additions & 8 deletions internal/controller/poolobservation/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

// Package poolobservation reports the state of a GPU pool in the status of a
// Package poolobservation reports the state of a pool in the status of a
// PoolObservation.
package poolobservation

Expand Down Expand Up @@ -57,7 +57,7 @@ const (
FieldOwner = "cluster-agent.modelplane.ai"
)

// A PoolObserver observes a GPU pool. It must not modify its inputs, which
// A PoolObserver observes a pool. It must not modify its inputs, which
// the Reconciler reads straight from its cache.
type PoolObserver interface {
ObservePool(ctx context.Context, name string, in pool.Inputs) (pool.Observation, error)
Expand All @@ -71,7 +71,7 @@ func (fn PoolObserverFn) ObservePool(ctx context.Context, name string, in pool.I
return fn(ctx, name, in)
}

// A Reconciler reports the state of a GPU pool in the status of a
// A Reconciler reports the state of a pool in the status of a
// PoolObservation.
type Reconciler struct {
client client.Client
Expand All @@ -89,7 +89,7 @@ func WithLogger(l *slog.Logger) ReconcilerOption {
}

// WithPoolObserver sets how the Reconciler observes pools. It uses a
// pool.Observer by default.
// pool.Observer that tolerates no taints by default.
func WithPoolObserver(o PoolObserver) ReconcilerOption {
return func(r *Reconciler) { r.observer = o }
}
Expand All @@ -112,8 +112,9 @@ func NewReconciler(c client.Client, record *allocation.Record, opts ...Reconcile
// Setup adds a PoolObservation controller to the manager. The controller
// observes every pool whenever an object it reads changes, but waits for the
// supplied delay first, so that a burst of changes produces one observation
// of each pool. It reads ResourceClaims through the supplied Record.
func Setup(mgr ctrl.Manager, record *allocation.Record, log *slog.Logger, delay time.Duration) error {
// of each pool. It reads ResourceClaims through the supplied Record, and
// configures its Reconciler with the supplied options.
func Setup(mgr ctrl.Manager, record *allocation.Record, delay time.Duration, opts ...ReconcilerOption) error {
c := mgr.GetClient()
enqueue := EnqueueAllAfter(c, delay)
err := ctrl.NewControllerManagedBy(mgr).
Expand All @@ -125,7 +126,7 @@ func Setup(mgr ctrl.Manager, record *allocation.Record, log *slog.Logger, delay
Watches(&resourcev1.ResourceClaim{}, enqueue).
Watches(&corev1.Pod{}, enqueue).
Watches(&corev1.Event{}, enqueue).
Complete(NewReconciler(c, record, WithLogger(log.With("controller", "poolobservation"))))
Complete(NewReconciler(c, record, opts...))
if err != nil {
return fmt.Errorf("cannot create controller: %w", err)
}
Expand Down Expand Up @@ -155,7 +156,7 @@ func EnqueueAllAfter(c client.Reader, delay time.Duration) handler.Funcs {
}
}

// Reconcile observes a GPU pool and reports it in the status of the requested
// Reconcile observes a pool and reports it in the status of the requested
// PoolObservation.
func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
log := r.log.With("request", req)
Expand Down
7 changes: 4 additions & 3 deletions internal/controller/replicaobservation/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,9 @@ func NewReconciler(c client.Client, record *allocation.Record, opts ...Reconcile
}

// Setup adds a ReplicaObservation controller to the manager. It reads
// ResourceClaims through the supplied Record.
func Setup(mgr ctrl.Manager, record *allocation.Record, log *slog.Logger) error {
// ResourceClaims through the supplied Record, and configures its Reconciler
// with the supplied options.
func Setup(mgr ctrl.Manager, record *allocation.Record, opts ...ReconcilerOption) error {
c := mgr.GetClient()
err := ctrl.NewControllerManagedBy(mgr).
Named("replicaobservation").
Expand All @@ -126,7 +127,7 @@ func Setup(mgr ctrl.Manager, record *allocation.Record, log *slog.Logger) error
// but the claim cache can lag the pod cache. Watching claims catches
// an allocation the pod's binding raced ahead of.
Watches(&resourcev1.ResourceClaim{}, handler.EnqueueRequestsFromMapFunc(ClaimRequests(c))).
Complete(NewReconciler(c, record, WithLogger(log.With("controller", "replicaobservation"))))
Complete(NewReconciler(c, record, opts...))
if err != nil {
return fmt.Errorf("cannot create controller: %w", err)
}
Expand Down
101 changes: 69 additions & 32 deletions internal/pool/pool.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

// Package pool observes a GPU pool: the nodes Modelplane labels with the
// Package pool observes a pool: the nodes Modelplane labels with the
// pool's name, the DRA devices they publish, and the pods waiting for them.
package pool

Expand All @@ -27,10 +27,12 @@ import (
"strings"
"time"

"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
resourcev1 "k8s.io/api/resource/v1"
"k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/dynamic-resource-allocation/cel"

"github.com/modelplaneai/cluster-agent/internal/dra"
Expand All @@ -40,10 +42,6 @@ import (
// Modelplane pins a pod to a pool with a node selector on it.
const LabelPool = "modelplane.ai/pool"

// TaintGPU is the taint Modelplane puts on the GPU pools it provisions. Its GPU
// pods tolerate it with effect NoSchedule.
const TaintGPU = "nvidia.com/gpu"

// Cluster autoscaler pod event reasons. See
// https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-events-are-emitted-by-ca
const (
Expand Down Expand Up @@ -111,8 +109,8 @@ type Nodes struct {
// Cordoned counts the nodes marked unschedulable, ready or not.
Cordoned int32

// Schedulable counts the nodes Modelplane's GPU pods can schedule onto.
// See IsSchedulable.
// Schedulable counts the nodes Modelplane's pods can schedule onto. See
// IsSchedulable.
Schedulable int32
}

Expand Down Expand Up @@ -179,24 +177,39 @@ type Membership struct {
// Nodes is the set of the pool's nodes.
Nodes map[string]bool

// Schedulable is the set of the pool's nodes that Modelplane's GPU pods
// can schedule onto. See IsSchedulable.
// Schedulable is the set of the pool's nodes that Modelplane's pods can
// schedule onto. See IsSchedulable.
Schedulable map[string]bool
}

// An Observer observes pools.
type Observer struct {
cel *cel.Cache
cel *cel.Cache
tolerations []corev1.Toleration
}

// An ObserverOption configures an Observer.
type ObserverOption func(*Observer)

// WithTolerations sets the tolerations Modelplane's pods carry. The Observer
// counts a node as schedulable only if they tolerate its taints. It tolerates
// no taints by default.
func WithTolerations(t []corev1.Toleration) ObserverOption {
return func(o *Observer) { o.tolerations = t }
}

// NewObserver returns an Observer.
func NewObserver() *Observer {
func NewObserver(opts ...ObserverOption) *Observer {
// Enable every CEL feature so the Observer can compile any expression
// the API server accepted.
return &Observer{cel: cel.NewCache(celCacheSize, cel.Features{
o := &Observer{cel: cel.NewCache(celCacheSize, cel.Features{
EnableConsumableCapacity: true,
EnableListTypeAttributes: true,
})}
for _, fn := range opts {
fn(o)
}
return o
}

// ObservePool returns what's true of the named pool. A pool's nodes are those
Expand All @@ -208,15 +221,15 @@ func (o *Observer) ObservePool(ctx context.Context, name string, in Inputs) (Obs
nodes = append(nodes, n)
}
}
m := Members(nodes)
m := Members(nodes, o.tolerations)
devices := dra.Devices(in.Slices)

byClass, err := o.DevicesByClass(ctx, devices, in.Classes, Allocated(in.Claims), m)
if err != nil {
return Observation{}, err
}
return Observation{
Nodes: CountNodes(nodes),
Nodes: CountNodes(nodes, o.tolerations),
Devices: byClass,
Free: Free(byClass, m),
LastAllocation: LastAllocation(in.Claims, in.AllocatedAt, devices, m),
Expand All @@ -227,20 +240,21 @@ func (o *Observer) ObservePool(ctx context.Context, name string, in Inputs) (Obs
}

// Members returns the supplied nodes as a pool's members, and which of them
// are schedulable.
func Members(nodes []corev1.Node) Membership {
// are schedulable by pods with the supplied tolerations.
func Members(nodes []corev1.Node, ts []corev1.Toleration) Membership {
m := Membership{Nodes: map[string]bool{}, Schedulable: map[string]bool{}}
for _, n := range nodes {
m.Nodes[n.Name] = true
if IsSchedulable(n) {
if IsSchedulable(n, ts) {
m.Schedulable[n.Name] = true
}
}
return m
}

// CountNodes counts a pool's nodes.
func CountNodes(nodes []corev1.Node) Nodes {
// CountNodes counts a pool's nodes, counting a node as schedulable if pods with
// the supplied tolerations can schedule onto it.
func CountNodes(nodes []corev1.Node, ts []corev1.Toleration) Nodes {
c := Nodes{}
for _, n := range nodes {
c.Total++
Expand All @@ -250,7 +264,7 @@ func CountNodes(nodes []corev1.Node) Nodes {
if IsReady(n) {
c.Ready++
}
if IsSchedulable(n) {
if IsSchedulable(n, ts) {
c.Schedulable++
}
}
Expand Down Expand Up @@ -453,28 +467,51 @@ func IsReady(n corev1.Node) bool {
return false
}

// IsSchedulable returns true if Modelplane's GPU pods can schedule onto the
// node: it's ready, it isn't cordoned, and every NoSchedule or NoExecute taint
// it carries is one they tolerate.
func IsSchedulable(n corev1.Node) bool {
// IsSchedulable returns true if pods with the supplied tolerations can
// schedule onto the node: it's ready, it isn't cordoned, and every NoSchedule
// or NoExecute taint it carries is one they tolerate.
func IsSchedulable(n corev1.Node, ts []corev1.Toleration) bool {
if !IsReady(n) || n.Spec.Unschedulable {
return false
}
for _, t := range n.Spec.Taints {
if t.Effect == corev1.TaintEffectPreferNoSchedule {
for _, taint := range n.Spec.Taints {
if taint.Effect == corev1.TaintEffectPreferNoSchedule {
continue
}
// compose-model-replica gives Modelplane's GPU pods this one
// toleration (_GPU_TOLERATION in
// functions/compose-model-replica/function/backends/base.py).
if t.Key == TaintGPU && t.Effect == corev1.TaintEffectNoSchedule {
continue
// ToleratesTaint only logs when comparing a Lt or Gt toleration's
// value, which it won't do with comparison operators disabled.
if !slices.ContainsFunc(ts, func(t corev1.Toleration) bool { return t.ToleratesTaint(logr.Discard(), &taint, false) }) {
return false
}
return false
}
return true
}

// ParseToleration parses a toleration of the taints written key[=value]:effect.
// The toleration tolerates taints with the key and effect. It tolerates only
// the value if there is one, and any value if there isn't. The effect must be
// NoSchedule or NoExecute, the effects IsSchedulable considers.
func ParseToleration(s string) (corev1.Toleration, error) {
kv, effect, ok := strings.Cut(s, ":")
if !ok {
return corev1.Toleration{}, fmt.Errorf("toleration %q has no effect, want key[=value]:effect", s)
}
t := corev1.Toleration{Key: kv, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffect(effect)}
if key, value, ok := strings.Cut(kv, "="); ok {
t = corev1.Toleration{Key: key, Operator: corev1.TolerationOpEqual, Value: value, Effect: t.Effect}
}
if errs := validation.IsQualifiedName(t.Key); len(errs) > 0 {
return corev1.Toleration{}, fmt.Errorf("toleration %q has an invalid key: %s", s, strings.Join(errs, ", "))
}
if errs := validation.IsValidLabelValue(t.Value); len(errs) > 0 {
return corev1.Toleration{}, fmt.Errorf("toleration %q has an invalid value: %s", s, strings.Join(errs, ", "))
}
if t.Effect != corev1.TaintEffectNoSchedule && t.Effect != corev1.TaintEffectNoExecute {
return corev1.Toleration{}, fmt.Errorf("toleration %q has an invalid effect, want NoSchedule or NoExecute", s)
}
return t, nil
}

// IsUnschedulable returns true if the scheduler couldn't find a node for the
// pod.
func IsUnschedulable(p corev1.Pod) bool {
Expand Down
Loading
Loading