diff --git a/apis/eksclusters/definition.yaml b/apis/eksclusters/definition.yaml index e3718703b..2abeaac9b 100644 --- a/apis/eksclusters/definition.yaml +++ b/apis/eksclusters/definition.yaml @@ -162,8 +162,12 @@ spec: type: integer default: 0 description: >- - Minimum number of nodes for autoscaling. Set to 1 or - higher for groups that must always be available. + Minimum number of nodes for autoscaling. Set it to 1 + or higher for groups that must always be available, + and for GPU and CPU groups: the upstream cluster + autoscaler's AWS provider can't scale a group whose + devices bind through DRA up from zero nodes, so a pod + scheduled to an empty group waits forever. minimum: 0 maximum: 1000 maxNodeCount: diff --git a/apis/inferenceclusters/definition.yaml b/apis/inferenceclusters/definition.yaml index da4b159df..0249c9381 100644 --- a/apis/inferenceclusters/definition.yaml +++ b/apis/inferenceclusters/definition.yaml @@ -528,6 +528,13 @@ spec: minimum: 0 minNodeCount: type: integer + description: >- + Minimum node count for autoscaling. Keep GPU pools on + EKS, AKS and CoreWeave, and CPU pools on EKS and + CoreWeave, at 1 or higher: their cluster autoscalers + can't scale a pool whose devices bind through DRA up + from zero nodes, so a pod scheduled to an empty pool + waits forever. minimum: 0 maxNodeCount: type: integer diff --git a/docs/manifests/recipes/llama-3.1-8b/inference-cluster-eks.yaml b/docs/manifests/recipes/llama-3.1-8b/inference-cluster-eks.yaml index 8ce473601..c31e23d04 100644 --- a/docs/manifests/recipes/llama-3.1-8b/inference-cluster-eks.yaml +++ b/docs/manifests/recipes/llama-3.1-8b/inference-cluster-eks.yaml @@ -19,5 +19,5 @@ spec: nodeCount: 1 zones: - us-west-2a - minNodeCount: 0 + minNodeCount: 1 maxNodeCount: 4 diff --git a/docs/manifests/reference/eksclusters.yaml b/docs/manifests/reference/eksclusters.yaml index dadc5eedb..3ef68892b 100644 --- a/docs/manifests/reference/eksclusters.yaml +++ b/docs/manifests/reference/eksclusters.yaml @@ -15,7 +15,10 @@ spec: role: GPU instanceType: g6.12xlarge diskSizeGb: 200 - nodeCount: 0 + # GPU groups need at least one node: the cluster autoscaler cannot + # scale a group whose GPUs bind through DRA up from zero. + nodeCount: 1 + minNodeCount: 1 maxNodeCount: 8 zones: [us-west-2a, us-west-2b] gpu: diff --git a/schemas/.lock.json b/schemas/.lock.json index f5e193428..fa75db8dc 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,6 +1,6 @@ { "packages": { - "fs://apis": "7765fe0a737410e9a3f3504455b088e8d2c9ba91fb33046a56acb5441f1ce741", + "fs://apis": "44bfccfab3ee8fdaccac6139473f1f96bb3f1a7d21fc3709b962bb6f3599b962", "git://https://github.com/crossplane/crossplane/cluster/crds": "14c53aac914d393b5bcc00962fbc90d4be5d35bc", "git://https://github.com/modelplaneai/cluster-agent/cluster/charts/cluster-agent/files/crds": "a833d3f6f7d9a6ac6083e7453375b58b65ee741a", "xpkg://xpkg.upbound.io/upbound/provider-aws-ec2:v2.9.4": "sha256:7a0febdfb8da2d59a9fdcdbc2d476811f00566fcb941880313fc2034b0ca6177", diff --git a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py index 09fcebc51..7a806c981 100644 --- a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py @@ -270,6 +270,9 @@ class NodePool(BaseModel): Maximum node count for autoscaling. Omit for fixed-size pools. """ minNodeCount: conint(ge=0) | None = None + """ + Minimum node count for autoscaling. Keep GPU pools on EKS, AKS and CoreWeave, and CPU pools on EKS and CoreWeave, at 1 or higher: their cluster autoscalers can't scale a pool whose devices bind through DRA up from zero nodes, so a pod scheduled to an empty pool waits forever. + """ name: constr(pattern=r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$', max_length=40) """ Name of the pool. Its nodes carry it as the value of the modelplane.ai/pool label, and the cluster agent observes the pool through an object of this name. diff --git a/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py index 37ce4eefa..0d12af18c 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py @@ -117,7 +117,7 @@ class NodePool(BaseModel): """ minNodeCount: conint(ge=0, le=1000) | None = 0 """ - Minimum number of nodes for autoscaling. Set to 1 or higher for groups that must always be available. + Minimum number of nodes for autoscaling. Set it to 1 or higher for groups that must always be available, and for GPU and CPU groups: the upstream cluster autoscaler's AWS provider can't scale a group whose devices bind through DRA up from zero nodes, so a pod scheduled to an empty group waits forever. """ name: constr(min_length=1, max_length=40) """