diff --git a/apis/aksclusters/definition.yaml b/apis/aksclusters/definition.yaml index 3cec68b50..caa724e73 100644 --- a/apis/aksclusters/definition.yaml +++ b/apis/aksclusters/definition.yaml @@ -219,6 +219,45 @@ spec: status: description: AKSClusterStatus defines the observed state of AKSCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools. + The system node pool Modelplane adds belongs to the cluster + itself, so its state is part of the ClusterReady condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/civoclusters/definition.yaml b/apis/civoclusters/definition.yaml index 6200f7336..262b92314 100644 --- a/apis/civoclusters/definition.yaml +++ b/apis/civoclusters/definition.yaml @@ -177,6 +177,45 @@ spec: status: description: CivoClusterStatus defines the observed state of CivoCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools. + The system node pool Modelplane adds belongs to the cluster + itself, so its state is part of the ClusterReady condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/eksclusters/definition.yaml b/apis/eksclusters/definition.yaml index 8bfa2dddd..1b4db61fb 100644 --- a/apis/eksclusters/definition.yaml +++ b/apis/eksclusters/definition.yaml @@ -247,6 +247,45 @@ spec: status: description: EKSClusterStatus defines the observed state of EKSCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools. + The system node pool Modelplane adds belongs to the cluster + itself, so its state is part of the ClusterReady condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/gkeclusters/definition.yaml b/apis/gkeclusters/definition.yaml index 588edc7d0..13e5d5008 100644 --- a/apis/gkeclusters/definition.yaml +++ b/apis/gkeclusters/definition.yaml @@ -201,6 +201,45 @@ spec: status: description: GKEClusterStatus defines the observed state of GKECluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools. + The system node pool Modelplane adds belongs to the cluster + itself, so its state is part of the ClusterReady condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/inferenceclusters/definition.yaml b/apis/inferenceclusters/definition.yaml index 0e0c8ca3c..9cdfd6459 100644 --- a/apis/inferenceclusters/definition.yaml +++ b/apis/inferenceclusters/definition.yaml @@ -581,6 +581,39 @@ spec: status: type: object properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools, + as the cluster Modelplane provisions reports it. Not set + when source is Existing, because Modelplane doesn't + provision those pools. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [name] + items: + type: object + required: [name, state] + properties: + name: + type: string + maxLength: 40 + description: Name of the node pool in spec.nodePools. + state: + type: string + enum: [Pending, Provisioning, Ready, Failed] + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. providerConfigRef: type: object properties: diff --git a/apis/nebiusclusters/definition.yaml b/apis/nebiusclusters/definition.yaml index 171592e49..daeb3ac55 100644 --- a/apis/nebiusclusters/definition.yaml +++ b/apis/nebiusclusters/definition.yaml @@ -241,6 +241,45 @@ spec: status: description: NebiusClusterStatus defines the observed state of NebiusCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools. + The system node pool Modelplane adds belongs to the cluster + itself, so its state is part of the ClusterReady condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/vultrclusters/definition.yaml b/apis/vultrclusters/definition.yaml index 26346cdee..cacbce83c 100644 --- a/apis/vultrclusters/definition.yaml +++ b/apis/vultrclusters/definition.yaml @@ -171,6 +171,45 @@ spec: status: description: VultrClusterStatus defines the observed state of VultrCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools. + The system node pool Modelplane adds belongs to the cluster + itself, so its state is part of the ClusterReady condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/functions/compose-inference-cluster/function/fn.py b/functions/compose-inference-cluster/function/fn.py index 822c54424..5e94b74fa 100644 --- a/functions/compose-inference-cluster/function/fn.py +++ b/functions/compose-inference-cluster/function/fn.py @@ -114,6 +114,9 @@ # Composed resource key for the backend XR. BACKEND_RESOURCE_KEY = "serving-stack" +# Composed resource key for the ClusterProviderConfig that reaches the cluster. +CLUSTER_PROVIDER_CONFIG_RESOURCE_KEY = "cluster-provider-config-kubernetes" + # Composed resource key for the ClusterUsage that blocks the InferenceCluster's # deletion while anything composes objects onto it. # @@ -358,6 +361,8 @@ def compose(self) -> None: source = cluster.source if not self.check_devices_supported(source): return + if not self.check_pools_provisionable(source): + return if source == CLUSTER_SOURCE_GKE: self.compose_gke(cluster.gke) @@ -779,6 +784,37 @@ def check_devices_supported(self, source: Cloud) -> bool: response.fatal(self.rsp, msg) return False + def check_pools_provisionable(self, source: Cloud) -> bool: + """Gate on a node pool whose class has no provisioning block for the cloud. + + The cloud cluster XR can't build a pool without one. Gating is a fatal + result for the same reason as check_devices_supported: a running + cluster whose shared class loses its block holds its resources rather + than having Crossplane collect them. Every offending class is named. + Existing never gates, because it provisions nothing. + """ + if source == CLUSTER_SOURCE_EXISTING: + return True + offending = [ + f"InferenceClass {name} has no {source} provisioning block" + for name in sorted({p.className for p in self.xr.spec.nodePools or []}) + if getattr(self.classes[name].spec.provisioning, source.lower(), None) is None + ] + if not offending: + return True + msg = "; ".join(offending) + response.set_conditions( + self.rsp, + resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_INVALID_NODE_POOL, + message=msg, + ), + ) + response.fatal(self.rsp, msg) + return False + def compose_gke(self, gke: v1alpha1.Gke | None) -> None: """Compose an InferenceCluster backed by a Modelplane-provisioned GKE cluster. Composes the GKECluster XR, waits for it to be ready, @@ -797,7 +833,7 @@ def compose_gke(self, gke: v1alpha1.Gke | None) -> None: sa_key = self.observed_gke_secret(_IDENTITY_TYPE_GCP) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if gke_ready and kubeconfig_secret: + if kubeconfig_secret and self.keep_cluster_provider_config(cluster_ready=gke_ready): self.compose_cluster_provider_config( kubeconfig_secret.name, kubeconfig_secret.key, @@ -843,7 +879,7 @@ def compose_eks(self, eks: v1alpha1.Eks | None) -> None: kubeconfig = self.observed_eks_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if eks_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=eks_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_eks_backend_secrets(eks_ready=eks_ready, backend_exists=backend_exists) @@ -882,7 +918,7 @@ def compose_aks(self, aks: v1alpha1.Aks | None) -> None: kubeconfig = self.observed_aks_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if aks_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=aks_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_aks_backend_secrets(aks_ready=aks_ready, backend_exists=backend_exists) @@ -926,7 +962,7 @@ def compose_nebius(self, nebius: v1alpha1.Nebius | None) -> None: credentials = self.observed_nebius_secret(_IDENTITY_TYPE_NEBIUS) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if nebius_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=nebius_ready): self.compose_cluster_provider_config( kubeconfig.name, kubeconfig.key, @@ -970,7 +1006,7 @@ def compose_vultr(self, vultr: v1alpha1.Vultr | None) -> None: kubeconfig = self.observed_vultr_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if vultr_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=vultr_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_vultr_backend_secrets(vultr_ready=vultr_ready, backend_exists=backend_exists) @@ -1009,7 +1045,7 @@ def compose_civo(self, civo: v1alpha1.Civo | None) -> None: kubeconfig = self.observed_civo_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if civo_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=civo_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_civo_backend_secrets(civo_ready=civo_ready, backend_exists=backend_exists) @@ -1109,6 +1145,18 @@ def compose_serving_stack( ), ) + def keep_cluster_provider_config(self, *, cluster_ready: bool) -> bool: + """Whether to compose the ClusterProviderConfig reaching a provisioned cluster. + + It's first composed once the cloud cluster XR is Ready, then kept while + it's observed. The cloud cluster XR goes unready whenever any resource it + composes does, and every Object on the cluster - its namespaces, + PoolObservations, and every ModelReplica's and ModelRoute's - reaches it + through this ProviderConfig, so dropping it then would have Crossplane + delete a ProviderConfig they all still use. + """ + return cluster_ready or CLUSTER_PROVIDER_CONFIG_RESOURCE_KEY in self.req.observed.resources + def compose_cluster_provider_config( self, kubeconfig_name: str, @@ -1151,10 +1199,10 @@ def compose_cluster_provider_config( ), ) resource.update( - self.rsp.desired.resources["cluster-provider-config-kubernetes"], + self.rsp.desired.resources[CLUSTER_PROVIDER_CONFIG_RESOURCE_KEY], cpc, ) - self.rsp.desired.resources["cluster-provider-config-kubernetes"].ready = fnv1.READY_TRUE + self.rsp.desired.resources[CLUSTER_PROVIDER_CONFIG_RESOURCE_KEY].ready = fnv1.READY_TRUE def write_status(self, schedulable_pools: list[v1alpha1.SchedulablePool]) -> None: """Write the InferenceCluster status.""" @@ -1248,21 +1296,12 @@ def compose_gke_cluster(self, gke: v1alpha1.Gke) -> None: gke_node_pools: list[gkev1alpha1.NodePool] = [] for pool in self.xr.spec.nodePools or []: - cls = self.classes.get(pool.className) - if not cls or not cls.spec.provisioning or not cls.spec.provisioning.gke: - msg = f"InferenceClass {pool.className} has no GKE provisioning block" - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="False", - reason=CONDITION_REASON_INVALID_NODE_POOL, - message=msg, - ), - ) - response.warning(self.rsp, msg) - return - prov = cls.spec.provisioning.gke + provisioning = self.classes[pool.className].spec.provisioning + # Unreachable: check_pools_provisionable gates on every pool's + # class having this block. + if not provisioning or not provisioning.gke: + continue + prov = provisioning.gke # A class with no accelerator describes a CPU pool: a standard # node image, no GPU attachment, and the modelplane.ai/cpu # label and taint composed by compose-gke-cluster. @@ -1319,21 +1358,11 @@ def compose_eks_cluster(self, eks: v1alpha1.Eks) -> None: eks_node_pools: list[eksv1alpha1.NodePool] = [] for pool in self.xr.spec.nodePools or []: - cls = self.classes.get(pool.className) - if not cls or not cls.spec.provisioning or not cls.spec.provisioning.eks: - msg = f"InferenceClass {pool.className} has no EKS provisioning block" - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="False", - reason=CONDITION_REASON_INVALID_NODE_POOL, - message=msg, - ), - ) - response.warning(self.rsp, msg) - return - prov = cls.spec.provisioning.eks + provisioning = self.classes[pool.className].spec.provisioning + # Unreachable (see the GKE builder above). + if not provisioning or not provisioning.eks: + continue + prov = provisioning.eks # A class with no accelerator describes a CPU pool (see the GKE # builder above). node_pool = eksv1alpha1.NodePool( @@ -1398,21 +1427,11 @@ def compose_aks_cluster(self, aks: v1alpha1.Aks) -> None: aks_node_pools: list[aksv1alpha1.NodePool] = [] for pool in self.xr.spec.nodePools or []: - cls = self.classes.get(pool.className) - if not cls or not cls.spec.provisioning or not cls.spec.provisioning.aks: - msg = f"InferenceClass {pool.className} has no AKS provisioning block" - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="False", - reason=CONDITION_REASON_INVALID_NODE_POOL, - message=msg, - ), - ) - response.warning(self.rsp, msg) - return - prov = cls.spec.provisioning.aks + provisioning = self.classes[pool.className].spec.provisioning + # Unreachable (see the GKE builder above). + if not provisioning or not provisioning.aks: + continue + prov = provisioning.aks # A class with no accelerator describes a CPU pool (see the GKE # builder above). node_pool = aksv1alpha1.NodePool( @@ -1472,21 +1491,11 @@ def compose_nebius_cluster(self, nebius: v1alpha1.Nebius) -> None: nebius_node_pools: list[nebiusv1alpha1.NodePool] = [] for pool in self.xr.spec.nodePools or []: - cls = self.classes.get(pool.className) - if not cls or not cls.spec.provisioning or not cls.spec.provisioning.nebius: - msg = f"InferenceClass {pool.className} has no Nebius provisioning block" - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="False", - reason=CONDITION_REASON_INVALID_NODE_POOL, - message=msg, - ), - ) - response.warning(self.rsp, msg) - return - prov = cls.spec.provisioning.nebius + provisioning = self.classes[pool.className].spec.provisioning + # Unreachable (see the GKE builder above). + if not provisioning or not provisioning.nebius: + continue + prov = provisioning.nebius # A class with no accelerator describes a CPU pool (see the GKE # builder above). driversPreset rides the gpu block: a CPU # platform preinstalls no NVIDIA stack. @@ -1633,21 +1642,11 @@ def compose_vultr_cluster(self, vultr: v1alpha1.Vultr) -> None: vultr_node_pools: list[vultrv1alpha1.NodePool] = [] for pool in self.xr.spec.nodePools or []: - cls = self.classes.get(pool.className) - if not cls or not cls.spec.provisioning or not cls.spec.provisioning.vultr: - msg = f"InferenceClass {pool.className} has no Vultr provisioning block" - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="False", - reason=CONDITION_REASON_INVALID_NODE_POOL, - message=msg, - ), - ) - response.warning(self.rsp, msg) - return - prov = cls.spec.provisioning.vultr + provisioning = self.classes[pool.className].spec.provisioning + # Unreachable (see the GKE builder above). + if not provisioning or not provisioning.vultr: + continue + prov = provisioning.vultr # A class with no accelerator describes a CPU pool (see the GKE # builder above). node_pool = vultrv1alpha1.NodePool( @@ -1762,7 +1761,7 @@ def civo_nvlink_disabled_pools(self) -> list[str]: setting to each listed pool through a dedicated NVIDIADriver, so multi-GPU pools in the same cluster keep NVLink between their GPUs. Pools whose class lacks a Civo provisioning block are - skipped here and reported by compose_civo_cluster.""" + skipped here; check_pools_provisionable gates on them.""" pools: list[str] = [] for pool in self.xr.spec.nodePools or []: cls = self.classes.get(pool.className) @@ -1783,21 +1782,11 @@ def compose_civo_cluster(self, civo: v1alpha1.Civo) -> None: civo_node_pools: list[civov1alpha1.NodePool] = [] for pool in self.xr.spec.nodePools or []: - cls = self.classes.get(pool.className) - if not cls or not cls.spec.provisioning or not cls.spec.provisioning.civo: - msg = f"InferenceClass {pool.className} has no Civo provisioning block" - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="False", - reason=CONDITION_REASON_INVALID_NODE_POOL, - message=msg, - ), - ) - response.warning(self.rsp, msg) - return - prov = cls.spec.provisioning.civo + provisioning = self.classes[pool.className].spec.provisioning + # Unreachable (see the GKE builder above). + if not provisioning or not provisioning.civo: + continue + prov = provisioning.civo # A class with no accelerator describes a CPU pool (see the GKE # builder above). node_pool = civov1alpha1.NodePool( diff --git a/functions/compose-inference-cluster/tests/test_fn.py b/functions/compose-inference-cluster/tests/test_fn.py index 6a9e7104d..d336f010a 100644 --- a/functions/compose-inference-cluster/tests/test_fn.py +++ b/functions/compose-inference-cluster/tests/test_fn.py @@ -2689,6 +2689,234 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + # An EKSCluster that was Ready goes unready whenever one of its resources + # does. Every Object on the cluster still reaches it through the + # ClusterProviderConfig, so dropping it would have Crossplane delete it from + # under them. + ComposeCase( + name="EKSUnreadyKeepsCPC", + reason="With its EKSCluster gone unready after composing the ClusterProviderConfig and ServingStack, an EKS cluster keeps both.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster(source="EKS", eks=v1alpha1.Eks(region="us-west-2")), + node_pools=[ + v1alpha1.NodePool( + name="l4-pool", + className="gpu-l4-eks", + nodeCount=2, + maxNodeCount=4, + zones=["us-west-2a", "us-west-2b"], + ), + ], + ), + resources={ + "eks-cluster": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "EKSCluster", + "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "spec": { + "region": "us-west-2", + "nodePools": [ + {"name": "l4-pool", "role": "GPU", "instanceType": "g6.xlarge", "nodeCount": 2}, + ], + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "Creating", + "message": "Unready resources: nodegroup-l4-pool", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + "secrets": [ + { + "type": "Kubeconfig", + "name": "test-cluster-kubeconfig-abcde", + "key": "kubeconfig", + }, + ], + "cache": {"storageClassName": "modelplane-rwx-efs"}, + }, + } + ), + ), + "cluster-provider-config-kubernetes": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "ClusterProviderConfig", + "metadata": {"name": "test-cluster-cluster-kubeconfig-d0f89"}, + } + ), + ), + "serving-stack": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "ServingStack", + "metadata": {"name": "test-cluster-serving-stack-fd00b"}, + "spec": { + "secrets": [ + { + "type": "Kubeconfig", + "name": "test-cluster-kubeconfig-abcde", + "key": "kubeconfig", + }, + ], + }, + } + ), + ), + "activation": _observed_activation_policy( + activated=[ + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", + ] + ), + }, + ), + required_resources={ + "class-gpu-l4-eks": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4-eks", + count=1, + memory="24Gi", + provisioning={ + "provider": "EKS", + "eks": { + "instanceType": "g6.xlarge", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 1}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + schedulable_pools=[ + { + "name": "l4-pool", + "nodes": 4, + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 1, + "capacity": {"memory": {"value": "24Gi"}}, + }, + ], + }, + ], + cache={"storageClassName": "modelplane-rwx-efs"}, + gateway=None, + ), + resources={ + "activation": _activation_policy( + activate=[ + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", + ], + ready=fnv1.READY_TRUE, + ), + "eks-cluster": _eks_cluster( + zones=["us-west-2a", "us-west-2b"], + capacity_block=None, + fabric=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "cluster-provider-config-kubernetes": _cluster_provider_config( + kubeconfig="test-cluster-kubeconfig-abcde", identity=None + ), + "serving-stack": _serving_stack( + cloud="EKS", + dra_drivers=("gpu.nvidia.com",), + secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], + client_cas=None, + gpu=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "usage-eks-by-backend": _backend_usage(cluster_kind="EKSCluster"), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4-eks": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4-eks" + ), + }, + ), + conditions=[ + fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), + fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + ], + ), + ), # compose-eks-cluster turns the pool's capacityBlock into a CAPACITY_BLOCK # node group. ComposeCase( @@ -5885,6 +6113,128 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + # Like UnsupportedDriverHolds: the fatal result keeps Crossplane from + # collecting the running cluster's GKECluster, ServingStack and Usages, and + # the condition still lands. + ComposeCase( + name="InvalidNodePoolHolds", + reason="A running GKE cluster whose pools' classes have no GKE provisioning block gets a fatal result naming each class.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster(source="GKE", gke=v1alpha1.Gke(region="us-central1")), + node_pools=[ + v1alpha1.NodePool(name="l4-pool", className="gpu-l4-eks", nodeCount=2, maxNodeCount=4), + v1alpha1.NodePool(name="h100-pool", className="gpu-h100-aks", nodeCount=1), + ], + ), + resources={ + "gke-cluster": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "GKECluster", + "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2026-06-08T00:00:00Z", + }, + ], + }, + } + ), + ), + }, + ), + required_resources={ + "class-gpu-l4-eks": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4-eks", + count=1, + memory="24Gi", + provisioning={ + "provider": "EKS", + "eks": { + "instanceType": "g6.xlarge", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 1}, + }, + }, + ), + ], + ), + "class-gpu-h100-aks": fnv1.Resources( + items=[ + _inference_class( + name="gpu-h100-aks", + count=1, + memory="80Gi", + provisioning={ + "provider": "AKS", + "aks": { + "vmSize": "Standard_NC40ads_H100_v5", + "diskSizeGb": 256, + "accelerator": {"type": "nvidia-h100", "count": 1}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State(), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_FATAL, + message=( + "InferenceClass gpu-h100-aks has no GKE provisioning block; " + "InferenceClass gpu-l4-eks has no GKE provisioning block" + ), + ), + ], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4-eks": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4-eks" + ), + "class-gpu-h100-aks": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-h100-aks" + ), + }, + ), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="InvalidNodePool", + message=( + "InferenceClass gpu-h100-aks has no GKE provisioning block; " + "InferenceClass gpu-l4-eks has no GKE provisioning block" + ), + ), + ], + ), + ), # The hostname gate, which is what keeps a cluster off the schedule until # traffic to it is mutually authenticated in both directions. These cases # observe an existing cluster's ServingStack ready, with whatever it and the diff --git a/schemas/.lock.json b/schemas/.lock.json index 1f7f6524b..c7f162118 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,6 +1,6 @@ { "packages": { - "fs://apis": "a214ca0e4ca2772e256e3da42cb23ba4c79d25a648e3eee69acc5c87b9b3dab2", + "fs://apis": "1de6ebfee914bfe44cd8c6693f322d9358160d00ef977982d1f2933ec35b28d5", "git://https://github.com/crossplane/crossplane/cluster/crds": "90d8b72ad8b829f0bcd7d7d5a98eaa0d579f244a", "git://https://github.com/modelplaneai/cluster-agent/cluster/charts/cluster-agent/files/crds": "ca7631d72ef8550a2d2f415f33db1ec52b20c65c", "xpkg://xpkg.upbound.io/upbound/provider-aws-ec2:v2.9.4": "sha256:7a0febdfb8da2d59a9fdcdbc2d476811f00566fcb941880313fc2034b0ca6177", diff --git a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py index d6a4a53a6..4cf9e05f7 100644 --- a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py @@ -342,6 +342,21 @@ class Gateway(BaseModel): """ +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class ProviderConfigRef(BaseModel): name: str | None = None """ @@ -425,6 +440,10 @@ class Status(BaseModel): """ Namespace where the internal XRs (cluster, backend) were created. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools, as the cluster Modelplane provisions reports it. Not set when source is Existing, because Modelplane doesn't provision those pools. + """ providerConfigRef: ProviderConfigRef | None = None schedulablePools: list[SchedulablePool] | None = Field(None, max_length=8) """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py index 95ef71792..268766d9a 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py @@ -156,6 +156,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -180,6 +195,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools. The system node pool Modelplane adds belongs to the cluster itself, so its state is part of the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this AKSCluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py index 2b7d90e04..905cea8a2 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py @@ -118,6 +118,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -138,6 +153,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools. The system node pool Modelplane adds belongs to the cluster itself, so its state is part of the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this CivoCluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py index 192f72dea..04f8f0005 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py @@ -178,6 +178,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -202,6 +217,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools. The system node pool Modelplane adds belongs to the cluster itself, so its state is part of the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this EKSCluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py index 7a408a0d7..2cccf467a 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py @@ -160,6 +160,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -184,6 +199,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools. The system node pool Modelplane adds belongs to the cluster itself, so its state is part of the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this GKECluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py index 4e44a948e..9145a057a 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py @@ -155,6 +155,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -183,6 +198,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools. The system node pool Modelplane adds belongs to the cluster itself, so its state is part of the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by or passed through this cluster. Consumers use these to authenticate to the cluster. Secrets are in the same namespace as this NebiusCluster unless an entry says otherwise. diff --git a/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py index be03c5199..edc8f35a5 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py @@ -118,6 +118,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -138,6 +153,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools. The system node pool Modelplane adds belongs to the cluster itself, so its state is part of the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this VultrCluster.