diff --git a/apis/aksclusters/definition.yaml b/apis/aksclusters/definition.yaml index 3cec68b50..caa724e73 100644 --- a/apis/aksclusters/definition.yaml +++ b/apis/aksclusters/definition.yaml @@ -219,6 +219,45 @@ spec: status: description: AKSClusterStatus defines the observed state of AKSCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools. + The system node pool Modelplane adds belongs to the cluster + itself, so its state is part of the ClusterReady condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/civoclusters/definition.yaml b/apis/civoclusters/definition.yaml index 6200f7336..262b92314 100644 --- a/apis/civoclusters/definition.yaml +++ b/apis/civoclusters/definition.yaml @@ -177,6 +177,45 @@ spec: status: description: CivoClusterStatus defines the observed state of CivoCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools. + The system node pool Modelplane adds belongs to the cluster + itself, so its state is part of the ClusterReady condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/eksclusters/definition.yaml b/apis/eksclusters/definition.yaml index 8bfa2dddd..1b4db61fb 100644 --- a/apis/eksclusters/definition.yaml +++ b/apis/eksclusters/definition.yaml @@ -247,6 +247,45 @@ spec: status: description: EKSClusterStatus defines the observed state of EKSCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools. + The system node pool Modelplane adds belongs to the cluster + itself, so its state is part of the ClusterReady condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/gkeclusters/definition.yaml b/apis/gkeclusters/definition.yaml index 588edc7d0..13e5d5008 100644 --- a/apis/gkeclusters/definition.yaml +++ b/apis/gkeclusters/definition.yaml @@ -201,6 +201,45 @@ spec: status: description: GKEClusterStatus defines the observed state of GKECluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools. + The system node pool Modelplane adds belongs to the cluster + itself, so its state is part of the ClusterReady condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/inferenceclusters/definition.yaml b/apis/inferenceclusters/definition.yaml index 0e0c8ca3c..9cdfd6459 100644 --- a/apis/inferenceclusters/definition.yaml +++ b/apis/inferenceclusters/definition.yaml @@ -581,6 +581,39 @@ spec: status: type: object properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools, + as the cluster Modelplane provisions reports it. Not set + when source is Existing, because Modelplane doesn't + provision those pools. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [name] + items: + type: object + required: [name, state] + properties: + name: + type: string + maxLength: 40 + description: Name of the node pool in spec.nodePools. + state: + type: string + enum: [Pending, Provisioning, Ready, Failed] + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. providerConfigRef: type: object properties: diff --git a/apis/nebiusclusters/definition.yaml b/apis/nebiusclusters/definition.yaml index 171592e49..daeb3ac55 100644 --- a/apis/nebiusclusters/definition.yaml +++ b/apis/nebiusclusters/definition.yaml @@ -241,6 +241,45 @@ spec: status: description: NebiusClusterStatus defines the observed state of NebiusCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools. + The system node pool Modelplane adds belongs to the cluster + itself, so its state is part of the ClusterReady condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/vultrclusters/definition.yaml b/apis/vultrclusters/definition.yaml index 26346cdee..cacbce83c 100644 --- a/apis/vultrclusters/definition.yaml +++ b/apis/vultrclusters/definition.yaml @@ -171,6 +171,45 @@ spec: status: description: VultrClusterStatus defines the observed state of VultrCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools. + The system node pool Modelplane adds belongs to the cluster + itself, so its state is part of the ClusterReady condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/functions/compose-aks-cluster/function/fn.py b/functions/compose-aks-cluster/function/fn.py index 77988a927..a1568c1d3 100644 --- a/functions/compose-aks-cluster/function/fn.py +++ b/functions/compose-aks-cluster/function/fn.py @@ -179,6 +179,38 @@ # Azure caps an AKS cluster's DNS prefix at 54 characters. _DNS_PREFIX_MAX_LEN = 54 +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: resource group, +# network, the AKS cluster with its system pool as the default node pool, and +# the in-cluster network operator and StorageClass. compose-inference-cluster +# relays both conditions onto the InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + + +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + def _name(meta: metav1.ObjectMeta | None) -> str: """The object's name, always set on resources read from the API server.""" @@ -260,8 +292,8 @@ def compose(self) -> None: self.compose_network_operator() self.compose_provider_configs() self.compose_storage_class() - self.write_status() self.mark_readiness() + self.write_status(self.derive_conditions()) def _cred_kind(self) -> str: creds = self.xr.spec.credentials @@ -594,8 +626,9 @@ def compose_provider_configs(self) -> None: ), ) - def write_status(self) -> None: + def write_status(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: status = v1alpha1.Status( + nodePools=node_pools, secrets=[ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -633,6 +666,108 @@ def mark_readiness(self) -> None: self.rsp.desired.resources["provider-config-kubernetes"].ready = fnv1.READY_TRUE self.rsp.desired.resources["provider-config-helm"].ready = fnv1.READY_TRUE + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool. The network operator Helm + release serves every InfiniBand pool, so it's part of the cluster + instead.""" + return [f"nodepool-{pool.name}"] + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def _node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each user node pool, from its resources.""" + statuses = [] + for pool in self.xr.spec.nodePools: + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self) -> list[v1alpha1.NodePoolModel]: + """Derive ClusterReady and NodePoolsReady from the readiness + mark_readiness set, and return each node pool's state for + status.nodePools. + + NodePoolsReady is True only once ClusterReady is, so the first False + condition names the stage provisioning is in. A failed pool is reported + Failed even while the cluster is still provisioning. + """ + node_pools = self._node_pool_statuses() + in_pools = {r for pool in self.xr.spec.nodePools for r in self.pool_resources(pool)} + cluster = sorted(r for r in self.rsp.desired.resources if r not in in_pools) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True": + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) + return node_pools + def _networking(self) -> v1alpha1.Networking: """Return the (defaulted) networking config from the XR.""" return self.xr.spec.networking or v1alpha1.Networking() diff --git a/functions/compose-aks-cluster/tests/test_fn.py b/functions/compose-aks-cluster/tests/test_fn.py index 94e42184f..6600a5a0b 100644 --- a/functions/compose-aks-cluster/tests/test_fn.py +++ b/functions/compose-aks-cluster/tests/test_fn.py @@ -39,6 +39,25 @@ class Case: want: fnv1.RunFunctionResponse +@dataclasses.dataclass +class Derived: + """What derive_conditions reports: each node pool's state, and the conditions it sets.""" + + node_pools: list[dict] + conditions: list[fnv1.Condition] + + +@dataclasses.dataclass +class DeriveConditionsCase: + """A test case for Composer.derive_conditions.""" + + name: str + reason: str + req: fnv1.RunFunctionRequest + desired: fnv1.State + want: Derived + + def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.NodePool]) -> fnv1.Resource: """The observed AKSCluster XR, with the given credentials and node pools.""" xr = v1alpha1.AKSCluster( @@ -52,12 +71,13 @@ def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.N return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr() -> fnv1.Resource: - """The desired XR, publishing its kubeconfig Secret and cache StorageClass.""" +def _desired_xr(*, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR, publishing node_pools' states, its kubeconfig Secret and cache StorageClass.""" return fnv1.Resource( resource=resource.dict_to_struct( { "status": { + "nodePools": node_pools, "secrets": [ { "type": "Kubeconfig", @@ -271,6 +291,27 @@ def _provider_config(*, api_version: str) -> fnv1.Resource: ) +def _observed_failed(*, message: str) -> fnv1.Resource: + """An observed managed resource whose provider reports its last operation failed with message.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "conditions": [ + { + "type": "LastAsyncOperation", + "status": "False", + "reason": "AsyncCreateFailure", + "message": message, + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + ), + ) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -304,7 +345,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpuh100", "state": "Provisioning"}]), resources={ "resource-group": _resource_group( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -326,6 +367,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -351,7 +406,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "cpupool", "state": "Provisioning"}]), resources={ "resource-group": _resource_group( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -371,6 +426,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -399,7 +468,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpuh100", "state": "Provisioning"}]), resources={ "resource-group": _resource_group( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -424,6 +493,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -452,7 +535,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpuh100", "state": "Provisioning"}]), resources={ "resource-group": _resource_group( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -474,6 +557,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -554,7 +651,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpuh100", "state": "Provisioning"}]), resources={ "resource-group": _resource_group( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -655,6 +752,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="4 of 8 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The ProviderConfigs reach the cluster through its kubeconfig, not the @@ -684,7 +795,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpuh100", "state": "Provisioning"}]), resources={ "resource-group": _resource_group( cred_kind="ProviderConfig", cred_name="my-azure-account", ready=fnv1.READY_UNSPECIFIED @@ -709,6 +820,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -905,7 +1030,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpuh100", "state": "Ready"}]), resources={ "resource-group": _resource_group( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE @@ -963,6 +1088,18 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], ), ), ] @@ -1030,3 +1167,283 @@ def test_dns_prefix_wired() -> None: got = asyncio.run(fn.FunctionRunner().RunFunction(req, None)) cluster = resource.struct_to_dict(got.desired.resources["cluster"].resource) assert cluster["spec"]["forProvider"]["dnsPrefix"] == "modelplane-system-mpiiiiiiiiiiiiiiiiiiiiiiiii-ak-0d844" + + +# Each case's XR has two pools, each built by its node pool: gpua10, and +# gpuh100, an InfiniBand pool whose network operator release counts toward the +# cluster. desired is what the function composed, with the readiness +# mark_readiness set. +DERIVE_CONDITIONS_CASES = [ + DeriveConditionsCase( + name="ClusterProvisioning", + reason="With cluster resources unready, ClusterReady counts how many are ready and NodePoolsReady waits on it while each pool provisions.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool(name="gpua10", role="GPU", vmSize="Standard_NV36ads_A10_v5", nodeCount=1), + v1alpha1.NodePool( + name="gpuh100", + role="GPU", + vmSize="Standard_ND96isr_H100_v5", + nodeCount=1, + fabric="InfiniBand", + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "resource-group": fnv1.Resource(ready=fnv1.READY_TRUE), + "virtual-network": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "subnet": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "release-network-operator": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-helm": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodepool-gpua10": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodepool-gpuh100": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpua10", "state": "Provisioning"}, {"name": "gpuh100", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolsProvisioning", + reason="With every cluster resource ready and the gpuh100 pool's node pool unready, only gpua10 is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool(name="gpua10", role="GPU", vmSize="Standard_NV36ads_A10_v5", nodeCount=1), + v1alpha1.NodePool( + name="gpuh100", + role="GPU", + vmSize="Standard_ND96isr_H100_v5", + nodeCount=1, + fabric="InfiniBand", + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "resource-group": fnv1.Resource(ready=fnv1.READY_TRUE), + "virtual-network": fnv1.Resource(ready=fnv1.READY_TRUE), + "subnet": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "release-network-operator": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-helm": fnv1.Resource(ready=fnv1.READY_TRUE), + "storage-class-rwx-fs": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodepool-gpua10": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodepool-gpuh100": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpua10", "state": "Ready"}, {"name": "gpuh100", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 node pools ready", + ), + ], + ), + ), + DeriveConditionsCase( + name="AllReady", + reason="With every resource ready, ClusterReady and NodePoolsReady are True and every pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool(name="gpua10", role="GPU", vmSize="Standard_NV36ads_A10_v5", nodeCount=1), + v1alpha1.NodePool( + name="gpuh100", + role="GPU", + vmSize="Standard_ND96isr_H100_v5", + nodeCount=1, + fabric="InfiniBand", + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "resource-group": fnv1.Resource(ready=fnv1.READY_TRUE), + "virtual-network": fnv1.Resource(ready=fnv1.READY_TRUE), + "subnet": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "release-network-operator": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-helm": fnv1.Resource(ready=fnv1.READY_TRUE), + "storage-class-rwx-fs": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodepool-gpua10": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodepool-gpuh100": fnv1.Resource(ready=fnv1.READY_TRUE), + }, + ), + want=Derived( + node_pools=[{"name": "gpua10", "state": "Ready"}, {"name": "gpuh100", "state": "Ready"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolFailed", + reason="With the provider reporting a pool's node pool create failed, the pool is Failed with the provider's error, even while the cluster provisions.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool(name="gpua10", role="GPU", vmSize="Standard_NV36ads_A10_v5", nodeCount=1), + v1alpha1.NodePool( + name="gpuh100", + role="GPU", + vmSize="Standard_ND96isr_H100_v5", + nodeCount=1, + fabric="InfiniBand", + ), + ], + ), + resources={ + "nodepool-gpuh100": _observed_failed(message="async create failed: SkuNotAvailable"), + }, + ), + ), + desired=fnv1.State( + resources={ + "resource-group": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodepool-gpua10": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodepool-gpuh100": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[ + {"name": "gpua10", "state": "Provisioning"}, + { + "name": "gpuh100", + "state": "Failed", + "message": "nodepool-gpuh100: async create failed: SkuNotAvailable", + }, + ], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 2 node pools failed: gpuh100", + ), + ], + ), + ), + DeriveConditionsCase( + name="ClusterFailed", + reason="With the provider reporting two cluster resources failed, ClusterReady is Failed with the first one's error.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool(name="gpua10", role="GPU", vmSize="Standard_NV36ads_A10_v5", nodeCount=1), + v1alpha1.NodePool( + name="gpuh100", + role="GPU", + vmSize="Standard_ND96isr_H100_v5", + nodeCount=1, + fabric="InfiniBand", + ), + ], + ), + resources={ + "cluster": _observed_failed(message="async create failed: K8sVersionNotSupported"), + "subnet": _observed_failed(message="async create failed: NetcfgInvalidSubnet"), + }, + ), + ), + desired=fnv1.State( + resources={ + "resource-group": fnv1.Resource(ready=fnv1.READY_TRUE), + "subnet": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodepool-gpua10": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodepool-gpuh100": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpua10", "state": "Provisioning"}, {"name": "gpuh100", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="cluster: async create failed: K8sVersionNotSupported (and 1 more)", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), +] + + +@pytest.mark.parametrize("case", DERIVE_CONDITIONS_CASES, ids=lambda case: case.name) +def test_derive_conditions(case: DeriveConditionsCase) -> None: + """derive_conditions reports which provisioning stage the cluster is in, and each node pool's state.""" + rsp = fnv1.RunFunctionResponse(desired=case.desired) + node_pools = fn.Composer(case.req, rsp).derive_conditions() + got = { + "nodePools": [p.model_dump(exclude_none=True) for p in node_pools], + "conditions": [_to_dict(c) for c in rsp.conditions], + } + want = {"nodePools": case.want.node_pools, "conditions": [_to_dict(c) for c in case.want.conditions]} + assert got == want, case.reason diff --git a/functions/compose-civo-cluster/function/fn.py b/functions/compose-civo-cluster/function/fn.py index 5416d682d..5193fe976 100644 --- a/functions/compose-civo-cluster/function/fn.py +++ b/functions/compose-civo-cluster/function/fn.py @@ -151,6 +151,38 @@ # nodeCount into forProvider, defeating the purpose.) _AUTOSCALED_POOL_MANAGEMENT: list[_ManagementPolicy] = ["Observe", "Create", "Update", "Delete"] +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: network, +# firewall, the cluster with its inline system node pool, the Helm +# ProviderConfig and the cluster autoscaler. compose-inference-cluster relays +# both conditions onto the InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + + +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + def _name(meta: metav1.ObjectMeta | None) -> str: """The object's name, always set on resources read from the API server.""" @@ -213,8 +245,8 @@ def compose(self) -> None: self.compose_node_pools() self.compose_provider_configs() self.compose_cluster_autoscaler() - self.write_status() self.mark_readiness() + self.write_status(self.derive_conditions()) def _cluster_ready(self) -> bool: return resource.get_condition(self.req.observed.resources.get("cluster"), "Ready").status == "True" @@ -490,8 +522,9 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> nodepoolv1beta1.NodePool: return nodepoolv1beta1.NodePool(spec=spec) - def write_status(self) -> None: + def write_status(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: status = v1alpha1.Status( + nodePools=node_pools, secrets=[ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -517,3 +550,104 @@ def mark_readiness(self) -> None: continue if resource.get_condition(self.req.observed.resources.get(r), "Ready").status == "True": self.rsp.desired.resources[r].ready = fnv1.READY_TRUE + + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool. The autoscaler release + scales every autoscaled pool, so it's part of the cluster instead.""" + return [f"node-pool-{pool.name}"] + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def _node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each user node pool, from its resources.""" + statuses = [] + for pool in self.xr.spec.nodePools: + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self) -> list[v1alpha1.NodePoolModel]: + """Derive ClusterReady and NodePoolsReady from the readiness + mark_readiness set, and return each node pool's state for + status.nodePools. + + NodePoolsReady is True only once ClusterReady is, so the first False + condition names the stage provisioning is in. A failed pool is reported + Failed even while the cluster is still provisioning. + """ + node_pools = self._node_pool_statuses() + in_pools = {r for pool in self.xr.spec.nodePools for r in self.pool_resources(pool)} + cluster = sorted(r for r in self.rsp.desired.resources if r not in in_pools) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True": + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) + return node_pools diff --git a/functions/compose-civo-cluster/tests/test_fn.py b/functions/compose-civo-cluster/tests/test_fn.py index 92edc33a0..549059779 100644 --- a/functions/compose-civo-cluster/tests/test_fn.py +++ b/functions/compose-civo-cluster/tests/test_fn.py @@ -40,6 +40,25 @@ class Case: want: fnv1.RunFunctionResponse +@dataclasses.dataclass +class Derived: + """What derive_conditions reports: each node pool's state, and the conditions it sets.""" + + node_pools: list[dict] + conditions: list[fnv1.Condition] + + +@dataclasses.dataclass +class DeriveConditionsCase: + """A test case for Composer.derive_conditions.""" + + name: str + reason: str + req: fnv1.RunFunctionRequest + desired: fnv1.State + want: Derived + + def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.NodePool]) -> fnv1.Resource: """The observed CivoCluster XR in LON1, with the given credentials and node pools.""" xr = v1alpha1.CivoCluster( @@ -56,12 +75,13 @@ def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.N return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr() -> fnv1.Resource: - """The desired XR, publishing the cluster's kubeconfig Secret.""" +def _desired_xr(*, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR, publishing node_pools' states and the cluster's kubeconfig Secret.""" return fnv1.Resource( resource=resource.dict_to_struct( { "status": { + "nodePools": node_pools, "secrets": [ { "type": "Kubeconfig", @@ -339,6 +359,27 @@ def _autoscaler_release(*, autoscaling_groups: list[dict]) -> fnv1.Resource: ) +def _observed_failed(*, message: str) -> fnv1.Resource: + """An observed managed resource whose provider reports its last operation failed with message.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "conditions": [ + { + "type": "LastAsyncOperation", + "status": "False", + "reason": "AsyncCreateFailure", + "message": message, + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + ), + ) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -367,7 +408,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Pending"}]), resources={ "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), @@ -377,6 +418,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 3 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -404,7 +459,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), @@ -420,6 +475,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -446,7 +515,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "cpu-pool", "state": "Provisioning"}]), resources={ "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), @@ -462,6 +531,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The autoscaler addresses node groups by the pool ID the provider writes @@ -532,7 +615,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Ready"}]), resources={ "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), @@ -552,6 +635,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The observed ProviderConfig shows the dependents were composed before. @@ -613,7 +710,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), @@ -631,6 +728,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -658,7 +769,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), @@ -674,6 +785,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The helm ProviderConfig and the Release reach the cluster through its @@ -702,7 +827,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "workers", "state": "Provisioning"}]), resources={ "network": _network(cred_kind="ProviderConfig", cred_name="team-a"), "firewall": _firewall(cred_kind="ProviderConfig", cred_name="team-a"), @@ -731,6 +856,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), ] @@ -741,3 +880,326 @@ def test_compose(case: Case) -> None: """RunFunction composes a Civo cluster's infrastructure.""" got = asyncio.run(fn.FunctionRunner().RunFunction(case.req, None)) assert _to_dict(got) == _to_dict(case.want), case.reason + + +# Each case's XR has two pools, gpu-l40s and cpu-pool, each built by its +# NodePool. desired is what the function composed, with the readiness +# mark_readiness set. +DERIVE_CONDITIONS_CASES = [ + DeriveConditionsCase( + name="ClusterProvisioning", + reason="With cluster resources unready, ClusterReady counts how many are ready and NodePoolsReady waits on it while each pool provisions.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + size="an.g1.l40s.kube.x1", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", size="g4s.kube.large", nodeCount=1), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "firewall": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "provider-config-helm": fnv1.Resource(ready=fnv1.READY_TRUE), + "release-cluster-autoscaler": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "node-pool-gpu-l40s": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "node-pool-cpu-pool": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}, {"name": "cpu-pool", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolsProvisioning", + reason="With every cluster resource ready and the cpu-pool pool's NodePool unready, only gpu-l40s is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + size="an.g1.l40s.kube.x1", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", size="g4s.kube.large", nodeCount=1), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "firewall": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-helm": fnv1.Resource(ready=fnv1.READY_TRUE), + "release-cluster-autoscaler": fnv1.Resource(ready=fnv1.READY_TRUE), + "node-pool-gpu-l40s": fnv1.Resource(ready=fnv1.READY_TRUE), + "node-pool-cpu-pool": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Ready"}, {"name": "cpu-pool", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 node pools ready", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolPending", + reason="With the cluster unready and so no NodePool composed, every pool is Pending.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + size="an.g1.l40s.kube.x1", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", size="g4s.kube.large", nodeCount=1), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "firewall": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Pending"}, {"name": "cpu-pool", "state": "Pending"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 3 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + DeriveConditionsCase( + name="AllReady", + reason="With every resource ready, ClusterReady and NodePoolsReady are True and every pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + size="an.g1.l40s.kube.x1", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", size="g4s.kube.large", nodeCount=1), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "firewall": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-helm": fnv1.Resource(ready=fnv1.READY_TRUE), + "release-cluster-autoscaler": fnv1.Resource(ready=fnv1.READY_TRUE), + "node-pool-gpu-l40s": fnv1.Resource(ready=fnv1.READY_TRUE), + "node-pool-cpu-pool": fnv1.Resource(ready=fnv1.READY_TRUE), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Ready"}, {"name": "cpu-pool", "state": "Ready"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolFailed", + reason="With the provider reporting a pool's NodePool create failed, the pool is Failed with the provider's error, even while the cluster provisions.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + size="an.g1.l40s.kube.x1", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", size="g4s.kube.large", nodeCount=1), + ], + ), + resources={ + "node-pool-gpu-l40s": _observed_failed( + message="async create failed: insufficient an.g1.l40s.kube.x1 capacity in LON1" + ), + }, + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "firewall": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "provider-config-helm": fnv1.Resource(ready=fnv1.READY_TRUE), + "release-cluster-autoscaler": fnv1.Resource(ready=fnv1.READY_TRUE), + "node-pool-gpu-l40s": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "node-pool-cpu-pool": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[ + { + "name": "gpu-l40s", + "state": "Failed", + "message": "node-pool-gpu-l40s: async create failed: insufficient an.g1.l40s.kube.x1 capacity in LON1", + }, + {"name": "cpu-pool", "state": "Provisioning"}, + ], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="4 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 2 node pools failed: gpu-l40s", + ), + ], + ), + ), + DeriveConditionsCase( + name="ClusterFailed", + reason="With the provider reporting two cluster resources failed, ClusterReady is Failed with the first one's error.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + size="an.g1.l40s.kube.x1", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", size="g4s.kube.large", nodeCount=1), + ], + ), + resources={ + "cluster": _observed_failed( + message="async create failed: kubernetes version 1.20.0-k3s1 is not supported" + ), + "firewall": _observed_failed( + message="async create failed: firewall name test-cluster is already in use" + ), + }, + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "firewall": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Pending"}, {"name": "cpu-pool", "state": "Pending"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="cluster: async create failed: kubernetes version 1.20.0-k3s1 is not supported (and 1 more)", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), +] + + +@pytest.mark.parametrize("case", DERIVE_CONDITIONS_CASES, ids=lambda case: case.name) +def test_derive_conditions(case: DeriveConditionsCase) -> None: + """derive_conditions reports which provisioning stage the cluster is in, and each node pool's state.""" + rsp = fnv1.RunFunctionResponse(desired=case.desired) + node_pools = fn.Composer(case.req, rsp).derive_conditions() + got = { + "nodePools": [p.model_dump(exclude_none=True) for p in node_pools], + "conditions": [_to_dict(c) for c in rsp.conditions], + } + want = {"nodePools": case.want.node_pools, "conditions": [_to_dict(c) for c in case.want.conditions]} + assert got == want, case.reason diff --git a/functions/compose-eks-cluster/function/fn.py b/functions/compose-eks-cluster/function/fn.py index 14759acee..2f96436ed 100644 --- a/functions/compose-eks-cluster/function/fn.py +++ b/functions/compose-eks-cluster/function/fn.py @@ -324,6 +324,37 @@ def _namespace(meta: metav1.ObjectMeta | None) -> str: # point per PVC inside the auto-provisioned filesystem. _MANAGED_STORAGE_CLASS = "modelplane-rwx-efs" +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: network, IAM, +# control plane, system node group and add-ons. compose-inference-cluster +# relays both conditions onto the InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + + +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + def _kubeconfig_secret_name(xr: v1alpha1.EKSCluster) -> str: """Derive the kubeconfig secret name from the XR.""" @@ -428,8 +459,8 @@ def compose(self) -> None: self.compose_efa_dra_driver() self.compose_provider_configs() self.compose_storage_class() - self.write_status() self.mark_readiness() + self.write_status(self.derive_conditions()) def compose_network(self) -> None: """Compose the VPC, its two subnet tiers, and internet routing. @@ -1687,8 +1718,9 @@ def compose_provider_configs(self) -> None: ), ) - def write_status(self) -> None: + def write_status(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: status = v1alpha1.Status( + nodePools=node_pools, secrets=[ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -1781,6 +1813,110 @@ def mark_readiness(self) -> None: self.rsp.desired.resources["provider-config-kubernetes"].ready = fnv1.READY_TRUE self.rsp.desired.resources["provider-config-helm"].ready = fnv1.READY_TRUE + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool. The EFA security group is + shared by every EFA pool, so it's part of the cluster instead.""" + resources = [f"nodegroup-{pool.name}"] + if pool.capacityBlock or pool.fabric == _FABRIC_EFA: + resources.append(f"launch-template-{pool.name}") + return resources + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def _node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each user node pool, from its resources.""" + statuses = [] + for pool in self.xr.spec.nodePools: + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self) -> list[v1alpha1.NodePoolModel]: + """Derive ClusterReady and NodePoolsReady from the readiness + mark_readiness set, and return each node pool's state for + status.nodePools. + + NodePoolsReady is True only once ClusterReady is, so the first False + condition names the stage provisioning is in. A failed pool is reported + Failed even while the cluster is still provisioning. + """ + node_pools = self._node_pool_statuses() + in_pools = {r for pool in self.xr.spec.nodePools for r in self.pool_resources(pool)} + cluster = sorted(r for r in self.rsp.desired.resources if r not in in_pools) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True": + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) + return node_pools + def _networking(self) -> v1alpha1.Networking: """Return the (defaulted) networking config from the XR.""" return self.xr.spec.networking or v1alpha1.Networking() diff --git a/functions/compose-eks-cluster/tests/test_fn.py b/functions/compose-eks-cluster/tests/test_fn.py index bd16d8e9c..8287b444a 100644 --- a/functions/compose-eks-cluster/tests/test_fn.py +++ b/functions/compose-eks-cluster/tests/test_fn.py @@ -39,6 +39,25 @@ class Case: want: fnv1.RunFunctionResponse +@dataclasses.dataclass +class Derived: + """What derive_conditions reports: each node pool's state, and the conditions it sets.""" + + node_pools: list[dict] + conditions: list[fnv1.Condition] + + +@dataclasses.dataclass +class DeriveConditionsCase: + """A test case for Composer.derive_conditions.""" + + name: str + reason: str + req: fnv1.RunFunctionRequest + desired: fnv1.State + want: Derived + + def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.NodePool]) -> fnv1.Resource: """The observed EKSCluster XR in us-west-2, with the given credentials and node pools.""" xr = v1alpha1.EKSCluster( @@ -52,12 +71,13 @@ def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.N return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr() -> fnv1.Resource: - """The desired XR, publishing its kubeconfig Secret and cache StorageClass.""" +def _desired_xr(*, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR, publishing node_pools' states, its kubeconfig Secret and cache StorageClass.""" return fnv1.Resource( resource=resource.dict_to_struct( { "status": { + "nodePools": node_pools, # `type` is emitted because the function sets it explicitly # on the Status model, so update_status (exclude_unset) # keeps it rather than dropping it as an unset field. @@ -1119,6 +1139,27 @@ def _provider_config(*, api_version: str) -> fnv1.Resource: ) +def _observed_failed(*, message: str) -> fnv1.Resource: + """An observed managed resource whose provider reports its last operation failed with message.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "conditions": [ + { + "type": "LastAsyncOperation", + "status": "False", + "reason": "AsyncCreateFailure", + "message": message, + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + ), + ) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -1150,7 +1191,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l4", "state": "Provisioning"}]), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), "subnet-0": _subnet( @@ -1318,6 +1359,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 49 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -1343,7 +1398,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "cpu-pool", "state": "Provisioning"}]), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), "subnet-0": _subnet( @@ -1511,6 +1566,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 49 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The observed filesystem id lets the function compose the StorageClass @@ -1599,7 +1668,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l4", "state": "Provisioning"}]), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), "subnet-0": _subnet( @@ -1800,6 +1869,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="6 of 51 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The launch template targets the reservation through the capacity-block @@ -1832,7 +1915,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-h200", "state": "Provisioning"}]), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), "subnet-0": _subnet( @@ -2065,6 +2148,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 49 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The node group's launch template carries one EFA interface per network card @@ -2099,7 +2196,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-h200", "state": "Provisioning"}]), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), "subnet-0": _subnet( @@ -2271,6 +2368,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 52 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # Both security group IDs are observed, the EFA group's in its external name @@ -2330,7 +2441,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-h200", "state": "Provisioning"}]), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), "subnet-0": _subnet( @@ -2532,6 +2643,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 54 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # Like the autoscaler, the EFA DRA driver release is gated on the cluster being @@ -2566,7 +2691,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-h200", "state": "Provisioning"}]), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), "subnet-0": _subnet( @@ -2768,6 +2893,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 54 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -2798,7 +2937,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l4", "state": "Provisioning"}]), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), "subnet-0": _subnet( @@ -2967,6 +3106,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 50 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The ProviderConfigs reach the cluster through its kubeconfig, so they don't @@ -2996,7 +3149,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l4", "state": "Provisioning"}]), resources={ "vpc": _vpc(cred_kind="ProviderConfig", cred_name="my-aws-account"), "subnet-0": _subnet( @@ -3164,6 +3317,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 49 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), ] @@ -3174,3 +3341,331 @@ def test_compose(case: Case) -> None: """RunFunction composes EKS cluster infrastructure.""" got = asyncio.run(fn.FunctionRunner().RunFunction(case.req, None)) assert _to_dict(got) == _to_dict(case.want), case.reason + + +# Each case's XR has two pools: gpu-l4, built by its node group, and gpu-h200, a +# Capacity Block pool built by its node group and launch template. desired is +# what the function composed, with the readiness mark_readiness set. +DERIVE_CONDITIONS_CASES = [ + DeriveConditionsCase( + name="ClusterProvisioning", + reason="With cluster resources unready, ClusterReady counts how many are ready and NodePoolsReady waits on it while each pool provisions.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool(name="gpu-l4", role="GPU", instanceType="g6.xlarge", nodeCount=1), + v1alpha1.NodePool( + name="gpu-h200", + role="GPU", + instanceType="p5e.48xlarge", + nodeCount=1, + capacityBlock=v1alpha1.CapacityBlock(capacityReservationId="cr-0123456789abcdef0"), + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "vpc": fnv1.Resource(ready=fnv1.READY_TRUE), + "nat-gateway": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "cluster-auth": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-system": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "release-cluster-autoscaler": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-gpu-l4": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-gpu-h200": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "launch-template-gpu-h200": fnv1.Resource(ready=fnv1.READY_TRUE), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l4", "state": "Provisioning"}, {"name": "gpu-h200", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolsProvisioning", + reason="With every cluster resource ready and one of the gpu-h200 pool's two resources unready, only gpu-l4 is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool(name="gpu-l4", role="GPU", instanceType="g6.xlarge", nodeCount=1), + v1alpha1.NodePool( + name="gpu-h200", + role="GPU", + instanceType="p5e.48xlarge", + nodeCount=1, + capacityBlock=v1alpha1.CapacityBlock(capacityReservationId="cr-0123456789abcdef0"), + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "vpc": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster-auth": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-system": fnv1.Resource(ready=fnv1.READY_TRUE), + "release-cluster-autoscaler": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-gpu-l4": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-gpu-h200": fnv1.Resource(ready=fnv1.READY_TRUE), + "launch-template-gpu-h200": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l4", "state": "Ready"}, {"name": "gpu-h200", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 node pools ready", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolPending", + reason="With a pool's resources not yet composed, the pool is Pending.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool(name="gpu-l4", role="GPU", instanceType="g6.xlarge", nodeCount=1), + v1alpha1.NodePool( + name="gpu-h200", + role="GPU", + instanceType="p5e.48xlarge", + nodeCount=1, + capacityBlock=v1alpha1.CapacityBlock(capacityReservationId="cr-0123456789abcdef0"), + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "vpc": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster-auth": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-system": fnv1.Resource(ready=fnv1.READY_TRUE), + "release-cluster-autoscaler": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-gpu-l4": fnv1.Resource(ready=fnv1.READY_TRUE), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l4", "state": "Ready"}, {"name": "gpu-h200", "state": "Pending"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 node pools ready", + ), + ], + ), + ), + DeriveConditionsCase( + name="AllReady", + reason="With every resource ready, ClusterReady and NodePoolsReady are True and every pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool(name="gpu-l4", role="GPU", instanceType="g6.xlarge", nodeCount=1), + v1alpha1.NodePool( + name="gpu-h200", + role="GPU", + instanceType="p5e.48xlarge", + nodeCount=1, + capacityBlock=v1alpha1.CapacityBlock(capacityReservationId="cr-0123456789abcdef0"), + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "vpc": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster-auth": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-system": fnv1.Resource(ready=fnv1.READY_TRUE), + "release-cluster-autoscaler": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-gpu-l4": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-gpu-h200": fnv1.Resource(ready=fnv1.READY_TRUE), + "launch-template-gpu-h200": fnv1.Resource(ready=fnv1.READY_TRUE), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l4", "state": "Ready"}, {"name": "gpu-h200", "state": "Ready"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolFailed", + reason="With the provider reporting a pool's node group create failed, the pool is Failed with the provider's error, even while the cluster provisions.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool(name="gpu-l4", role="GPU", instanceType="g6.xlarge", nodeCount=1), + v1alpha1.NodePool( + name="gpu-h200", + role="GPU", + instanceType="p5e.48xlarge", + nodeCount=1, + capacityBlock=v1alpha1.CapacityBlock(capacityReservationId="cr-0123456789abcdef0"), + ), + ], + ), + resources={ + "nodegroup-gpu-h200": _observed_failed(message="async create failed: InsufficientInstanceCapacity"), + }, + ), + ), + desired=fnv1.State( + resources={ + "vpc": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-gpu-l4": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-gpu-h200": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "launch-template-gpu-h200": fnv1.Resource(ready=fnv1.READY_TRUE), + }, + ), + want=Derived( + node_pools=[ + {"name": "gpu-l4", "state": "Provisioning"}, + { + "name": "gpu-h200", + "state": "Failed", + "message": "nodegroup-gpu-h200: async create failed: InsufficientInstanceCapacity", + }, + ], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 2 node pools failed: gpu-h200", + ), + ], + ), + ), + DeriveConditionsCase( + name="ClusterFailed", + reason="With the provider reporting two cluster resources failed, ClusterReady is Failed with the first one's error.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool(name="gpu-l4", role="GPU", instanceType="g6.xlarge", nodeCount=1), + v1alpha1.NodePool( + name="gpu-h200", + role="GPU", + instanceType="p5e.48xlarge", + nodeCount=1, + capacityBlock=v1alpha1.CapacityBlock(capacityReservationId="cr-0123456789abcdef0"), + ), + ], + ), + resources={ + "cluster": _observed_failed(message="async create failed: UnsupportedAvailabilityZoneException"), + "nat-gateway": _observed_failed(message="async create failed: NatGatewayLimitExceeded"), + }, + ), + ), + desired=fnv1.State( + resources={ + "vpc": fnv1.Resource(ready=fnv1.READY_TRUE), + "nat-gateway": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-gpu-l4": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-gpu-h200": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "launch-template-gpu-h200": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l4", "state": "Provisioning"}, {"name": "gpu-h200", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="cluster: async create failed: UnsupportedAvailabilityZoneException (and 1 more)", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), +] + + +@pytest.mark.parametrize("case", DERIVE_CONDITIONS_CASES, ids=lambda case: case.name) +def test_derive_conditions(case: DeriveConditionsCase) -> None: + """derive_conditions reports which provisioning stage the cluster is in, and each node pool's state.""" + rsp = fnv1.RunFunctionResponse(desired=case.desired) + node_pools = fn.Composer(case.req, rsp).derive_conditions() + got = { + "nodePools": [p.model_dump(exclude_none=True) for p in node_pools], + "conditions": [_to_dict(c) for c in rsp.conditions], + } + want = {"nodePools": case.want.node_pools, "conditions": [_to_dict(c) for c in case.want.conditions]} + assert got == want, case.reason diff --git a/functions/compose-gke-cluster/function/fn.py b/functions/compose-gke-cluster/function/fn.py index a45542bd0..617182bd6 100644 --- a/functions/compose-gke-cluster/function/fn.py +++ b/functions/compose-gke-cluster/function/fn.py @@ -123,6 +123,38 @@ _GKE_IMAGE_TYPE = "COS_CONTAINERD" _GKE_OAUTH_SCOPE = "https://www.googleapis.com/auth/cloud-platform" +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: network, Filestore +# API, service account and IAM, control plane, system node pool, ProviderConfigs +# and the RWX StorageClass. compose-inference-cluster relays both conditions +# onto the InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + + +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + def _name(meta: metav1.ObjectMeta | None) -> str: """The object's name, always set on resources read from the API server.""" @@ -216,6 +248,21 @@ def resolve_project(self) -> str | None: ) return project response.normal(self.rsp, f"Waiting for GCP {cred_kind} {cred_name}") + response.set_conditions( + self.rsp, + resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"Waiting for GCP {cred_kind} {cred_name}", + ), + resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ), + ) # Nothing is composed while waiting for the project, and an XR with # no composed resources would otherwise be trivially ready. self.rsp.desired.composite.ready = fnv1.READY_FALSE @@ -256,8 +303,8 @@ def compose(self) -> None: self.compose_iam_binding() self.compose_provider_configs() self.compose_storage_class() - self.write_status() self.mark_readiness() + self.write_status(self.derive_conditions()) def compose_network(self) -> None: resource.update( @@ -631,8 +678,9 @@ def compose_storage_class(self) -> None: ) self.rsp.desired.resources["storage-class-rwx"].ready = fnv1.READY_TRUE - def write_status(self) -> None: + def write_status(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: status = v1alpha1.Status( + nodePools=node_pools, secrets=[ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -686,6 +734,106 @@ def mark_readiness(self) -> None: self.rsp.desired.resources["provider-config-kubernetes"].ready = fnv1.READY_TRUE self.rsp.desired.resources["provider-config-helm"].ready = fnv1.READY_TRUE + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool.""" + return [f"nodepool-{pool.name}"] + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def _node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each user node pool, from its resources.""" + statuses = [] + for pool in self.xr.spec.nodePools: + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self) -> list[v1alpha1.NodePoolModel]: + """Derive ClusterReady and NodePoolsReady from the readiness + mark_readiness set, and return each node pool's state for + status.nodePools. + + NodePoolsReady is True only once ClusterReady is, so the first False + condition names the stage provisioning is in. A failed pool is reported + Failed even while the cluster is still provisioning. + """ + node_pools = self._node_pool_statuses() + in_pools = {r for pool in self.xr.spec.nodePools for r in self.pool_resources(pool)} + cluster = sorted(r for r in self.rsp.desired.resources if r not in in_pools) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True": + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) + return node_pools + def observed_sa_email(self) -> str | None: """Read the service account email from observed state.""" observed_sa = self.req.observed.resources.get("service-account") diff --git a/functions/compose-gke-cluster/tests/test_fn.py b/functions/compose-gke-cluster/tests/test_fn.py index fe12aabf4..fd5e1e346 100644 --- a/functions/compose-gke-cluster/tests/test_fn.py +++ b/functions/compose-gke-cluster/tests/test_fn.py @@ -39,6 +39,25 @@ class Case: want: fnv1.RunFunctionResponse +@dataclasses.dataclass +class Derived: + """What derive_conditions reports: each node pool's state, and the conditions it sets.""" + + node_pools: list[dict] + conditions: list[fnv1.Condition] + + +@dataclasses.dataclass +class DeriveConditionsCase: + """A test case for Composer.derive_conditions.""" + + name: str + reason: str + req: fnv1.RunFunctionRequest + desired: fnv1.State + want: Derived + + def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.NodePool]) -> fnv1.Resource: """The observed GKECluster XR, with the given credentials and node pools.""" xr = v1alpha1.GKECluster( @@ -55,12 +74,13 @@ def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.N return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr() -> fnv1.Resource: - """The desired XR, publishing its connection Secrets and cache StorageClass.""" +def _desired_xr(*, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR, publishing node_pools' states, its connection Secrets and cache StorageClass.""" return fnv1.Resource( resource=resource.dict_to_struct( { "status": { + "nodePools": node_pools, "secrets": [ { "type": "Kubeconfig", @@ -392,6 +412,27 @@ def _provider_config(*, api_version: str) -> fnv1.Resource: ) +def _observed_failed(*, message: str) -> fnv1.Resource: + """An observed managed resource whose provider reports its last operation failed with message.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "conditions": [ + { + "type": "LastAsyncOperation", + "status": "False", + "reason": "AsyncCreateFailure", + "message": message, + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + ), + ) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -424,7 +465,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-pool", "state": "Provisioning"}]), resources={ "network": _network( cred_kind="ClusterProviderConfig", @@ -450,6 +491,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 9 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "gcp-provider-config": fnv1.ResourceSelector( @@ -486,7 +541,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "cpu-pool", "state": "Provisioning"}]), resources={ "network": _network( cred_kind="ClusterProviderConfig", @@ -512,6 +567,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 9 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "gcp-provider-config": fnv1.ResourceSelector( @@ -527,7 +596,7 @@ def _to_dict(msg: message.Message) -> dict: # the function marks it not ready. Case( name="ProviderConfigMissing", - reason="With the GCP provider config missing and no cluster observed to take the project from, a GKECluster composes nothing and isn't ready.", + reason="With the GCP provider config missing and no cluster observed to take the project from, a GKECluster composes nothing, isn't ready, and reports ClusterReady waiting for the provider config.", req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_xr( @@ -554,6 +623,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="Waiting for GCP ClusterProviderConfig default", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "gcp-provider-config": fnv1.ResourceSelector( @@ -644,7 +727,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-pool", "state": "Provisioning"}]), resources={ "network": _network( cred_kind="ClusterProviderConfig", @@ -723,6 +806,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="5 of 11 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "gcp-provider-config": fnv1.ResourceSelector( @@ -784,7 +881,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-pool", "state": "Provisioning"}]), resources={ "network": _network( cred_kind="ProviderConfig", @@ -826,6 +923,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 10 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], # A ProviderConfig, unlike a ClusterProviderConfig, is namespaced, # so the function requires it from the XR's namespace. Crossplane # wouldn't return the request's crossplane-system one for this @@ -850,3 +961,304 @@ def test_compose(case: Case) -> None: """RunFunction composes GKE cluster infrastructure.""" got = asyncio.run(fn.FunctionRunner().RunFunction(case.req, None)) assert _to_dict(got) == _to_dict(case.want), case.reason + + +# Each case's XR has two pools, gpu-l4 and gpu-a100, each built by its node +# pool. desired is what the function composed, with the readiness +# mark_readiness set. +DERIVE_CONDITIONS_CASES = [ + DeriveConditionsCase( + name="ClusterProvisioning", + reason="With cluster resources unready, ClusterReady counts how many are ready and NodePoolsReady waits on it while each pool provisions.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l4", + role="GPU", + machineType="g2-standard-8", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l4", acceleratorCount=1), + ), + v1alpha1.NodePool( + name="gpu-a100", + role="GPU", + machineType="a2-highgpu-8g", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-tesla-a100", acceleratorCount=8), + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "subnet": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodepool-system": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "service-account": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "service-account-key": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodepool-gpu-l4": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodepool-gpu-a100": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l4", "state": "Provisioning"}, {"name": "gpu-a100", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolsProvisioning", + reason="With every cluster resource ready and the gpu-a100 pool's node pool unready, only gpu-l4 is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l4", + role="GPU", + machineType="g2-standard-8", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l4", acceleratorCount=1), + ), + v1alpha1.NodePool( + name="gpu-a100", + role="GPU", + machineType="a2-highgpu-8g", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-tesla-a100", acceleratorCount=8), + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "subnet": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodepool-system": fnv1.Resource(ready=fnv1.READY_TRUE), + "service-account": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodepool-gpu-l4": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodepool-gpu-a100": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l4", "state": "Ready"}, {"name": "gpu-a100", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 node pools ready", + ), + ], + ), + ), + DeriveConditionsCase( + name="AllReady", + reason="With every resource ready, ClusterReady and NodePoolsReady are True and every pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l4", + role="GPU", + machineType="g2-standard-8", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l4", acceleratorCount=1), + ), + v1alpha1.NodePool( + name="gpu-a100", + role="GPU", + machineType="a2-highgpu-8g", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-tesla-a100", acceleratorCount=8), + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "subnet": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodepool-system": fnv1.Resource(ready=fnv1.READY_TRUE), + "service-account": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodepool-gpu-l4": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodepool-gpu-a100": fnv1.Resource(ready=fnv1.READY_TRUE), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l4", "state": "Ready"}, {"name": "gpu-a100", "state": "Ready"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolFailed", + reason="With the provider reporting a pool's node pool create failed, the pool is Failed with the provider's error, even while the cluster provisions.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l4", + role="GPU", + machineType="g2-standard-8", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l4", acceleratorCount=1), + ), + v1alpha1.NodePool( + name="gpu-a100", + role="GPU", + machineType="a2-highgpu-8g", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-tesla-a100", acceleratorCount=8), + ), + ], + ), + resources={ + "nodepool-gpu-a100": _observed_failed( + message="async create failed: googleapi: Error 403: Insufficient regional quota to satisfy request" + ), + }, + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodepool-gpu-l4": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodepool-gpu-a100": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[ + {"name": "gpu-l4", "state": "Provisioning"}, + { + "name": "gpu-a100", + "state": "Failed", + "message": "nodepool-gpu-a100: async create failed: googleapi: Error 403: Insufficient regional quota to satisfy request", + }, + ], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 2 node pools failed: gpu-a100", + ), + ], + ), + ), + DeriveConditionsCase( + name="ClusterFailed", + reason="With the provider reporting two cluster resources failed, ClusterReady is Failed with the first one's error.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l4", + role="GPU", + machineType="g2-standard-8", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l4", acceleratorCount=1), + ), + v1alpha1.NodePool( + name="gpu-a100", + role="GPU", + machineType="a2-highgpu-8g", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-tesla-a100", acceleratorCount=8), + ), + ], + ), + resources={ + "cluster": _observed_failed( + message="async create failed: googleapi: Error 400: Master version unsupported" + ), + "subnet": _observed_failed( + message="async create failed: googleapi: Error 400: Invalid IPCidrRange" + ), + }, + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "subnet": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodepool-gpu-l4": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodepool-gpu-a100": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l4", "state": "Provisioning"}, {"name": "gpu-a100", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="cluster: async create failed: googleapi: Error 400: Master version unsupported (and 1 more)", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), +] + + +@pytest.mark.parametrize("case", DERIVE_CONDITIONS_CASES, ids=lambda case: case.name) +def test_derive_conditions(case: DeriveConditionsCase) -> None: + """derive_conditions reports which provisioning stage the cluster is in, and each node pool's state.""" + rsp = fnv1.RunFunctionResponse(desired=case.desired) + node_pools = fn.Composer(case.req, rsp).derive_conditions() + got = { + "nodePools": [p.model_dump(exclude_none=True) for p in node_pools], + "conditions": [_to_dict(c) for c in rsp.conditions], + } + want = {"nodePools": case.want.node_pools, "conditions": [_to_dict(c) for c in case.want.conditions]} + assert got == want, case.reason diff --git a/functions/compose-inference-cluster/function/fn.py b/functions/compose-inference-cluster/function/fn.py index 822c54424..5e94b74fa 100644 --- a/functions/compose-inference-cluster/function/fn.py +++ b/functions/compose-inference-cluster/function/fn.py @@ -114,6 +114,9 @@ # Composed resource key for the backend XR. BACKEND_RESOURCE_KEY = "serving-stack" +# Composed resource key for the ClusterProviderConfig that reaches the cluster. +CLUSTER_PROVIDER_CONFIG_RESOURCE_KEY = "cluster-provider-config-kubernetes" + # Composed resource key for the ClusterUsage that blocks the InferenceCluster's # deletion while anything composes objects onto it. # @@ -358,6 +361,8 @@ def compose(self) -> None: source = cluster.source if not self.check_devices_supported(source): return + if not self.check_pools_provisionable(source): + return if source == CLUSTER_SOURCE_GKE: self.compose_gke(cluster.gke) @@ -779,6 +784,37 @@ def check_devices_supported(self, source: Cloud) -> bool: response.fatal(self.rsp, msg) return False + def check_pools_provisionable(self, source: Cloud) -> bool: + """Gate on a node pool whose class has no provisioning block for the cloud. + + The cloud cluster XR can't build a pool without one. Gating is a fatal + result for the same reason as check_devices_supported: a running + cluster whose shared class loses its block holds its resources rather + than having Crossplane collect them. Every offending class is named. + Existing never gates, because it provisions nothing. + """ + if source == CLUSTER_SOURCE_EXISTING: + return True + offending = [ + f"InferenceClass {name} has no {source} provisioning block" + for name in sorted({p.className for p in self.xr.spec.nodePools or []}) + if getattr(self.classes[name].spec.provisioning, source.lower(), None) is None + ] + if not offending: + return True + msg = "; ".join(offending) + response.set_conditions( + self.rsp, + resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_INVALID_NODE_POOL, + message=msg, + ), + ) + response.fatal(self.rsp, msg) + return False + def compose_gke(self, gke: v1alpha1.Gke | None) -> None: """Compose an InferenceCluster backed by a Modelplane-provisioned GKE cluster. Composes the GKECluster XR, waits for it to be ready, @@ -797,7 +833,7 @@ def compose_gke(self, gke: v1alpha1.Gke | None) -> None: sa_key = self.observed_gke_secret(_IDENTITY_TYPE_GCP) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if gke_ready and kubeconfig_secret: + if kubeconfig_secret and self.keep_cluster_provider_config(cluster_ready=gke_ready): self.compose_cluster_provider_config( kubeconfig_secret.name, kubeconfig_secret.key, @@ -843,7 +879,7 @@ def compose_eks(self, eks: v1alpha1.Eks | None) -> None: kubeconfig = self.observed_eks_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if eks_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=eks_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_eks_backend_secrets(eks_ready=eks_ready, backend_exists=backend_exists) @@ -882,7 +918,7 @@ def compose_aks(self, aks: v1alpha1.Aks | None) -> None: kubeconfig = self.observed_aks_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if aks_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=aks_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_aks_backend_secrets(aks_ready=aks_ready, backend_exists=backend_exists) @@ -926,7 +962,7 @@ def compose_nebius(self, nebius: v1alpha1.Nebius | None) -> None: credentials = self.observed_nebius_secret(_IDENTITY_TYPE_NEBIUS) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if nebius_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=nebius_ready): self.compose_cluster_provider_config( kubeconfig.name, kubeconfig.key, @@ -970,7 +1006,7 @@ def compose_vultr(self, vultr: v1alpha1.Vultr | None) -> None: kubeconfig = self.observed_vultr_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if vultr_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=vultr_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_vultr_backend_secrets(vultr_ready=vultr_ready, backend_exists=backend_exists) @@ -1009,7 +1045,7 @@ def compose_civo(self, civo: v1alpha1.Civo | None) -> None: kubeconfig = self.observed_civo_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if civo_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=civo_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_civo_backend_secrets(civo_ready=civo_ready, backend_exists=backend_exists) @@ -1109,6 +1145,18 @@ def compose_serving_stack( ), ) + def keep_cluster_provider_config(self, *, cluster_ready: bool) -> bool: + """Whether to compose the ClusterProviderConfig reaching a provisioned cluster. + + It's first composed once the cloud cluster XR is Ready, then kept while + it's observed. The cloud cluster XR goes unready whenever any resource it + composes does, and every Object on the cluster - its namespaces, + PoolObservations, and every ModelReplica's and ModelRoute's - reaches it + through this ProviderConfig, so dropping it then would have Crossplane + delete a ProviderConfig they all still use. + """ + return cluster_ready or CLUSTER_PROVIDER_CONFIG_RESOURCE_KEY in self.req.observed.resources + def compose_cluster_provider_config( self, kubeconfig_name: str, @@ -1151,10 +1199,10 @@ def compose_cluster_provider_config( ), ) resource.update( - self.rsp.desired.resources["cluster-provider-config-kubernetes"], + self.rsp.desired.resources[CLUSTER_PROVIDER_CONFIG_RESOURCE_KEY], cpc, ) - self.rsp.desired.resources["cluster-provider-config-kubernetes"].ready = fnv1.READY_TRUE + self.rsp.desired.resources[CLUSTER_PROVIDER_CONFIG_RESOURCE_KEY].ready = fnv1.READY_TRUE def write_status(self, schedulable_pools: list[v1alpha1.SchedulablePool]) -> None: """Write the InferenceCluster status.""" @@ -1248,21 +1296,12 @@ def compose_gke_cluster(self, gke: v1alpha1.Gke) -> None: gke_node_pools: list[gkev1alpha1.NodePool] = [] for pool in self.xr.spec.nodePools or []: - cls = self.classes.get(pool.className) - if not cls or not cls.spec.provisioning or not cls.spec.provisioning.gke: - msg = f"InferenceClass {pool.className} has no GKE provisioning block" - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="False", - reason=CONDITION_REASON_INVALID_NODE_POOL, - message=msg, - ), - ) - response.warning(self.rsp, msg) - return - prov = cls.spec.provisioning.gke + provisioning = self.classes[pool.className].spec.provisioning + # Unreachable: check_pools_provisionable gates on every pool's + # class having this block. + if not provisioning or not provisioning.gke: + continue + prov = provisioning.gke # A class with no accelerator describes a CPU pool: a standard # node image, no GPU attachment, and the modelplane.ai/cpu # label and taint composed by compose-gke-cluster. @@ -1319,21 +1358,11 @@ def compose_eks_cluster(self, eks: v1alpha1.Eks) -> None: eks_node_pools: list[eksv1alpha1.NodePool] = [] for pool in self.xr.spec.nodePools or []: - cls = self.classes.get(pool.className) - if not cls or not cls.spec.provisioning or not cls.spec.provisioning.eks: - msg = f"InferenceClass {pool.className} has no EKS provisioning block" - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="False", - reason=CONDITION_REASON_INVALID_NODE_POOL, - message=msg, - ), - ) - response.warning(self.rsp, msg) - return - prov = cls.spec.provisioning.eks + provisioning = self.classes[pool.className].spec.provisioning + # Unreachable (see the GKE builder above). + if not provisioning or not provisioning.eks: + continue + prov = provisioning.eks # A class with no accelerator describes a CPU pool (see the GKE # builder above). node_pool = eksv1alpha1.NodePool( @@ -1398,21 +1427,11 @@ def compose_aks_cluster(self, aks: v1alpha1.Aks) -> None: aks_node_pools: list[aksv1alpha1.NodePool] = [] for pool in self.xr.spec.nodePools or []: - cls = self.classes.get(pool.className) - if not cls or not cls.spec.provisioning or not cls.spec.provisioning.aks: - msg = f"InferenceClass {pool.className} has no AKS provisioning block" - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="False", - reason=CONDITION_REASON_INVALID_NODE_POOL, - message=msg, - ), - ) - response.warning(self.rsp, msg) - return - prov = cls.spec.provisioning.aks + provisioning = self.classes[pool.className].spec.provisioning + # Unreachable (see the GKE builder above). + if not provisioning or not provisioning.aks: + continue + prov = provisioning.aks # A class with no accelerator describes a CPU pool (see the GKE # builder above). node_pool = aksv1alpha1.NodePool( @@ -1472,21 +1491,11 @@ def compose_nebius_cluster(self, nebius: v1alpha1.Nebius) -> None: nebius_node_pools: list[nebiusv1alpha1.NodePool] = [] for pool in self.xr.spec.nodePools or []: - cls = self.classes.get(pool.className) - if not cls or not cls.spec.provisioning or not cls.spec.provisioning.nebius: - msg = f"InferenceClass {pool.className} has no Nebius provisioning block" - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="False", - reason=CONDITION_REASON_INVALID_NODE_POOL, - message=msg, - ), - ) - response.warning(self.rsp, msg) - return - prov = cls.spec.provisioning.nebius + provisioning = self.classes[pool.className].spec.provisioning + # Unreachable (see the GKE builder above). + if not provisioning or not provisioning.nebius: + continue + prov = provisioning.nebius # A class with no accelerator describes a CPU pool (see the GKE # builder above). driversPreset rides the gpu block: a CPU # platform preinstalls no NVIDIA stack. @@ -1633,21 +1642,11 @@ def compose_vultr_cluster(self, vultr: v1alpha1.Vultr) -> None: vultr_node_pools: list[vultrv1alpha1.NodePool] = [] for pool in self.xr.spec.nodePools or []: - cls = self.classes.get(pool.className) - if not cls or not cls.spec.provisioning or not cls.spec.provisioning.vultr: - msg = f"InferenceClass {pool.className} has no Vultr provisioning block" - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="False", - reason=CONDITION_REASON_INVALID_NODE_POOL, - message=msg, - ), - ) - response.warning(self.rsp, msg) - return - prov = cls.spec.provisioning.vultr + provisioning = self.classes[pool.className].spec.provisioning + # Unreachable (see the GKE builder above). + if not provisioning or not provisioning.vultr: + continue + prov = provisioning.vultr # A class with no accelerator describes a CPU pool (see the GKE # builder above). node_pool = vultrv1alpha1.NodePool( @@ -1762,7 +1761,7 @@ def civo_nvlink_disabled_pools(self) -> list[str]: setting to each listed pool through a dedicated NVIDIADriver, so multi-GPU pools in the same cluster keep NVLink between their GPUs. Pools whose class lacks a Civo provisioning block are - skipped here and reported by compose_civo_cluster.""" + skipped here; check_pools_provisionable gates on them.""" pools: list[str] = [] for pool in self.xr.spec.nodePools or []: cls = self.classes.get(pool.className) @@ -1783,21 +1782,11 @@ def compose_civo_cluster(self, civo: v1alpha1.Civo) -> None: civo_node_pools: list[civov1alpha1.NodePool] = [] for pool in self.xr.spec.nodePools or []: - cls = self.classes.get(pool.className) - if not cls or not cls.spec.provisioning or not cls.spec.provisioning.civo: - msg = f"InferenceClass {pool.className} has no Civo provisioning block" - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="False", - reason=CONDITION_REASON_INVALID_NODE_POOL, - message=msg, - ), - ) - response.warning(self.rsp, msg) - return - prov = cls.spec.provisioning.civo + provisioning = self.classes[pool.className].spec.provisioning + # Unreachable (see the GKE builder above). + if not provisioning or not provisioning.civo: + continue + prov = provisioning.civo # A class with no accelerator describes a CPU pool (see the GKE # builder above). node_pool = civov1alpha1.NodePool( diff --git a/functions/compose-inference-cluster/tests/test_fn.py b/functions/compose-inference-cluster/tests/test_fn.py index 6a9e7104d..d336f010a 100644 --- a/functions/compose-inference-cluster/tests/test_fn.py +++ b/functions/compose-inference-cluster/tests/test_fn.py @@ -2689,6 +2689,234 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + # An EKSCluster that was Ready goes unready whenever one of its resources + # does. Every Object on the cluster still reaches it through the + # ClusterProviderConfig, so dropping it would have Crossplane delete it from + # under them. + ComposeCase( + name="EKSUnreadyKeepsCPC", + reason="With its EKSCluster gone unready after composing the ClusterProviderConfig and ServingStack, an EKS cluster keeps both.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster(source="EKS", eks=v1alpha1.Eks(region="us-west-2")), + node_pools=[ + v1alpha1.NodePool( + name="l4-pool", + className="gpu-l4-eks", + nodeCount=2, + maxNodeCount=4, + zones=["us-west-2a", "us-west-2b"], + ), + ], + ), + resources={ + "eks-cluster": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "EKSCluster", + "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "spec": { + "region": "us-west-2", + "nodePools": [ + {"name": "l4-pool", "role": "GPU", "instanceType": "g6.xlarge", "nodeCount": 2}, + ], + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "Creating", + "message": "Unready resources: nodegroup-l4-pool", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + "secrets": [ + { + "type": "Kubeconfig", + "name": "test-cluster-kubeconfig-abcde", + "key": "kubeconfig", + }, + ], + "cache": {"storageClassName": "modelplane-rwx-efs"}, + }, + } + ), + ), + "cluster-provider-config-kubernetes": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "ClusterProviderConfig", + "metadata": {"name": "test-cluster-cluster-kubeconfig-d0f89"}, + } + ), + ), + "serving-stack": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "ServingStack", + "metadata": {"name": "test-cluster-serving-stack-fd00b"}, + "spec": { + "secrets": [ + { + "type": "Kubeconfig", + "name": "test-cluster-kubeconfig-abcde", + "key": "kubeconfig", + }, + ], + }, + } + ), + ), + "activation": _observed_activation_policy( + activated=[ + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", + ] + ), + }, + ), + required_resources={ + "class-gpu-l4-eks": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4-eks", + count=1, + memory="24Gi", + provisioning={ + "provider": "EKS", + "eks": { + "instanceType": "g6.xlarge", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 1}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + schedulable_pools=[ + { + "name": "l4-pool", + "nodes": 4, + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 1, + "capacity": {"memory": {"value": "24Gi"}}, + }, + ], + }, + ], + cache={"storageClassName": "modelplane-rwx-efs"}, + gateway=None, + ), + resources={ + "activation": _activation_policy( + activate=[ + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", + ], + ready=fnv1.READY_TRUE, + ), + "eks-cluster": _eks_cluster( + zones=["us-west-2a", "us-west-2b"], + capacity_block=None, + fabric=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "cluster-provider-config-kubernetes": _cluster_provider_config( + kubeconfig="test-cluster-kubeconfig-abcde", identity=None + ), + "serving-stack": _serving_stack( + cloud="EKS", + dra_drivers=("gpu.nvidia.com",), + secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], + client_cas=None, + gpu=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "usage-eks-by-backend": _backend_usage(cluster_kind="EKSCluster"), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4-eks": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4-eks" + ), + }, + ), + conditions=[ + fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), + fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + ], + ), + ), # compose-eks-cluster turns the pool's capacityBlock into a CAPACITY_BLOCK # node group. ComposeCase( @@ -5885,6 +6113,128 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + # Like UnsupportedDriverHolds: the fatal result keeps Crossplane from + # collecting the running cluster's GKECluster, ServingStack and Usages, and + # the condition still lands. + ComposeCase( + name="InvalidNodePoolHolds", + reason="A running GKE cluster whose pools' classes have no GKE provisioning block gets a fatal result naming each class.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster(source="GKE", gke=v1alpha1.Gke(region="us-central1")), + node_pools=[ + v1alpha1.NodePool(name="l4-pool", className="gpu-l4-eks", nodeCount=2, maxNodeCount=4), + v1alpha1.NodePool(name="h100-pool", className="gpu-h100-aks", nodeCount=1), + ], + ), + resources={ + "gke-cluster": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "GKECluster", + "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2026-06-08T00:00:00Z", + }, + ], + }, + } + ), + ), + }, + ), + required_resources={ + "class-gpu-l4-eks": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4-eks", + count=1, + memory="24Gi", + provisioning={ + "provider": "EKS", + "eks": { + "instanceType": "g6.xlarge", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 1}, + }, + }, + ), + ], + ), + "class-gpu-h100-aks": fnv1.Resources( + items=[ + _inference_class( + name="gpu-h100-aks", + count=1, + memory="80Gi", + provisioning={ + "provider": "AKS", + "aks": { + "vmSize": "Standard_NC40ads_H100_v5", + "diskSizeGb": 256, + "accelerator": {"type": "nvidia-h100", "count": 1}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State(), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_FATAL, + message=( + "InferenceClass gpu-h100-aks has no GKE provisioning block; " + "InferenceClass gpu-l4-eks has no GKE provisioning block" + ), + ), + ], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4-eks": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4-eks" + ), + "class-gpu-h100-aks": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-h100-aks" + ), + }, + ), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="InvalidNodePool", + message=( + "InferenceClass gpu-h100-aks has no GKE provisioning block; " + "InferenceClass gpu-l4-eks has no GKE provisioning block" + ), + ), + ], + ), + ), # The hostname gate, which is what keeps a cluster off the schedule until # traffic to it is mutually authenticated in both directions. These cases # observe an existing cluster's ServingStack ready, with whatever it and the diff --git a/functions/compose-nebius-cluster/function/fn.py b/functions/compose-nebius-cluster/function/fn.py index 9dfa7db89..428545382 100644 --- a/functions/compose-nebius-cluster/function/fn.py +++ b/functions/compose-nebius-cluster/function/fn.py @@ -164,6 +164,38 @@ # modelplane-rwx-efs on EKS. _MANAGED_STORAGE_CLASS = "modelplane-rwx-fs" +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: network, control +# plane, system node group, cache filesystem and its CSI driver and +# StorageClass, and ProviderConfigs. compose-inference-cluster relays both +# conditions onto the InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + + +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + def _name(meta: metav1.ObjectMeta | None) -> str: """The object's name, always set on resources read from the API server.""" @@ -259,8 +291,8 @@ def compose(self) -> None: self.compose_provider_configs(creds) self.compose_csi_driver() self.compose_storage_class() - self.write_status(creds) self.mark_readiness(provider_configs_composed=creds is not None) + self.write_status(creds, self.derive_conditions()) def resolve_credentials(self) -> Credentials | None: """Resolve the Nebius service account credentials Secret consumers @@ -823,7 +855,7 @@ def compose_provider_configs(self, creds: Credentials) -> None: ), ) - def write_status(self, creds: Credentials | None) -> None: + def write_status(self, creds: Credentials | None, node_pools: list[v1alpha1.NodePoolModel]) -> None: secrets = [ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -843,6 +875,7 @@ def write_status(self, creds: Credentials | None) -> None: credential.namespace = creds.namespace secrets.append(credential) status = v1alpha1.Status( + nodePools=node_pools, secrets=secrets, # The RWX StorageClass Modelplane composes for ModelCache. # Published immediately so ModelCache can target it; the class may @@ -878,6 +911,111 @@ def mark_readiness(self, *, provider_configs_composed: bool) -> None: self.rsp.desired.resources["provider-config-kubernetes"].ready = fnv1.READY_TRUE self.rsp.desired.resources["provider-config-helm"].ready = fnv1.READY_TRUE + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool. A GPU cluster is shared by + every pool on its InfiniBand fabric, so it belongs to each of them.""" + resources = [f"nodegroup-{pool.name}"] + fabric = _pool_fabric(pool) + if fabric: + resources.append(f"gpu-cluster-{fabric}") + return resources + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def _node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each user node pool, from its resources.""" + statuses = [] + for pool in self.xr.spec.nodePools: + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self) -> list[v1alpha1.NodePoolModel]: + """Derive ClusterReady and NodePoolsReady from the readiness + mark_readiness set, and return each node pool's state for + status.nodePools. + + NodePoolsReady is True only once ClusterReady is, so the first False + condition names the stage provisioning is in. A failed pool is reported + Failed even while the cluster is still provisioning. + """ + node_pools = self._node_pool_statuses() + in_pools = {r for pool in self.xr.spec.nodePools for r in self.pool_resources(pool)} + cluster = sorted(r for r in self.rsp.desired.resources if r not in in_pools) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True": + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) + return node_pools + def _fabrics(self) -> list[str]: """The distinct InfiniBand fabrics used by the node pools, in first-use order.""" diff --git a/functions/compose-nebius-cluster/tests/test_fn.py b/functions/compose-nebius-cluster/tests/test_fn.py index 419bc18e8..7b4c60369 100644 --- a/functions/compose-nebius-cluster/tests/test_fn.py +++ b/functions/compose-nebius-cluster/tests/test_fn.py @@ -40,6 +40,25 @@ class Case: want: fnv1.RunFunctionResponse +@dataclasses.dataclass +class Derived: + """What derive_conditions reports: each node pool's state, and the conditions it sets.""" + + node_pools: list[dict] + conditions: list[fnv1.Condition] + + +@dataclasses.dataclass +class DeriveConditionsCase: + """A test case for Composer.derive_conditions.""" + + name: str + reason: str + req: fnv1.RunFunctionRequest + desired: fnv1.State + want: Derived + + def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.NodePool]) -> fnv1.Resource: """The observed NebiusCluster XR, with the given credentials and node pools.""" xr = v1alpha1.NebiusCluster( @@ -52,8 +71,8 @@ def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.N return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr(*, credentials_secret: bool) -> fnv1.Resource: - """The desired XR's status, which names the credentials Secret only if credentials_secret.""" +def _desired_xr(*, credentials_secret: bool, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR's status, publishing node_pools' states and naming the credentials Secret only if credentials_secret.""" secrets = [{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-55b57", "key": "kubeconfig"}] if credentials_secret: secrets.append( @@ -66,7 +85,13 @@ def _desired_xr(*, credentials_secret: bool) -> fnv1.Resource: ) return fnv1.Resource( resource=resource.dict_to_struct( - {"status": {"secrets": secrets, "cache": {"storageClassName": "modelplane-rwx-fs"}}}, + { + "status": { + "nodePools": node_pools, + "secrets": secrets, + "cache": {"storageClassName": "modelplane-rwx-fs"}, + } + }, ), ) @@ -436,6 +461,27 @@ def _provider_config(*, api_version: str) -> fnv1.Resource: ) +def _observed_failed(*, message: str) -> fnv1.Resource: + """An observed managed resource whose provider reports its last operation failed with message.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "conditions": [ + { + "type": "LastAsyncOperation", + "status": "False", + "reason": "AsyncCreateFailure", + "message": message, + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + ), + ) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -480,7 +526,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr(credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Pending"}]), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -506,6 +552,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -559,7 +619,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr( + credentials_secret=True, node_pools=[{"name": "cpu-pool", "state": "Provisioning"}] + ), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -587,6 +649,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 8 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -633,7 +709,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=False), + composite=_desired_xr( + credentials_secret=False, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -668,6 +746,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -745,7 +837,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr( + credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -783,6 +877,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 8 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -837,7 +945,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr( + credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -884,6 +994,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 8 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -1180,7 +1304,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr(credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Ready"}]), resources={ "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), @@ -1267,6 +1391,18 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -1328,7 +1464,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr( + credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "network": _network( cred_kind="ProviderConfig", cred_name="my-nebius-account", ready=fnv1.READY_UNSPECIFIED @@ -1359,6 +1497,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 8 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -1454,7 +1606,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr(credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Pending"}]), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -1480,6 +1632,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -1537,7 +1703,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr( + credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -1568,6 +1736,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 8 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -1587,3 +1769,416 @@ def test_compose(case: Case) -> None: """RunFunction composes a NebiusCluster's network, mk8s cluster, node groups and ProviderConfigs.""" got = asyncio.run(fn.FunctionRunner().RunFunction(case.req, None)) assert _to_dict(got) == _to_dict(case.want), case.reason + + +# Each case's XR has two pools: gpu-l40s, built by its node group, and gpu-h100, +# an InfiniBand pool built by its node group and the GPU cluster for fabric-2. +# desired is what the function composed, with the readiness mark_readiness set. +DERIVE_CONDITIONS_CASES = [ + DeriveConditionsCase( + name="ClusterProvisioning", + reason="With cluster resources unready, ClusterReady counts how many are ready and NodePoolsReady waits on it while each pool provisions.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + platform="gpu-l40s-a", + preset="1gpu-8vcpu-32gb", + nodeCount=1, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool( + name="gpu-h100", + role="GPU", + platform="gpu-h100-sxm", + preset="8gpu-128vcpu-1600gb", + nodeCount=2, + fabric=v1alpha1.Fabric( + type="InfiniBand", + infiniband=v1alpha1.Infiniband(fabric="fabric-2"), + ), + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "subnet": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "filesystem": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "cloud-init": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-system": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-gpu-l40s": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-gpu-h100": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "gpu-cluster-fabric-2": fnv1.Resource(ready=fnv1.READY_TRUE), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}, {"name": "gpu-h100", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolsProvisioning", + reason="With every cluster resource ready and the gpu-h100 pool's GPU cluster unready, only gpu-l40s is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + platform="gpu-l40s-a", + preset="1gpu-8vcpu-32gb", + nodeCount=1, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool( + name="gpu-h100", + role="GPU", + platform="gpu-h100-sxm", + preset="8gpu-128vcpu-1600gb", + nodeCount=2, + fabric=v1alpha1.Fabric( + type="InfiniBand", + infiniband=v1alpha1.Infiniband(fabric="fabric-2"), + ), + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "subnet": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "filesystem": fnv1.Resource(ready=fnv1.READY_TRUE), + "cloud-init": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-system": fnv1.Resource(ready=fnv1.READY_TRUE), + "release-csi-mounted-fs-path": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-gpu-l40s": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-gpu-h100": fnv1.Resource(ready=fnv1.READY_TRUE), + "gpu-cluster-fabric-2": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Ready"}, {"name": "gpu-h100", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 node pools ready", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolPending", + reason="With a pool's resources not yet composed, the pool is Pending.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + platform="gpu-l40s-a", + preset="1gpu-8vcpu-32gb", + nodeCount=1, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool( + name="gpu-h100", + role="GPU", + platform="gpu-h100-sxm", + preset="8gpu-128vcpu-1600gb", + nodeCount=2, + fabric=v1alpha1.Fabric( + type="InfiniBand", + infiniband=v1alpha1.Infiniband(fabric="fabric-2"), + ), + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "subnet": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "filesystem": fnv1.Resource(ready=fnv1.READY_TRUE), + "cloud-init": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-system": fnv1.Resource(ready=fnv1.READY_TRUE), + "release-csi-mounted-fs-path": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-gpu-h100": fnv1.Resource(ready=fnv1.READY_TRUE), + "gpu-cluster-fabric-2": fnv1.Resource(ready=fnv1.READY_TRUE), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Pending"}, {"name": "gpu-h100", "state": "Ready"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 node pools ready", + ), + ], + ), + ), + DeriveConditionsCase( + name="AllReady", + reason="With every resource ready, ClusterReady and NodePoolsReady are True and every pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + platform="gpu-l40s-a", + preset="1gpu-8vcpu-32gb", + nodeCount=1, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool( + name="gpu-h100", + role="GPU", + platform="gpu-h100-sxm", + preset="8gpu-128vcpu-1600gb", + nodeCount=2, + fabric=v1alpha1.Fabric( + type="InfiniBand", + infiniband=v1alpha1.Infiniband(fabric="fabric-2"), + ), + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "subnet": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "filesystem": fnv1.Resource(ready=fnv1.READY_TRUE), + "cloud-init": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-system": fnv1.Resource(ready=fnv1.READY_TRUE), + "release-csi-mounted-fs-path": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-gpu-l40s": fnv1.Resource(ready=fnv1.READY_TRUE), + "nodegroup-gpu-h100": fnv1.Resource(ready=fnv1.READY_TRUE), + "gpu-cluster-fabric-2": fnv1.Resource(ready=fnv1.READY_TRUE), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Ready"}, {"name": "gpu-h100", "state": "Ready"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolFailed", + reason="With the provider reporting a pool's GPU cluster create failed, the pool is Failed with the provider's error, even while the cluster provisions.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + platform="gpu-l40s-a", + preset="1gpu-8vcpu-32gb", + nodeCount=1, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool( + name="gpu-h100", + role="GPU", + platform="gpu-h100-sxm", + preset="8gpu-128vcpu-1600gb", + nodeCount=2, + fabric=v1alpha1.Fabric( + type="InfiniBand", + infiniband=v1alpha1.Infiniband(fabric="fabric-2"), + ), + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + ], + ), + resources={ + "gpu-cluster-fabric-2": _observed_failed( + message="async create failed: RESOURCE_EXHAUSTED: GPU cluster quota exceeded" + ), + }, + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-gpu-l40s": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-gpu-h100": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "gpu-cluster-fabric-2": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[ + {"name": "gpu-l40s", "state": "Provisioning"}, + { + "name": "gpu-h100", + "state": "Failed", + "message": "gpu-cluster-fabric-2: async create failed: RESOURCE_EXHAUSTED: GPU cluster quota exceeded", + }, + ], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 2 node pools failed: gpu-h100", + ), + ], + ), + ), + DeriveConditionsCase( + name="ClusterFailed", + reason="With the provider reporting two cluster resources failed, ClusterReady is Failed with the first one's error.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + platform="gpu-l40s-a", + preset="1gpu-8vcpu-32gb", + nodeCount=1, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool( + name="gpu-h100", + role="GPU", + platform="gpu-h100-sxm", + preset="8gpu-128vcpu-1600gb", + nodeCount=2, + fabric=v1alpha1.Fabric( + type="InfiniBand", + infiniband=v1alpha1.Infiniband(fabric="fabric-2"), + ), + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + ], + ), + resources={ + "cluster": _observed_failed( + message="async create failed: INVALID_ARGUMENT: unsupported Kubernetes version" + ), + "subnet": _observed_failed( + message="async create failed: RESOURCE_EXHAUSTED: no free addresses in network pools" + ), + }, + ), + ), + desired=fnv1.State( + resources={ + "network": fnv1.Resource(ready=fnv1.READY_TRUE), + "subnet": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-gpu-l40s": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-gpu-h100": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "gpu-cluster-fabric-2": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}, {"name": "gpu-h100", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="cluster: async create failed: INVALID_ARGUMENT: unsupported Kubernetes version (and 1 more)", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), +] + + +@pytest.mark.parametrize("case", DERIVE_CONDITIONS_CASES, ids=lambda case: case.name) +def test_derive_conditions(case: DeriveConditionsCase) -> None: + """derive_conditions reports which provisioning stage the cluster is in, and each node pool's state.""" + rsp = fnv1.RunFunctionResponse(desired=case.desired) + node_pools = fn.Composer(case.req, rsp).derive_conditions() + got = { + "nodePools": [p.model_dump(exclude_none=True) for p in node_pools], + "conditions": [_to_dict(c) for c in rsp.conditions], + } + want = {"nodePools": case.want.node_pools, "conditions": [_to_dict(c) for c in case.want.conditions]} + assert got == want, case.reason diff --git a/functions/compose-vultr-cluster/function/fn.py b/functions/compose-vultr-cluster/function/fn.py index 12670ba2d..b60571f53 100644 --- a/functions/compose-vultr-cluster/function/fn.py +++ b/functions/compose-vultr-cluster/function/fn.py @@ -110,6 +110,37 @@ " && object.status.numberReady == object.status.desiredNumberScheduled" ) +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: the VKE cluster +# with its inline system node pool, and the provider-kubernetes ProviderConfig. +# compose-inference-cluster relays both conditions onto the InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + + +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + def _name(meta: metav1.ObjectMeta | None) -> str: """The object's name, always set on resources read from the API server.""" @@ -175,8 +206,8 @@ def compose(self) -> None: # hold the cluster un-Ready forever. if any(pool.role == "GPU" for pool in self.xr.spec.nodePools): self.compose_gpu_observer() - self.write_status() self.mark_readiness() + self.write_status(self.derive_conditions()) def _cluster_ready(self) -> bool: return resource.get_condition(self.req.observed.resources.get("cluster"), "Ready").status == "True" @@ -350,8 +381,9 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> vkenodepoolv1beta1.KubernetesNo ), ) - def write_status(self) -> None: + def write_status(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: status = v1alpha1.Status( + nodePools=node_pools, secrets=[ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -378,3 +410,109 @@ def mark_readiness(self) -> None: continue if resource.get_condition(self.req.observed.resources.get(r), "Ready").status == "True": self.rsp.desired.resources[r].ready = fnv1.READY_TRUE + + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool. Every GPU pool includes the + gpu-observer, which stays unready until the NVIDIA operator validator + has rolled out on the GPU nodes, so counting it toward the cluster + would hold ClusterReady on the GPU pools.""" + resources = [f"node-pool-{pool.name}"] + if pool.role == "GPU": + resources.append("gpu-observer") + return resources + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def _node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each user node pool, from its resources.""" + statuses = [] + for pool in self.xr.spec.nodePools: + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self) -> list[v1alpha1.NodePoolModel]: + """Derive ClusterReady and NodePoolsReady from the readiness + mark_readiness set, and return each node pool's state for + status.nodePools. + + NodePoolsReady is True only once ClusterReady is, so the first False + condition names the stage provisioning is in. A failed pool is reported + Failed even while the cluster is still provisioning. + """ + node_pools = self._node_pool_statuses() + in_pools = {r for pool in self.xr.spec.nodePools for r in self.pool_resources(pool)} + cluster = sorted(r for r in self.rsp.desired.resources if r not in in_pools) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True": + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) + return node_pools diff --git a/functions/compose-vultr-cluster/tests/test_fn.py b/functions/compose-vultr-cluster/tests/test_fn.py index 7e735694c..9713f597e 100644 --- a/functions/compose-vultr-cluster/tests/test_fn.py +++ b/functions/compose-vultr-cluster/tests/test_fn.py @@ -40,6 +40,25 @@ class Case: want: fnv1.RunFunctionResponse +@dataclasses.dataclass +class Derived: + """What derive_conditions reports: each node pool's state, and the conditions it sets.""" + + node_pools: list[dict] + conditions: list[fnv1.Condition] + + +@dataclasses.dataclass +class DeriveConditionsCase: + """A test case for Composer.derive_conditions.""" + + name: str + reason: str + req: fnv1.RunFunctionRequest + desired: fnv1.State + want: Derived + + def _xr(*, node_pools: list[v1alpha1.NodePool]) -> fnv1.Resource: """The observed VultrCluster XR, with the given node pools.""" xr = v1alpha1.VultrCluster( @@ -55,12 +74,13 @@ def _xr(*, node_pools: list[v1alpha1.NodePool]) -> fnv1.Resource: return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr() -> fnv1.Resource: - """The desired XR, publishing the cluster's kubeconfig Secret.""" +def _desired_xr(*, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR, publishing node_pools' states and the cluster's kubeconfig Secret.""" return fnv1.Resource( resource=resource.dict_to_struct( { "status": { + "nodePools": node_pools, "secrets": [ { "type": "Kubeconfig", @@ -275,6 +295,27 @@ def _gpu_observer(*, ready: fnv1.Ready) -> fnv1.Resource: ) +def _observed_failed(*, message: str) -> fnv1.Resource: + """An observed managed resource whose provider reports its last operation failed with message.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "conditions": [ + { + "type": "LastAsyncOperation", + "status": "False", + "reason": "AsyncCreateFailure", + "message": message, + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + ), + ) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -302,12 +343,26 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Pending"}]), resources={ "cluster": _cluster(ready=fnv1.READY_UNSPECIFIED), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -334,7 +389,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ "cluster": _cluster(ready=fnv1.READY_TRUE), "node-pool-gpu-l40s": _node_pool_gpu( @@ -347,6 +402,19 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 node pools ready", + ), + ], ), ), Case( @@ -376,7 +444,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "cpu-pool", "state": "Provisioning"}]), resources={ "cluster": _cluster(ready=fnv1.READY_TRUE), "node-pool-cpu-pool": _node_pool_cpu( @@ -388,6 +456,19 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 node pools ready", + ), + ], ), ), Case( @@ -446,7 +527,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ "cluster": _cluster(ready=fnv1.READY_UNSPECIFIED), "node-pool-gpu-l40s": _node_pool_gpu( @@ -459,6 +540,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -521,7 +616,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ "cluster": _cluster(ready=fnv1.READY_UNSPECIFIED), "node-pool-gpu-l40s": _node_pool_gpu( @@ -534,6 +629,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -560,7 +669,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ "cluster": _cluster(ready=fnv1.READY_TRUE), "node-pool-gpu-l40s": _node_pool_gpu( @@ -573,6 +682,19 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 node pools ready", + ), + ], ), ), # The pool's nodeCount and minNodeCount are both 2, so this case can't tell @@ -605,7 +727,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "workers", "state": "Provisioning"}]), resources={ "cluster": _cluster(ready=fnv1.READY_TRUE), "node-pool-workers": fnv1.Resource( @@ -633,6 +755,19 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 node pools ready", + ), + ], ), ), Case( @@ -739,7 +874,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Ready"}]), resources={ "cluster": _cluster(ready=fnv1.READY_TRUE), "node-pool-gpu-l40s": _node_pool_gpu( @@ -752,6 +887,18 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], ), ), ] @@ -762,3 +909,297 @@ def test_compose(case: Case) -> None: """RunFunction composes VKE cluster infrastructure.""" got = asyncio.run(fn.FunctionRunner().RunFunction(case.req, None)) assert _to_dict(got) == _to_dict(case.want), case.reason + + +# Each case's XR has two pools: gpu-l40s, built by its KubernetesNodePool and the +# gpu-observer, and cpu-pool, built by its KubernetesNodePool. desired is what +# the function composed, with the readiness mark_readiness set. +DERIVE_CONDITIONS_CASES = [ + DeriveConditionsCase( + name="ClusterProvisioning", + reason="With the cluster unready, ClusterReady counts how many cluster resources are ready and NodePoolsReady waits on it while each pool provisions.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="vcg-l40s-16c-180g-48vram", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="vc2-6c-16gb"), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "node-pool-gpu-l40s": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "node-pool-cpu-pool": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "gpu-observer": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}, {"name": "cpu-pool", "state": "Provisioning"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolsProvisioning", + reason="With the cluster and both node pools ready but the gpu-observer unready, the GPU pool is Provisioning and only cpu-pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="vcg-l40s-16c-180g-48vram", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="vc2-6c-16gb"), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "node-pool-gpu-l40s": fnv1.Resource(ready=fnv1.READY_TRUE), + "node-pool-cpu-pool": fnv1.Resource(ready=fnv1.READY_TRUE), + "gpu-observer": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}, {"name": "cpu-pool", "state": "Ready"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 node pools ready", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolPending", + reason="With only the cluster composed, neither pool's resources are, so both pools are Pending.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="vcg-l40s-16c-180g-48vram", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="vc2-6c-16gb"), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Pending"}, {"name": "cpu-pool", "state": "Pending"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + DeriveConditionsCase( + name="AllReady", + reason="With every resource ready, ClusterReady and NodePoolsReady are True and every pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="vcg-l40s-16c-180g-48vram", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="vc2-6c-16gb"), + ], + ), + ), + ), + desired=fnv1.State( + resources={ + "cluster": fnv1.Resource(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "node-pool-gpu-l40s": fnv1.Resource(ready=fnv1.READY_TRUE), + "node-pool-cpu-pool": fnv1.Resource(ready=fnv1.READY_TRUE), + "gpu-observer": fnv1.Resource(ready=fnv1.READY_TRUE), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Ready"}, {"name": "cpu-pool", "state": "Ready"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + ), + ), + DeriveConditionsCase( + name="PoolFailed", + reason="With the provider reporting a pool's KubernetesNodePool create failed, the pool is Failed with the provider's error, even while the cluster provisions.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="vcg-l40s-16c-180g-48vram", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="vc2-6c-16gb"), + ], + ), + resources={ + "node-pool-cpu-pool": _observed_failed( + message="async create failed: Plan is not available in the selected region" + ), + }, + ), + ), + desired=fnv1.State( + resources={ + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "provider-config-kubernetes": fnv1.Resource(ready=fnv1.READY_TRUE), + "node-pool-gpu-l40s": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "node-pool-cpu-pool": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + "gpu-observer": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[ + {"name": "gpu-l40s", "state": "Provisioning"}, + { + "name": "cpu-pool", + "state": "Failed", + "message": "node-pool-cpu-pool: async create failed: Plan is not available in the selected region", + }, + ], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 2 node pools failed: cpu-pool", + ), + ], + ), + ), + DeriveConditionsCase( + name="ClusterFailed", + reason="With the provider reporting the cluster's create failed, ClusterReady is Failed with its error and NodePoolsReady waits on it.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="vcg-l40s-16c-180g-48vram", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="vc2-6c-16gb"), + ], + ), + resources={ + "cluster": _observed_failed(message="async create failed: Region is not available"), + }, + ), + ), + desired=fnv1.State( + resources={ + "cluster": fnv1.Resource(ready=fnv1.READY_UNSPECIFIED), + }, + ), + want=Derived( + node_pools=[{"name": "gpu-l40s", "state": "Pending"}, {"name": "cpu-pool", "state": "Pending"}], + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="cluster: async create failed: Region is not available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), +] + + +@pytest.mark.parametrize("case", DERIVE_CONDITIONS_CASES, ids=lambda case: case.name) +def test_derive_conditions(case: DeriveConditionsCase) -> None: + """derive_conditions reports which provisioning stage the cluster is in, and each node pool's state.""" + rsp = fnv1.RunFunctionResponse(desired=case.desired) + node_pools = fn.Composer(case.req, rsp).derive_conditions() + got = { + "nodePools": [p.model_dump(exclude_none=True) for p in node_pools], + "conditions": [_to_dict(c) for c in rsp.conditions], + } + want = {"nodePools": case.want.node_pools, "conditions": [_to_dict(c) for c in case.want.conditions]} + assert got == want, case.reason diff --git a/schemas/.lock.json b/schemas/.lock.json index 1f7f6524b..c7f162118 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,6 +1,6 @@ { "packages": { - "fs://apis": "a214ca0e4ca2772e256e3da42cb23ba4c79d25a648e3eee69acc5c87b9b3dab2", + "fs://apis": "1de6ebfee914bfe44cd8c6693f322d9358160d00ef977982d1f2933ec35b28d5", "git://https://github.com/crossplane/crossplane/cluster/crds": "90d8b72ad8b829f0bcd7d7d5a98eaa0d579f244a", "git://https://github.com/modelplaneai/cluster-agent/cluster/charts/cluster-agent/files/crds": "ca7631d72ef8550a2d2f415f33db1ec52b20c65c", "xpkg://xpkg.upbound.io/upbound/provider-aws-ec2:v2.9.4": "sha256:7a0febdfb8da2d59a9fdcdbc2d476811f00566fcb941880313fc2034b0ca6177", diff --git a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py index d6a4a53a6..4cf9e05f7 100644 --- a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py @@ -342,6 +342,21 @@ class Gateway(BaseModel): """ +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class ProviderConfigRef(BaseModel): name: str | None = None """ @@ -425,6 +440,10 @@ class Status(BaseModel): """ Namespace where the internal XRs (cluster, backend) were created. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools, as the cluster Modelplane provisions reports it. Not set when source is Existing, because Modelplane doesn't provision those pools. + """ providerConfigRef: ProviderConfigRef | None = None schedulablePools: list[SchedulablePool] | None = Field(None, max_length=8) """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py index 95ef71792..268766d9a 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py @@ -156,6 +156,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -180,6 +195,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools. The system node pool Modelplane adds belongs to the cluster itself, so its state is part of the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this AKSCluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py index 2b7d90e04..905cea8a2 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py @@ -118,6 +118,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -138,6 +153,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools. The system node pool Modelplane adds belongs to the cluster itself, so its state is part of the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this CivoCluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py index 192f72dea..04f8f0005 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py @@ -178,6 +178,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -202,6 +217,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools. The system node pool Modelplane adds belongs to the cluster itself, so its state is part of the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this EKSCluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py index 7a408a0d7..2cccf467a 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py @@ -160,6 +160,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -184,6 +199,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools. The system node pool Modelplane adds belongs to the cluster itself, so its state is part of the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this GKECluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py index 4e44a948e..9145a057a 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py @@ -155,6 +155,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -183,6 +198,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools. The system node pool Modelplane adds belongs to the cluster itself, so its state is part of the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by or passed through this cluster. Consumers use these to authenticate to the cluster. Secrets are in the same namespace as this NebiusCluster unless an entry says otherwise. diff --git a/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py index be03c5199..edc8f35a5 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py @@ -118,6 +118,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -138,6 +153,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools. The system node pool Modelplane adds belongs to the cluster itself, so its state is part of the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this VultrCluster.