diff --git a/apis/aksclusters/definition.yaml b/apis/aksclusters/definition.yaml index 3cec68b50..60e6a970c 100644 --- a/apis/aksclusters/definition.yaml +++ b/apis/aksclusters/definition.yaml @@ -219,6 +219,46 @@ spec: status: description: AKSClusterStatus defines the observed state of AKSCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools, + except System pools. System pools run the cluster's controllers + and serving stack and are covered under the ClusterReady + condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/civoclusters/definition.yaml b/apis/civoclusters/definition.yaml index 6200f7336..180a9d8b6 100644 --- a/apis/civoclusters/definition.yaml +++ b/apis/civoclusters/definition.yaml @@ -177,6 +177,46 @@ spec: status: description: CivoClusterStatus defines the observed state of CivoCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools, + except System pools. System pools run the cluster's controllers + and serving stack and are covered under the ClusterReady + condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/coreweaveclusters/definition.yaml b/apis/coreweaveclusters/definition.yaml index de8f223a3..bc0a8ed67 100644 --- a/apis/coreweaveclusters/definition.yaml +++ b/apis/coreweaveclusters/definition.yaml @@ -177,6 +177,46 @@ spec: status: description: CoreWeaveClusterStatus defines the observed state of CoreWeaveCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools, + except System pools. System pools run the cluster's controllers + and serving stack and are covered under the ClusterReady + condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/eksclusters/definition.yaml b/apis/eksclusters/definition.yaml index e3718703b..adde2d2cd 100644 --- a/apis/eksclusters/definition.yaml +++ b/apis/eksclusters/definition.yaml @@ -253,6 +253,46 @@ spec: status: description: EKSClusterStatus defines the observed state of EKSCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools, + except System pools. System pools run the cluster's controllers + and serving stack and are covered under the ClusterReady + condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/gkeclusters/definition.yaml b/apis/gkeclusters/definition.yaml index 2c7165716..d5711864c 100644 --- a/apis/gkeclusters/definition.yaml +++ b/apis/gkeclusters/definition.yaml @@ -225,6 +225,46 @@ spec: status: description: GKEClusterStatus defines the observed state of GKECluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools, + except System pools. System pools run the cluster's controllers + and serving stack and are covered under the ClusterReady + condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/inferenceclusters/definition.yaml b/apis/inferenceclusters/definition.yaml index 606504e41..455678eb0 100644 --- a/apis/inferenceclusters/definition.yaml +++ b/apis/inferenceclusters/definition.yaml @@ -18,6 +18,20 @@ spec: - name: SOURCE type: string jsonPath: .spec.cluster.source + - name: CLUSTER + type: string + description: >- + The ClusterReady reason, for the network, control plane, system nodes + and cluster add-ons. + jsonPath: .status.conditions[?(@.type=='ClusterReady')].reason + - name: POOLS + type: string + description: The NodePoolsReady reason, for the node pools in spec.nodePools. + jsonPath: .status.conditions[?(@.type=='NodePoolsReady')].reason + - name: STACK + type: string + description: The ServingStackReady reason, for the serving stack installed on the cluster. + jsonPath: .status.conditions[?(@.type=='ServingStackReady')].reason - name: GATEWAY type: string jsonPath: .status.gateway.address @@ -706,6 +720,39 @@ spec: status: type: object properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools, + as the cluster Modelplane provisions reports it. Not set + when source is Existing, because Modelplane doesn't + provision those pools. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [name] + items: + type: object + required: [name, state] + properties: + name: + type: string + maxLength: 40 + description: Name of the node pool in spec.nodePools. + state: + type: string + enum: [Pending, Provisioning, Ready, Failed] + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. providerConfigRef: type: object properties: diff --git a/apis/nebiusclusters/definition.yaml b/apis/nebiusclusters/definition.yaml index 171592e49..9d8efa2ef 100644 --- a/apis/nebiusclusters/definition.yaml +++ b/apis/nebiusclusters/definition.yaml @@ -241,6 +241,46 @@ spec: status: description: NebiusClusterStatus defines the observed state of NebiusCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools, + except System pools. System pools run the cluster's controllers + and serving stack and are covered under the ClusterReady + condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/upcloudclusters/definition.yaml b/apis/upcloudclusters/definition.yaml index fe1c66a0e..f2a333d43 100644 --- a/apis/upcloudclusters/definition.yaml +++ b/apis/upcloudclusters/definition.yaml @@ -227,6 +227,46 @@ spec: status: description: UpCloudClusterStatus defines the observed state of UpCloudCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools, + except System pools. System pools run the cluster's controllers + and serving stack and are covered under the ClusterReady + condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 63 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/apis/vultrclusters/definition.yaml b/apis/vultrclusters/definition.yaml index d07d8f43a..11e7d6353 100644 --- a/apis/vultrclusters/definition.yaml +++ b/apis/vultrclusters/definition.yaml @@ -172,6 +172,46 @@ spec: status: description: VultrClusterStatus defines the observed state of VultrCluster. properties: + nodePools: + type: array + description: >- + The provisioning state of each node pool in spec.nodePools, + except System pools. System pools run the cluster's controllers + and serving stack and are covered under the ClusterReady + condition. + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - state + properties: + name: + type: string + description: Name of the node pool in spec.nodePools. + maxLength: 40 + state: + type: string + description: >- + Pending until Modelplane composes the pool's cloud + resources, Provisioning until their provider reports + them all Ready, then Ready. Failed while the provider + reports that its last create, update or delete of one + of them failed. + enum: + - Pending + - Provisioning + - Ready + - Failed + message: + type: string + description: >- + The provider's error, prefixed with the name of the + composed resource it's about. Set only when state is + Failed. secrets: type: array description: >- diff --git a/design/multi-accelerator.md b/design/multi-accelerator.md index 59c391214..949af50ae 100644 --- a/design/multi-accelerator.md +++ b/design/multi-accelerator.md @@ -229,7 +229,7 @@ its own and would protect every cloud, not just this filter's uninstall. its cloud can't serve would provision devices nothing can drive, so two layers reject the mismatch: -1. `compose-inference-cluster` gates with `ClusterReady=False` (reason +1. `compose-inference-cluster` gates with `Accepted=False` (reason `UnsupportedDevices`) when a referenced class claims a driver the cluster's cloud neither tags components with nor declares external. A typo'd or not-yet-supported driver lands here too; unknown and unsupported @@ -257,7 +257,7 @@ provisions: ```yaml status: conditions: - - type: ClusterReady + - type: Accepted status: "False" reason: UnsupportedDevices message: "class amd-mi355x-8x claims gpu.amd.com; Vultr clusters support dra.cpu, gpu.nvidia.com" diff --git a/docs/content/getting-started/build-the-platform.md b/docs/content/getting-started/build-the-platform.md index 3680f1704..793fddc01 100644 --- a/docs/content/getting-started/build-the-platform.md +++ b/docs/content/getting-started/build-the-platform.md @@ -334,6 +334,13 @@ kubectl wait --for=condition=Ready ic/upcloud-fi-hel2 --timeout=20m {{< /tab >}} {{< /tabs >}} +To see which stage provisioning is in, run `kubectl get ic`. The `CLUSTER`, +`POOLS` and `STACK` columns show each stage's progress. `CLUSTER` or `POOLS` +reads `Failed` while the cloud provider reports that it failed to create, update +or delete one of the stage's resources. Not every resource reports failures; +[provisioning status]({{< ref "platform/inference-cluster.md#provisioning-status" >}}) +says which ones do. + {{< hint "note" >}} Modelplane is reconciling the infrastructure against the source of truth, the manifest you just applied. diff --git a/docs/content/platform/inference-cluster.md b/docs/content/platform/inference-cluster.md index 4c569e881..e093349e2 100644 --- a/docs/content/platform/inference-cluster.md +++ b/docs/content/platform/inference-cluster.md @@ -110,6 +110,50 @@ must meet what Modelplane would otherwise set up for you: describes how the collector discovers the exporter, the DCGM fields it needs, and how to check what yours publishes. +## Provisioning status + +An `InferenceCluster` reports how far provisioning has got through four +conditions, one for each stage: + +| Condition | Covers | Reasons while not `True` | +|---|---|---| +| `Accepted` | The node pools' classes and capacity reservations exist and suit the cloud, and the classes claim devices the cloud supports. | `WaitingForClasses`, `WaitingForReservations`, `InvalidNodePool`, `UnsupportedDevices` | +| `ClusterReady` | The network, control plane, system nodes and cluster add-ons. | `Pending`, `Provisioning`, `Failed` | +| `NodePoolsReady` | The node pools in `spec.nodePools`. | `Pending`, `Provisioning`, `Failed` | +| `ServingStackReady` | The serving stack. | `Pending`, `Provisioning` | + +A stage that's done is `True` with the reason `Available`. A stage is `Pending` +while it waits for an earlier one, so while a new cluster provisions, the first +condition that isn't `True` names the stage it's in. Once a stage has started, +it reports its own state: on a running cluster, the serving stack stays +`Available` while a node pool you add is `Provisioning`. + +`Provisioning` means Modelplane is creating or updating the stage's resources. +`ClusterReady`'s message counts how many of them are ready, and +`NodePoolsReady`'s counts the ready pools. + +`Failed` means the cloud provider reports that its last create, update or +delete of one of the stage's resources failed. `ClusterReady`'s message is the +provider's error, prefixed with the resource's name. `NodePoolsReady`'s message +names the failed pools, and each pool's error is in `status.nodePools`. Only +resources the cloud provider creates asynchronously, such as the control plane +and node groups, report failures this way. A failed IAM role, Helm release or +in-cluster object leaves its stage `Provisioning`. + +An `Existing` cluster reports `ClusterReady` and `NodePoolsReady` with the +reason `Existing`, because Modelplane doesn't provision it. + +`kubectl get` shows each stage's reason: + +```bash +kubectl get ic eks-us-east +``` + +```text +NAME SOURCE CLUSTER POOLS STACK GATEWAY SYNCED READY COMPOSITION AGE +eks-us-east EKS Available Provisioning Pending True False inferenceclusters.modelplane.ai 24m +``` + ## Serving stack `spec.stack` selects the serving layer the cluster runs: `Standard` (the default) diff --git a/functions/compose-aks-cluster/function/fn.py b/functions/compose-aks-cluster/function/fn.py index 007a92a69..c142d3479 100644 --- a/functions/compose-aks-cluster/function/fn.py +++ b/functions/compose-aks-cluster/function/fn.py @@ -183,6 +183,38 @@ # Azure caps an AKS cluster's DNS prefix at 54 characters. _DNS_PREFIX_MAX_LEN = 54 +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: resource group, +# network, the AKS cluster with its system pool as the default node pool, and +# the in-cluster network operator and StorageClass. compose-inference-cluster +# relays both conditions onto the InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + + +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + def _name(meta: metav1.ObjectMeta | None) -> str: """The object's name, always set on resources read from the API server.""" @@ -264,8 +296,10 @@ def compose(self) -> None: self.compose_network_operator() self.compose_provider_configs() self.compose_storage_class() - self.write_status() self.mark_readiness() + node_pools = self.node_pool_statuses() + self.derive_conditions(node_pools) + self.write_status(node_pools) def _cred_kind(self) -> str: creds = self.xr.spec.credentials @@ -601,8 +635,9 @@ def compose_provider_configs(self) -> None: ), ) - def write_status(self) -> None: + def write_status(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: status = v1alpha1.Status( + nodePools=node_pools, secrets=[ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -640,6 +675,112 @@ def mark_readiness(self) -> None: self.rsp.desired.resources["provider-config-kubernetes"].ready = fnv1.READY_TRUE self.rsp.desired.resources["provider-config-helm"].ready = fnv1.READY_TRUE + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool. The network operator Helm + release serves every InfiniBand pool, so it's part of the cluster + instead.""" + return [f"nodepool-{pool.name}"] + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def _stage_pools(self) -> list[v1alpha1.NodePool]: + """The node pools NodePoolsReady and status.nodePools report. A System + pool runs the cluster's controllers and serving stack, so it's part of + ClusterReady.""" + return [pool for pool in self.xr.spec.nodePools if pool.role != "System"] + + def node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each node pool, from its resources.""" + statuses = [] + for pool in self._stage_pools(): + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: + """Derive ClusterReady from the readiness mark_readiness set, and + NodePoolsReady from node_pools. + + NodePoolsReady waits on ClusterReady only while no pool is Ready, so on + a running cluster the pools report their own state while the cluster + or a new pool provisions. A failed pool is reported Failed even while + the cluster is still provisioning. + """ + in_pools = {r for pool in self._stage_pools() for r in self.pool_resources(pool)} + cluster = sorted(r for r in self.rsp.desired.resources if r not in in_pools) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True" and not ready: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) + def _networking(self) -> v1alpha1.Networking: """Return the (defaulted) networking config from the XR.""" return self.xr.spec.networking or v1alpha1.Networking() diff --git a/functions/compose-aks-cluster/tests/test_fn.py b/functions/compose-aks-cluster/tests/test_fn.py index 180bd9980..a921782d2 100644 --- a/functions/compose-aks-cluster/tests/test_fn.py +++ b/functions/compose-aks-cluster/tests/test_fn.py @@ -52,12 +52,13 @@ def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.N return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr() -> fnv1.Resource: - """The desired XR, publishing its kubeconfig Secret and cache StorageClass.""" +def _desired_xr(*, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR, publishing node_pools' states, its kubeconfig Secret and cache StorageClass.""" return fnv1.Resource( resource=resource.dict_to_struct( { "status": { + "nodePools": node_pools, "secrets": [ { "type": "Kubeconfig", @@ -212,7 +213,7 @@ def _nodepool_gpu(*, cred_kind: str, cred_name: str, zones: list[str] | None, re return fnv1.Resource(resource=resource.dict_to_struct(nodepool), ready=ready) -def _nodepool_cpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _nodepool_cpu(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed cpupool node pool, with the CPU label and taint and no GPU driver.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -244,6 +245,7 @@ def _nodepool_cpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) @@ -271,6 +273,74 @@ def _provider_config(*, api_version: str) -> fnv1.Resource: ) +def _storage_class() -> fnv1.Resource: + """The composed Object holding the RWX StorageClass, ready once created.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "managementPolicies": ["Observe", "Create", "Update"], + "providerConfigRef": { + "kind": "ProviderConfig", + "name": "test-cluster-kubeconfig-55b57", + }, + "readiness": {"policy": "SuccessfulCreate"}, + "forProvider": { + "manifest": { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "metadata": {"name": "modelplane-rwx-fs"}, + "provisioner": "file.csi.azure.com", + "parameters": {"skuName": "Premium_LRS"}, + "mountOptions": [ + "dir_mode=0777", + "file_mode=0777", + "uid=0", + "gid=0", + "mfsymlinks", + "cache=strict", + "actimeo=30", + "nosharesock", + ], + "reclaimPolicy": "Delete", + "allowVolumeExpansion": True, + "volumeBindingMode": "WaitForFirstConsumer", + }, + }, + }, + } + ), + ready=fnv1.READY_TRUE, + ) + + +def _observed_managed_resource(*, ready: bool, failure: str | None) -> fnv1.Resource: + """An observed managed resource with the given Ready status, whose provider + reports its last operation failed with failure unless it's None.""" + conditions = [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Creating", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ] + if failure is not None: + conditions.append( + { + "type": "LastAsyncOperation", + "status": "False", + "reason": "AsyncCreateFailure", + "message": failure, + "lastTransitionTime": "2024-01-01T00:00:00Z", + } + ) + return fnv1.Resource(resource=resource.dict_to_struct({"status": {"conditions": conditions}})) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -304,7 +374,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpuh100", "state": "Provisioning"}]), resources={ "resource-group": _resource_group( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -326,6 +396,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -351,7 +435,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "cpupool", "state": "Provisioning"}]), resources={ "resource-group": _resource_group( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -365,12 +449,28 @@ def _to_dict(msg: message.Message) -> dict: "cluster": _cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "nodepool-cpupool": _nodepool_cpu(cred_kind="ClusterProviderConfig", cred_name="default"), + "nodepool-cpupool": _nodepool_cpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -399,7 +499,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpuh100", "state": "Provisioning"}]), resources={ "resource-group": _resource_group( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -424,6 +524,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -452,7 +566,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpuh100", "state": "Provisioning"}]), resources={ "resource-group": _resource_group( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -474,6 +588,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -554,7 +682,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpuh100", "state": "Provisioning"}]), resources={ "resource-group": _resource_group( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -613,51 +741,26 @@ def _to_dict(msg: message.Message) -> dict: } ), ), - "storage-class-rwx-fs": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", - "kind": "Object", - "metadata": {"namespace": "modelplane-system"}, - "spec": { - "managementPolicies": ["Observe", "Create", "Update"], - "providerConfigRef": { - "kind": "ProviderConfig", - "name": "test-cluster-kubeconfig-55b57", - }, - "readiness": {"policy": "SuccessfulCreate"}, - "forProvider": { - "manifest": { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "metadata": {"name": "modelplane-rwx-fs"}, - "provisioner": "file.csi.azure.com", - "parameters": {"skuName": "Premium_LRS"}, - "mountOptions": [ - "dir_mode=0777", - "file_mode=0777", - "uid=0", - "gid=0", - "mfsymlinks", - "cache=strict", - "actimeo=30", - "nosharesock", - ], - "reclaimPolicy": "Delete", - "allowVolumeExpansion": True, - "volumeBindingMode": "WaitForFirstConsumer", - }, - }, - }, - } - ), - ready=fnv1.READY_TRUE, - ), + "storage-class-rwx-fs": _storage_class(), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="4 of 8 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The ProviderConfigs reach the cluster through its kubeconfig, not the @@ -687,7 +790,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpuh100", "state": "Provisioning"}]), resources={ "resource-group": _resource_group( cred_kind="ProviderConfig", cred_name="my-azure-account", ready=fnv1.READY_UNSPECIFIED @@ -712,6 +815,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -908,7 +1025,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpuh100", "state": "Ready"}]), resources={ "resource-group": _resource_group( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE @@ -918,54 +1035,532 @@ def _to_dict(msg: message.Message) -> dict: ), "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), - "storage-class-rwx-fs": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", - "kind": "Object", - "metadata": {"namespace": "modelplane-system"}, - "spec": { - "managementPolicies": ["Observe", "Create", "Update"], - "providerConfigRef": { - "kind": "ProviderConfig", - "name": "test-cluster-kubeconfig-55b57", - }, - "readiness": {"policy": "SuccessfulCreate"}, - "forProvider": { - "manifest": { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "metadata": {"name": "modelplane-rwx-fs"}, - "provisioner": "file.csi.azure.com", - "parameters": {"skuName": "Premium_LRS"}, - "mountOptions": [ - "dir_mode=0777", - "file_mode=0777", - "uid=0", - "gid=0", - "mfsymlinks", - "cache=strict", - "actimeo=30", - "nosharesock", - ], - "reclaimPolicy": "Delete", - "allowVolumeExpansion": True, - "volumeBindingMode": "WaitForFirstConsumer", - }, - }, - }, - } + "storage-class-rwx-fs": _storage_class(), + "nodepool-gpuh100": _nodepool_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", zones=None, ready=fnv1.READY_TRUE + ), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + ), + ), + Case( + name="StagesClusterProvisioning", + reason="With the KubernetesCluster unready, ClusterReady is Provisioning with a count of its ready resources, and NodePoolsReady is Pending on it while no pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpuh100", + role="GPU", + vmSize="Standard_ND96isr_H100_v5", + diskSizeGb=200, + nodeCount=1, + minNodeCount=1, + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + v1alpha1.NodePool( + name="cpupool", + role="CPU", + vmSize="Standard_D8s_v5", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + ), + ], + ), + resources={ + "resource-group": _observed_managed_resource(ready=True, failure=None), + "cluster": _observed_managed_resource(ready=False, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[ + {"name": "gpuh100", "state": "Provisioning"}, + {"name": "cpupool", "state": "Provisioning"}, + ] + ), + resources={ + "resource-group": _resource_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "virtual-network": _virtual_network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "subnet": _subnet( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-gpuh100": _nodepool_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", zones=None, ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-cpupool": _nodepool_cpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "storage-class-rwx-fs": _storage_class(), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="4 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + Case( + name="StagesPoolsProvisioning", + reason="With every cluster resource Ready and the cpupool node pool unready, ClusterReady is True, gpuh100 is Ready, cpupool is Provisioning, and NodePoolsReady counts the Ready pools.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpuh100", + role="GPU", + vmSize="Standard_ND96isr_H100_v5", + diskSizeGb=200, + nodeCount=1, + minNodeCount=1, + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + v1alpha1.NodePool( + name="cpupool", + role="CPU", + vmSize="Standard_D8s_v5", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + ), + ], + ), + resources={ + "resource-group": _observed_managed_resource(ready=True, failure=None), + "virtual-network": _observed_managed_resource(ready=True, failure=None), + "subnet": _observed_managed_resource(ready=True, failure=None), + "cluster": _observed_managed_resource(ready=True, failure=None), + "nodepool-gpuh100": _observed_managed_resource(ready=True, failure=None), + "nodepool-cpupool": _observed_managed_resource(ready=False, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpuh100", "state": "Ready"}, {"name": "cpupool", "state": "Provisioning"}] + ), + resources={ + "resource-group": _resource_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "virtual-network": _virtual_network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "nodepool-gpuh100": _nodepool_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", zones=None, ready=fnv1.READY_TRUE + ), + "nodepool-cpupool": _nodepool_cpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "storage-class-rwx-fs": _storage_class(), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 node pools ready", + ), + ], + ), + ), + Case( + name="StagesAllReady", + reason="With every resource Ready, ClusterReady and NodePoolsReady are True and both pools are Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpuh100", + role="GPU", + vmSize="Standard_ND96isr_H100_v5", + diskSizeGb=200, + nodeCount=1, + minNodeCount=1, + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + v1alpha1.NodePool( + name="cpupool", + role="CPU", + vmSize="Standard_D8s_v5", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + ), + ], + ), + resources={ + "resource-group": _observed_managed_resource(ready=True, failure=None), + "virtual-network": _observed_managed_resource(ready=True, failure=None), + "subnet": _observed_managed_resource(ready=True, failure=None), + "cluster": _observed_managed_resource(ready=True, failure=None), + "nodepool-gpuh100": _observed_managed_resource(ready=True, failure=None), + "nodepool-cpupool": _observed_managed_resource(ready=True, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpuh100", "state": "Ready"}, {"name": "cpupool", "state": "Ready"}] + ), + resources={ + "resource-group": _resource_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "virtual-network": _virtual_network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "nodepool-gpuh100": _nodepool_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", zones=None, ready=fnv1.READY_TRUE + ), + "nodepool-cpupool": _nodepool_cpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "storage-class-rwx-fs": _storage_class(), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + ), + ), + Case( + name="StagesPoolFailed", + reason="With the provider reporting the gpuh100 node pool's create failed, gpuh100 is Failed with the provider's error prefixed by nodepool-gpuh100, and NodePoolsReady is Failed while ClusterReady is still Provisioning.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpuh100", + role="GPU", + vmSize="Standard_ND96isr_H100_v5", + diskSizeGb=200, + nodeCount=1, + minNodeCount=1, + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + v1alpha1.NodePool( + name="cpupool", + role="CPU", + vmSize="Standard_D8s_v5", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + ), + ], + ), + resources={ + "resource-group": _observed_managed_resource(ready=True, failure=None), + "cluster": _observed_managed_resource(ready=False, failure=None), + "nodepool-gpuh100": _observed_managed_resource( + ready=False, failure="async create failed: SkuNotAvailable" + ), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[ + { + "name": "gpuh100", + "state": "Failed", + "message": "nodepool-gpuh100: async create failed: SkuNotAvailable", + }, + {"name": "cpupool", "state": "Provisioning"}, + ] + ), + resources={ + "resource-group": _resource_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "virtual-network": _virtual_network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "subnet": _subnet( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-gpuh100": _nodepool_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", zones=None, ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-cpupool": _nodepool_cpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "storage-class-rwx-fs": _storage_class(), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="4 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 2 node pools failed: gpuh100", + ), + ], + ), + ), + Case( + name="StagesClusterFailed", + reason="With the provider reporting the cluster and subnet creates failed, ClusterReady is Failed with the cluster's error and a count of the other failure, and NodePoolsReady is Pending on it.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpuh100", + role="GPU", + vmSize="Standard_ND96isr_H100_v5", + diskSizeGb=200, + nodeCount=1, + minNodeCount=1, + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + v1alpha1.NodePool( + name="cpupool", + role="CPU", + vmSize="Standard_D8s_v5", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + ), + ], + ), + resources={ + "resource-group": _observed_managed_resource(ready=True, failure=None), + "subnet": _observed_managed_resource( + ready=False, failure="async create failed: NetcfgInvalidSubnet" + ), + "cluster": _observed_managed_resource( + ready=False, failure="async create failed: K8sVersionNotSupported" + ), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[ + {"name": "gpuh100", "state": "Provisioning"}, + {"name": "cpupool", "state": "Provisioning"}, + ] + ), + resources={ + "resource-group": _resource_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "virtual-network": _virtual_network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "subnet": _subnet( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-gpuh100": _nodepool_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", zones=None, ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-cpupool": _nodepool_cpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "storage-class-rwx-fs": _storage_class(), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="cluster: async create failed: K8sVersionNotSupported (and 1 more)", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + Case( + name="ClusterUnreadyPoolsReady", + reason="With the KubernetesCluster gone unready while both node pools are Ready, ClusterReady is Provisioning and NodePoolsReady reports the pools' own state, True, not Pending.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpuh100", + role="GPU", + vmSize="Standard_ND96isr_H100_v5", + diskSizeGb=200, + nodeCount=1, + minNodeCount=1, + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), ), - ready=fnv1.READY_TRUE, + v1alpha1.NodePool( + name="cpupool", + role="CPU", + vmSize="Standard_D8s_v5", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + ), + ], + ), + resources={ + "resource-group": _observed_managed_resource(ready=True, failure=None), + "virtual-network": _observed_managed_resource(ready=True, failure=None), + "subnet": _observed_managed_resource(ready=True, failure=None), + "cluster": _observed_managed_resource(ready=False, failure=None), + "nodepool-gpuh100": _observed_managed_resource(ready=True, failure=None), + "nodepool-cpupool": _observed_managed_resource(ready=True, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpuh100", "state": "Ready"}, {"name": "cpupool", "state": "Ready"}] + ), + resources={ + "resource-group": _resource_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "virtual-network": _virtual_network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "nodepool-gpuh100": _nodepool_gpu( cred_kind="ClusterProviderConfig", cred_name="default", zones=None, ready=fnv1.READY_TRUE ), + "nodepool-cpupool": _nodepool_cpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "storage-class-rwx-fs": _storage_class(), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="6 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], ), ), ] diff --git a/functions/compose-civo-cluster/function/fn.py b/functions/compose-civo-cluster/function/fn.py index e14eb2839..1bed976af 100644 --- a/functions/compose-civo-cluster/function/fn.py +++ b/functions/compose-civo-cluster/function/fn.py @@ -155,6 +155,38 @@ # nodeCount into forProvider, defeating the purpose.) _AUTOSCALED_POOL_MANAGEMENT: list[_ManagementPolicy] = ["Observe", "Create", "Update", "Delete"] +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: network, +# firewall, the cluster with its inline system node pool, the Helm +# ProviderConfig and the cluster autoscaler. compose-inference-cluster relays +# both conditions onto the InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + + +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + def _name(meta: metav1.ObjectMeta | None) -> str: """The object's name, always set on resources read from the API server.""" @@ -217,8 +249,10 @@ def compose(self) -> None: self.compose_node_pools() self.compose_provider_configs() self.compose_cluster_autoscaler() - self.write_status() self.mark_readiness() + node_pools = self.node_pool_statuses() + self.derive_conditions(node_pools) + self.write_status(node_pools) def _cluster_ready(self) -> bool: return resource.get_condition(self.req.observed.resources.get("cluster"), "Ready").status == "True" @@ -497,8 +531,9 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> nodepoolv1beta1.NodePool: return nodepoolv1beta1.NodePool(spec=spec) - def write_status(self) -> None: + def write_status(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: status = v1alpha1.Status( + nodePools=node_pools, secrets=[ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -524,3 +559,108 @@ def mark_readiness(self) -> None: continue if resource.get_condition(self.req.observed.resources.get(r), "Ready").status == "True": self.rsp.desired.resources[r].ready = fnv1.READY_TRUE + + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool. The autoscaler release + scales every autoscaled pool, so it's part of the cluster instead.""" + return [f"node-pool-{pool.name}"] + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def _stage_pools(self) -> list[v1alpha1.NodePool]: + """The node pools NodePoolsReady and status.nodePools report. A System + pool runs the cluster's controllers and serving stack, so it's part of + ClusterReady.""" + return [pool for pool in self.xr.spec.nodePools if pool.role != "System"] + + def node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each node pool, from its resources.""" + statuses = [] + for pool in self._stage_pools(): + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: + """Derive ClusterReady from the readiness mark_readiness set, and + NodePoolsReady from node_pools. + + NodePoolsReady waits on ClusterReady only while no pool is Ready, so on + a running cluster the pools report their own state while the cluster + or a new pool provisions. A failed pool is reported Failed even while + the cluster is still provisioning. + """ + in_pools = {r for pool in self._stage_pools() for r in self.pool_resources(pool)} + cluster = sorted(r for r in self.rsp.desired.resources if r not in in_pools) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True" and not ready: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) diff --git a/functions/compose-civo-cluster/tests/test_fn.py b/functions/compose-civo-cluster/tests/test_fn.py index f580c995b..22ccc86f1 100644 --- a/functions/compose-civo-cluster/tests/test_fn.py +++ b/functions/compose-civo-cluster/tests/test_fn.py @@ -56,12 +56,13 @@ def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.N return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr() -> fnv1.Resource: - """The desired XR, publishing the cluster's kubeconfig Secret.""" +def _desired_xr(*, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR, publishing node_pools' states and the cluster's kubeconfig Secret.""" return fnv1.Resource( resource=resource.dict_to_struct( { "status": { + "nodePools": node_pools, "secrets": [ { "type": "Kubeconfig", @@ -75,7 +76,7 @@ def _desired_xr() -> fnv1.Resource: ) -def _network(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _network(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed Network.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -91,10 +92,11 @@ def _network(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _firewall(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _firewall(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed Firewall, with Civo's default rules.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -112,6 +114,7 @@ def _firewall(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) @@ -306,7 +309,7 @@ def _provider_config() -> fnv1.Resource: ) -def _autoscaler_release(*, autoscaling_groups: list[dict]) -> fnv1.Resource: +def _autoscaler_release(*, autoscaling_groups: list[dict], ready: fnv1.Ready) -> fnv1.Resource: """The composed cluster autoscaler Release, scaling the given node groups.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -339,9 +342,34 @@ def _autoscaler_release(*, autoscaling_groups: list[dict]) -> fnv1.Resource: }, } ), + ready=ready, ) +def _observed_managed_resource(*, ready: bool, failure: str | None) -> fnv1.Resource: + """An observed managed resource whose Ready condition is True if ready and False if not, and whose + LastAsyncOperation condition is False with failure as its message unless failure is None.""" + conditions = [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Creating", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ] + if failure is not None: + conditions.append( + { + "type": "LastAsyncOperation", + "status": "False", + "reason": "AsyncCreateFailure", + "message": failure, + "lastTransitionTime": "2024-01-01T00:00:00Z", + } + ) + return fnv1.Resource(resource=resource.dict_to_struct({"status": {"conditions": conditions}})) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -370,16 +398,34 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Pending"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), - "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "firewall": _firewall( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 3 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -407,10 +453,14 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), - "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "firewall": _firewall( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), "node-pool-gpu-l40s": _node_pool_gpu( management_policies=["Observe", "Create", "Update", "Delete"], @@ -419,10 +469,26 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_UNSPECIFIED, ), "provider-config-helm": _provider_config(), - "release-cluster-autoscaler": _autoscaler_release(autoscaling_groups=[]), + "release-cluster-autoscaler": _autoscaler_release( + autoscaling_groups=[], ready=fnv1.READY_UNSPECIFIED + ), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -449,10 +515,14 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "cpu-pool", "state": "Provisioning"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), - "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "firewall": _firewall( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), "node-pool-cpu-pool": _node_pool_cpu( management_policies=["Observe", "Create", "Update", "Delete"], @@ -461,10 +531,26 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_UNSPECIFIED, ), "provider-config-helm": _provider_config(), - "release-cluster-autoscaler": _autoscaler_release(autoscaling_groups=[]), + "release-cluster-autoscaler": _autoscaler_release( + autoscaling_groups=[], ready=fnv1.READY_UNSPECIFIED + ), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The autoscaler addresses node groups by the pool ID the provider writes @@ -473,7 +559,7 @@ def _to_dict(msg: message.Message) -> dict: # taken from nodeCount from one fixed at 1. Case( name="PoolIDObserved", - reason="With the GPU pool observed Ready under its Civo ID, a CivoCluster marks the pool ready and has the autoscaler scale that ID from one node to its maxNodeCount.", + reason="With the GPU pool observed Ready under its Civo ID, a CivoCluster marks the pool ready, reports NodePoolsReady True from it while ClusterReady is Provisioning, and has the autoscaler scale that ID from one node to its maxNodeCount.", req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_xr( @@ -535,10 +621,14 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Ready"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), - "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "firewall": _firewall( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), "node-pool-gpu-l40s": _node_pool_gpu( management_policies=["Observe", "Create", "Update", "Delete"], @@ -551,10 +641,24 @@ def _to_dict(msg: message.Message) -> dict: autoscaling_groups=[ {"name": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", "minSize": 1, "maxSize": 4} ], + ready=fnv1.READY_UNSPECIFIED, ), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], ), ), # The observed ProviderConfig shows the dependents were composed before. @@ -616,10 +720,14 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), - "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "firewall": _firewall( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), @@ -630,10 +738,26 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_UNSPECIFIED, ), "provider-config-helm": _provider_config(), - "release-cluster-autoscaler": _autoscaler_release(autoscaling_groups=[]), + "release-cluster-autoscaler": _autoscaler_release( + autoscaling_groups=[], ready=fnv1.READY_UNSPECIFIED + ), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -661,10 +785,14 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), - "firewall": _firewall(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "firewall": _firewall( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), "node-pool-gpu-l40s": _node_pool_gpu( management_policies=None, @@ -673,17 +801,33 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_UNSPECIFIED, ), "provider-config-helm": _provider_config(), - "release-cluster-autoscaler": _autoscaler_release(autoscaling_groups=[]), + "release-cluster-autoscaler": _autoscaler_release( + autoscaling_groups=[], ready=fnv1.READY_UNSPECIFIED + ), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The helm ProviderConfig and the Release reach the cluster through its # kubeconfig, so they don't carry the Civo credentials. Case( name="CustomCredentials", - reason="A CivoCluster naming its own ProviderConfig gets it on every Civo resource, and its System-role pool gets no GPU label or taint.", + reason="A CivoCluster naming its own ProviderConfig gets it on every Civo resource, and its System-role pool gets no GPU label or taint, counts under ClusterReady, and is left out of status.nodePools.", req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_xr( @@ -705,10 +849,10 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[]), resources={ - "network": _network(cred_kind="ProviderConfig", cred_name="team-a"), - "firewall": _firewall(cred_kind="ProviderConfig", cred_name="team-a"), + "network": _network(cred_kind="ProviderConfig", cred_name="team-a", ready=fnv1.READY_UNSPECIFIED), + "firewall": _firewall(cred_kind="ProviderConfig", cred_name="team-a", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster(cred_kind="ProviderConfig", cred_name="team-a", ready=fnv1.READY_TRUE), "node-pool-workers": fnv1.Resource( resource=resource.dict_to_struct( @@ -730,10 +874,448 @@ def _to_dict(msg: message.Message) -> dict: ), ), "provider-config-helm": _provider_config(), - "release-cluster-autoscaler": _autoscaler_release(autoscaling_groups=[]), + "release-cluster-autoscaler": _autoscaler_release( + autoscaling_groups=[], ready=fnv1.READY_UNSPECIFIED + ), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + Case( + name="ClusterProvisioning", + reason="With the firewall and cluster observed unready, ClusterReady counts the 1 of 3 resources ready, NodePoolsReady is Pending on it, and each pool is Pending because its NodePool isn't composed.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + size="an.g1.l40s.kube.x1", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", size="g4s.kube.large", nodeCount=1), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "firewall": _observed_managed_resource(ready=False, failure=None), + "cluster": _observed_cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=False), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Pending"}, {"name": "cpu-pool", "state": "Pending"}] + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "firewall": _firewall( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 3 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + Case( + name="PoolsProvisioning", + reason="With every cluster resource observed Ready and the cpu-pool NodePool unready, ClusterReady is True, NodePoolsReady is Provisioning, and only gpu-l40s is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + size="an.g1.l40s.kube.x1", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", size="g4s.kube.large", nodeCount=1), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "firewall": _observed_managed_resource(ready=True, failure=None), + "cluster": _observed_cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=True), + "provider-config-helm": _observed_managed_resource(ready=True, failure=None), + "release-cluster-autoscaler": _observed_managed_resource(ready=True, failure=None), + "node-pool-gpu-l40s": _observed_managed_resource(ready=True, failure=None), + "node-pool-cpu-pool": _observed_managed_resource(ready=False, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Ready"}, {"name": "cpu-pool", "state": "Provisioning"}] + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "firewall": _firewall( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "node-pool-gpu-l40s": _node_pool_gpu( + management_policies=["Observe", "Create", "Update", "Delete"], + init_node_count=1, + node_count=None, + ready=fnv1.READY_TRUE, + ), + "node-pool-cpu-pool": _node_pool_cpu( + management_policies=None, + init_node_count=None, + node_count=1, + ready=fnv1.READY_UNSPECIFIED, + ), + "provider-config-helm": _provider_config(), + "release-cluster-autoscaler": _autoscaler_release(autoscaling_groups=[], ready=fnv1.READY_TRUE), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 node pools ready", + ), + ], + ), + ), + Case( + name="AllReady", + reason="With every resource observed Ready, ClusterReady and NodePoolsReady are True and every pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + size="an.g1.l40s.kube.x1", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", size="g4s.kube.large", nodeCount=1), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "firewall": _observed_managed_resource(ready=True, failure=None), + "cluster": _observed_cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=True), + "provider-config-helm": _observed_managed_resource(ready=True, failure=None), + "release-cluster-autoscaler": _observed_managed_resource(ready=True, failure=None), + "node-pool-gpu-l40s": _observed_managed_resource(ready=True, failure=None), + "node-pool-cpu-pool": _observed_managed_resource(ready=True, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Ready"}, {"name": "cpu-pool", "state": "Ready"}] + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "firewall": _firewall( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "node-pool-gpu-l40s": _node_pool_gpu( + management_policies=["Observe", "Create", "Update", "Delete"], + init_node_count=1, + node_count=None, + ready=fnv1.READY_TRUE, + ), + "node-pool-cpu-pool": _node_pool_cpu( + management_policies=None, + init_node_count=None, + node_count=1, + ready=fnv1.READY_TRUE, + ), + "provider-config-helm": _provider_config(), + "release-cluster-autoscaler": _autoscaler_release(autoscaling_groups=[], ready=fnv1.READY_TRUE), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + ), + ), + Case( + name="PoolFailed", + reason="With the provider reporting the gpu-l40s NodePool's LastAsyncOperation False, that pool is Failed with the provider's error prefixed by the resource name, and NodePoolsReady is Failed naming it.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + size="an.g1.l40s.kube.x1", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", size="g4s.kube.large", nodeCount=1), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "firewall": _observed_managed_resource(ready=True, failure=None), + "cluster": _observed_cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=True), + "provider-config-helm": _observed_managed_resource(ready=True, failure=None), + "release-cluster-autoscaler": _observed_managed_resource(ready=True, failure=None), + "node-pool-gpu-l40s": _observed_managed_resource( + ready=False, failure="async create failed: insufficient an.g1.l40s.kube.x1 capacity in LON1" + ), + "node-pool-cpu-pool": _observed_managed_resource(ready=True, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[ + { + "name": "gpu-l40s", + "state": "Failed", + "message": "node-pool-gpu-l40s: async create failed: insufficient an.g1.l40s.kube.x1 capacity in LON1", + }, + {"name": "cpu-pool", "state": "Ready"}, + ] + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "firewall": _firewall( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "node-pool-gpu-l40s": _node_pool_gpu( + management_policies=["Observe", "Create", "Update", "Delete"], + init_node_count=1, + node_count=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "node-pool-cpu-pool": _node_pool_cpu( + management_policies=None, + init_node_count=None, + node_count=1, + ready=fnv1.READY_TRUE, + ), + "provider-config-helm": _provider_config(), + "release-cluster-autoscaler": _autoscaler_release(autoscaling_groups=[], ready=fnv1.READY_TRUE), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 2 node pools failed: gpu-l40s", + ), + ], + ), + ), + Case( + name="ClusterFailed", + reason="With the provider reporting the firewall's and cluster's LastAsyncOperation False, ClusterReady is Failed with the first failed resource's error and a count of the rest, and NodePoolsReady is Pending on it.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + size="an.g1.l40s.kube.x1", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", size="g4s.kube.large", nodeCount=1), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "firewall": _observed_managed_resource( + ready=False, failure="async create failed: firewall name test-cluster is already in use" + ), + "cluster": _observed_managed_resource( + ready=False, failure="async create failed: kubernetes version 1.20.0-k3s1 is not supported" + ), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Pending"}, {"name": "cpu-pool", "state": "Pending"}] + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "firewall": _firewall( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="cluster: async create failed: kubernetes version 1.20.0-k3s1 is not supported (and 1 more)", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + Case( + name="ClusterRegressedPoolsReady", + reason="With the cluster observed unready after its pools came up, ClusterReady is Provisioning and NodePoolsReady reports the pools' own state, True, rather than Pending on ClusterReady.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + size="an.g1.l40s.kube.x1", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", size="g4s.kube.large", nodeCount=1), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "firewall": _observed_managed_resource(ready=True, failure=None), + "cluster": _observed_cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=False), + "provider-config-helm": _observed_managed_resource(ready=True, failure=None), + "release-cluster-autoscaler": _observed_managed_resource(ready=True, failure=None), + "node-pool-gpu-l40s": _observed_managed_resource(ready=True, failure=None), + "node-pool-cpu-pool": _observed_managed_resource(ready=True, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Ready"}, {"name": "cpu-pool", "state": "Ready"}] + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "firewall": _firewall( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "node-pool-gpu-l40s": _node_pool_gpu( + management_policies=["Observe", "Create", "Update", "Delete"], + init_node_count=1, + node_count=None, + ready=fnv1.READY_TRUE, + ), + "node-pool-cpu-pool": _node_pool_cpu( + management_policies=None, + init_node_count=None, + node_count=1, + ready=fnv1.READY_TRUE, + ), + "provider-config-helm": _provider_config(), + "release-cluster-autoscaler": _autoscaler_release(autoscaling_groups=[], ready=fnv1.READY_TRUE), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="4 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], ), ), ] diff --git a/functions/compose-coreweave-cluster/function/fn.py b/functions/compose-coreweave-cluster/function/fn.py index 25a1c5f3a..c62efd629 100644 --- a/functions/compose-coreweave-cluster/function/fn.py +++ b/functions/compose-coreweave-cluster/function/fn.py @@ -91,9 +91,8 @@ _CPU_TAINT_EFFECT = "NoSchedule" # A NodePool is Ready once CKS validates it. Readiness ignores AtTarget, -# which changes with every autoscaler step and node replacement, because -# the InferenceCluster stops composing the serving stack's ProviderConfig -# while this XR isn't Ready. +# which changes with every autoscaler step and node replacement, so this XR +# doesn't go unready each time. _NODE_POOL_READY_CEL = ( "has(object.status) && has(object.status.conditions)" " && object.status.conditions.exists(c, c.type == 'Validated' && c.status == 'True')" @@ -123,6 +122,36 @@ # can't contain a dot, which rules out a collision with another pool's. _SETTINGS_SUFFIX = ".settings" +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: the VPC, the CKS +# cluster, its kubeconfig, the system pool, the ProviderConfig, storage classes +# and usages. compose-inference-cluster relays both conditions onto the +# InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# Cluster resources composed only after the CKS cluster is up. ClusterReady +# counts them before they're composed, so it isn't True while only the VPC and +# CKS cluster exist. +_LATE_CLUSTER_RESOURCES = ("kubeconfig", f"node-pool-{_SYSTEM_POOL_NAME}") + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + # Secret type written to XR status. compose-inference-cluster reads this to # wire the kubeconfig into a ClusterProviderConfig. _SECRET_TYPE_KUBECONFIG = "Kubeconfig" @@ -145,6 +174,13 @@ _STORAGE_CLASS_FIELDS = ("provisioner", "parameters", "mountOptions", "allowVolumeExpansion", "volumeBindingMode") +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + + def _name(meta: metav1.ObjectMeta | None) -> str: """The object's name, always set on resources read from the API server.""" if meta is None or meta.name is None: @@ -250,8 +286,10 @@ def compose(self) -> None: # The VPC and cluster can be Ready before the kubeconfig and node # pools exist. self.rsp.desired.composite.ready = fnv1.READY_FALSE - self.write_status() self.mark_readiness() + node_pools = self.node_pool_statuses() + self.derive_conditions(node_pools) + self.write_status(node_pools) def _dependents_ready(self) -> bool: """Whether the cluster is Ready and its kubeconfig Secret exists.""" @@ -669,8 +707,9 @@ def compose_usages(self) -> None: ), ) - def write_status(self) -> None: + def write_status(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: status = v1alpha1.Status( + nodePools=node_pools, secrets=[ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -702,3 +741,104 @@ def mark_readiness(self) -> None: continue if resource.get_condition(self.req.observed.resources.get(r), "Ready").status == "True": self.rsp.desired.resources[r].ready = fnv1.READY_TRUE + + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool.""" + resources = [f"node-pool-{pool.name}"] + if pool.maxNodeCount is not None: + resources.append(f"node-pool-{pool.name}{_SETTINGS_SUFFIX}") + return resources + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each node pool, from its resources.""" + statuses = [] + for pool in self.xr.spec.nodePools: + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: + """Derive ClusterReady from the readiness mark_readiness set, and + NodePoolsReady from node_pools. + + NodePoolsReady waits on ClusterReady only while no pool is Ready, so on + a running cluster the pools report their own state while the cluster + or a new pool provisions. A failed pool is reported Failed even while + the cluster is still provisioning. + """ + in_pools = {r for pool in self.xr.spec.nodePools for r in self.pool_resources(pool)} + cluster = sorted({r for r in self.rsp.desired.resources if r not in in_pools} | set(_LATE_CLUSTER_RESOURCES)) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True" and not ready: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) diff --git a/functions/compose-coreweave-cluster/tests/test_fn.py b/functions/compose-coreweave-cluster/tests/test_fn.py index 8c89b5d21..3cf54bedd 100644 --- a/functions/compose-coreweave-cluster/tests/test_fn.py +++ b/functions/compose-coreweave-cluster/tests/test_fn.py @@ -102,12 +102,13 @@ def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.N return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr(*, ready: fnv1.Ready) -> fnv1.Resource: - """The desired XR, publishing the kubeconfig Secret and the cache StorageClass.""" +def _desired_xr(*, ready: fnv1.Ready, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR, publishing the kubeconfig Secret, the cache StorageClass and node_pools' states.""" return fnv1.Resource( resource=resource.dict_to_struct( { "status": { + "nodePools": node_pools, "secrets": [ { "type": "Kubeconfig", @@ -574,7 +575,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(ready=fnv1.READY_FALSE), + composite=_desired_xr(ready=fnv1.READY_FALSE, node_pools=[{"name": "gpu-h100", "state": "Pending"}]), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster( @@ -590,6 +591,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "coreweave-provider-config": fnv1.ResourceSelector( @@ -647,7 +662,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(ready=fnv1.READY_FALSE), + composite=_desired_xr(ready=fnv1.READY_FALSE, node_pools=[{"name": "gpu-h100", "state": "Pending"}]), resources={ "vpc": _vpc(cred_kind="ProviderConfig", cred_name="team", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster(cred_kind="ProviderConfig", cred_name="team", ready=fnv1.READY_UNSPECIFIED), @@ -661,6 +676,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "coreweave-provider-config": fnv1.ResourceSelector( @@ -713,7 +742,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(ready=fnv1.READY_FALSE), + composite=_desired_xr(ready=fnv1.READY_FALSE, node_pools=[{"name": "gpu-h100", "state": "Pending"}]), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), @@ -722,6 +751,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "coreweave-provider-config": fnv1.ResourceSelector( @@ -775,7 +818,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(ready=fnv1.READY_UNSPECIFIED), + composite=_desired_xr( + ready=fnv1.READY_UNSPECIFIED, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), @@ -809,6 +854,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="7 of 9 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "coreweave-provider-config": fnv1.ResourceSelector( @@ -875,7 +934,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(ready=fnv1.READY_UNSPECIFIED), + composite=_desired_xr( + ready=fnv1.READY_UNSPECIFIED, node_pools=[{"name": "gpu-h100", "state": "Ready"}] + ), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), @@ -923,6 +984,19 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="8 of 9 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], requirements=fnv1.Requirements( resources={ "coreweave-provider-config": fnv1.ResourceSelector( @@ -984,7 +1058,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(ready=fnv1.READY_UNSPECIFIED), + composite=_desired_xr( + ready=fnv1.READY_UNSPECIFIED, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), @@ -1032,6 +1108,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="8 of 9 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "coreweave-provider-config": fnv1.ResourceSelector( @@ -1127,7 +1217,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(ready=fnv1.READY_UNSPECIFIED), + composite=_desired_xr( + ready=fnv1.READY_UNSPECIFIED, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), @@ -1194,6 +1286,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="8 of 10 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "coreweave-provider-config": fnv1.ResourceSelector( @@ -1246,7 +1352,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(ready=fnv1.READY_UNSPECIFIED), + composite=_desired_xr( + ready=fnv1.READY_UNSPECIFIED, node_pools=[{"name": "cpu-pool", "state": "Provisioning"}] + ), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), @@ -1276,6 +1384,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="7 of 9 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "coreweave-provider-config": fnv1.ResourceSelector( @@ -1405,7 +1527,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(ready=fnv1.READY_UNSPECIFIED), + composite=_desired_xr( + ready=fnv1.READY_UNSPECIFIED, node_pools=[{"name": "gpu-h100", "state": "Ready"}] + ), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), @@ -1456,6 +1580,18 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], requirements=fnv1.Requirements( resources={ "coreweave-provider-config": fnv1.ResourceSelector( @@ -1524,7 +1660,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(ready=fnv1.READY_UNSPECIFIED), + composite=_desired_xr( + ready=fnv1.READY_UNSPECIFIED, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster( @@ -1577,6 +1715,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="6 of 9 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "coreweave-provider-config": fnv1.ResourceSelector( @@ -1626,7 +1778,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(ready=fnv1.READY_FALSE), + composite=_desired_xr(ready=fnv1.READY_FALSE, node_pools=[{"name": "gpu-h100", "state": "Pending"}]), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster( @@ -1657,6 +1809,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "coreweave-provider-config": fnv1.ResourceSelector( @@ -1697,7 +1863,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(ready=fnv1.READY_FALSE), + composite=_desired_xr(ready=fnv1.READY_FALSE, node_pools=[{"name": "gpu-h100", "state": "Pending"}]), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), @@ -1714,6 +1880,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "coreweave-provider-config": fnv1.ResourceSelector( @@ -1777,7 +1957,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(ready=fnv1.READY_FALSE), + composite=_desired_xr(ready=fnv1.READY_FALSE, node_pools=[{"name": "gpu-h100", "state": "Pending"}]), resources={ "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), @@ -1794,6 +1974,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 5 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "coreweave-provider-config": fnv1.ResourceSelector( diff --git a/functions/compose-eks-cluster/function/fn.py b/functions/compose-eks-cluster/function/fn.py index 062fd1509..6b531e503 100644 --- a/functions/compose-eks-cluster/function/fn.py +++ b/functions/compose-eks-cluster/function/fn.py @@ -330,6 +330,37 @@ def _namespace(meta: metav1.ObjectMeta | None) -> str: # point per PVC inside the auto-provisioned filesystem. _MANAGED_STORAGE_CLASS = "modelplane-rwx-efs" +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: network, IAM, +# control plane, system node group and add-ons. compose-inference-cluster +# relays both conditions onto the InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + + +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + def _kubeconfig_secret_name(xr: v1alpha1.EKSCluster) -> str: """Derive the kubeconfig secret name from the XR.""" @@ -434,8 +465,10 @@ def compose(self) -> None: self.compose_efa_dra_driver() self.compose_provider_configs() self.compose_storage_class() - self.write_status() self.mark_readiness() + node_pools = self.node_pool_statuses() + self.derive_conditions(node_pools) + self.write_status(node_pools) def compose_network(self) -> None: """Compose the VPC, its two subnet tiers, and internet routing. @@ -1700,8 +1733,9 @@ def compose_provider_configs(self) -> None: ), ) - def write_status(self) -> None: + def write_status(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: status = v1alpha1.Status( + nodePools=node_pools, secrets=[ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -1796,6 +1830,114 @@ def mark_readiness(self) -> None: self.rsp.desired.resources["provider-config-kubernetes"].ready = fnv1.READY_TRUE self.rsp.desired.resources["provider-config-helm"].ready = fnv1.READY_TRUE + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool. The EFA security group is + shared by every EFA pool, so it's part of the cluster instead.""" + resources = [f"nodegroup-{pool.name}"] + if pool.capacityReservation or pool.fabric == _FABRIC_EFA: + resources.append(f"launch-template-{pool.name}") + return resources + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def _stage_pools(self) -> list[v1alpha1.NodePool]: + """The node pools NodePoolsReady and status.nodePools report. A System + pool runs the cluster's controllers and serving stack, so it's part of + ClusterReady.""" + return [pool for pool in self.xr.spec.nodePools if pool.role != "System"] + + def node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each node pool, from its resources.""" + statuses = [] + for pool in self._stage_pools(): + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: + """Derive ClusterReady from the readiness mark_readiness set, and + NodePoolsReady from node_pools. + + NodePoolsReady waits on ClusterReady only while no pool is Ready, so on + a running cluster the pools report their own state while the cluster + or a new pool provisions. A failed pool is reported Failed even while + the cluster is still provisioning. + """ + in_pools = {r for pool in self._stage_pools() for r in self.pool_resources(pool)} + cluster = sorted(r for r in self.rsp.desired.resources if r not in in_pools) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True" and not ready: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) + def _networking(self) -> v1alpha1.Networking: """Return the (defaulted) networking config from the XR.""" return self.xr.spec.networking or v1alpha1.Networking() diff --git a/functions/compose-eks-cluster/tests/test_fn.py b/functions/compose-eks-cluster/tests/test_fn.py index 7587f6f34..877c722ff 100644 --- a/functions/compose-eks-cluster/tests/test_fn.py +++ b/functions/compose-eks-cluster/tests/test_fn.py @@ -52,12 +52,13 @@ def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.N return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr() -> fnv1.Resource: - """The desired XR, publishing its kubeconfig Secret and cache StorageClass.""" +def _desired_xr(*, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR, publishing node_pools' states, its kubeconfig Secret and cache StorageClass.""" return fnv1.Resource( resource=resource.dict_to_struct( { "status": { + "nodePools": node_pools, # `type` is emitted because the function sets it explicitly # on the Status model, so update_status (exclude_unset) # keeps it rather than dropping it as an unset field. @@ -78,7 +79,7 @@ def _desired_xr() -> fnv1.Resource: ) -def _vpc(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _vpc(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The VPC.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -96,10 +97,11 @@ def _vpc(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _subnet(*, name: str, az: str, cidr: str, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _subnet(*, name: str, az: str, cidr: str, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """A public subnet in az.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -123,10 +125,13 @@ def _subnet(*, name: str, az: str, cidr: str, cred_kind: str, cred_name: str) -> }, } ), + ready=ready, ) -def _private_subnet(*, name: str, az: str, cidr: str, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _private_subnet( + *, name: str, az: str, cidr: str, cred_kind: str, cred_name: str, ready: fnv1.Ready +) -> fnv1.Resource: """A private subnet in az.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -149,10 +154,11 @@ def _private_subnet(*, name: str, az: str, cidr: str, cred_kind: str, cred_name: }, } ), + ready=ready, ) -def _internet_gateway(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _internet_gateway(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The VPC's internet gateway.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -168,10 +174,11 @@ def _internet_gateway(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _route_table(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _route_table(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The public route table.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -188,10 +195,11 @@ def _route_table(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _route_default(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _route_default(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The public route table's default route, through the internet gateway.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -212,10 +220,11 @@ def _route_default(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _nat_eip(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _nat_eip(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The NAT gateway's elastic IP.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -231,10 +240,11 @@ def _nat_eip(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _nat_gateway(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _nat_gateway(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The NAT gateway, in the first AZ's public subnet.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -257,10 +267,11 @@ def _nat_gateway(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _private_route_table(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _private_route_table(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The private route table.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -277,10 +288,11 @@ def _private_route_table(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _private_route_default(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _private_route_default(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The private route table's default route, through the NAT gateway.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -301,10 +313,11 @@ def _private_route_default(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _route_table_association(*, az: str, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _route_table_association(*, az: str, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The association of az's public subnet with the public route table.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -330,10 +343,11 @@ def _route_table_association(*, az: str, cred_kind: str, cred_name: str) -> fnv1 }, } ), + ready=ready, ) -def _private_route_table_association(*, az: str, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _private_route_table_association(*, az: str, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The association of az's private subnet with the private route table.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -359,10 +373,11 @@ def _private_route_table_association(*, az: str, cred_kind: str, cred_name: str) }, } ), + ready=ready, ) -def _cluster_role(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _cluster_role(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The IAM role the EKS control plane assumes.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -382,10 +397,11 @@ def _cluster_role(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _node_role(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _node_role(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The IAM role the nodes assume.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -405,10 +421,11 @@ def _node_role(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _pod_identity_role(*, role: str, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _pod_identity_role(*, role: str, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """An IAM role a ServiceAccount assumes through EKS Pod Identity.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -428,10 +445,11 @@ def _pod_identity_role(*, role: str, cred_kind: str, cred_name: str) -> fnv1.Res }, } ), + ready=ready, ) -def _role_policy_attachment(*, role: str, arn: str, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _role_policy_attachment(*, role: str, arn: str, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """An attachment of the managed policy arn to the role labelled role.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -450,6 +468,7 @@ def _role_policy_attachment(*, role: str, arn: str, cred_kind: str, cred_name: s }, } ), + ready=ready, ) @@ -554,7 +573,7 @@ def _cluster_auth(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1. ) -def _nodegroup_system(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _nodegroup_system(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed system node group every cluster gets.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -584,10 +603,11 @@ def _nodegroup_system(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _nodegroup_gpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _nodegroup_gpu(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed gpu-l4 node group, which needs no launch template.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -628,6 +648,7 @@ def _nodegroup_gpu(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) @@ -843,7 +864,7 @@ def _nodegroup_gpu_efa() -> fnv1.Resource: ) -def _addon(*, name: str, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _addon(*, name: str, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The EKS addon called name.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -860,6 +881,7 @@ def _addon(*, name: str, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) @@ -880,7 +902,7 @@ def _efs_filesystem(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv ) -def _efs_security_group(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _efs_security_group(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The security group on the EFS mount targets.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -899,10 +921,11 @@ def _efs_security_group(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _efs_security_group_ingress(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _efs_security_group_ingress(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The EFS security group's rule admitting NFS from the VPC.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -925,10 +948,11 @@ def _efs_security_group_ingress(*, cred_kind: str, cred_name: str) -> fnv1.Resou }, } ), + ready=ready, ) -def _efs_mount_target(*, subnet_name: str, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _efs_mount_target(*, subnet_name: str, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """An EFS mount target in the named subnet.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -949,10 +973,11 @@ def _efs_mount_target(*, subnet_name: str, cred_kind: str, cred_name: str) -> fn }, } ), + ready=ready, ) -def _pod_identity_association(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _pod_identity_association(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The Pod Identity association for the EFS CSI controller.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -974,10 +999,11 @@ def _pod_identity_association(*, cred_kind: str, cred_name: str) -> fnv1.Resourc }, } ), + ready=ready, ) -def _autoscaler_policy(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _autoscaler_policy(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The IAM policy the cluster autoscaler needs.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -1010,10 +1036,11 @@ def _autoscaler_policy(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _autoscaler_attachment(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _autoscaler_attachment(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The attachment of the autoscaler's policy to its role.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -1035,10 +1062,11 @@ def _autoscaler_attachment(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _autoscaler_pod_identity(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _autoscaler_pod_identity(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The Pod Identity association for the cluster autoscaler.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -1060,10 +1088,11 @@ def _autoscaler_pod_identity(*, cred_kind: str, cred_name: str) -> fnv1.Resource }, } ), + ready=ready, ) -def _autoscaler_release() -> fnv1.Resource: +def _autoscaler_release(*, ready: fnv1.Ready) -> fnv1.Resource: """The cluster autoscaler's Helm release.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -1095,6 +1124,7 @@ def _autoscaler_release() -> fnv1.Resource: }, } ), + ready=ready, ) @@ -1122,6 +1152,114 @@ def _provider_config(*, api_version: str) -> fnv1.Resource: ) +def _observed_resource(*, ready: bool) -> fnv1.Resource: + """An observed managed resource, Ready or not.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Creating", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + ), + ) + + +def _observed_efs_filesystem(*, ready: bool) -> fnv1.Resource: + """The observed EFS FileSystem, whose external name is the id the StorageClass pins to.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "efs.aws.m.upbound.io/v1beta1", + "kind": "FileSystem", + "metadata": {"annotations": {"crossplane.io/external-name": "fs-0abc123"}}, + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, + "forProvider": { + "region": "us-west-2", + "throughputMode": "elastic", + "encrypted": True, + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Creating", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + ), + ) + + +def _storage_class_rwx_efs() -> fnv1.Resource: + """The composed EFS StorageClass, pinned to the observed FileSystem's id.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "managementPolicies": ["Observe", "Create", "Update"], + "providerConfigRef": { + "kind": "ProviderConfig", + "name": "test-cluster-kubeconfig-55b57", + }, + "readiness": {"policy": "SuccessfulCreate"}, + "forProvider": { + "manifest": { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "metadata": {"name": "modelplane-rwx-efs"}, + "provisioner": "efs.csi.aws.com", + "parameters": { + "provisioningMode": "efs-ap", + "fileSystemId": "fs-0abc123", + "directoryPerms": "700", + }, + "volumeBindingMode": "Immediate", + }, + }, + }, + } + ), + ready=fnv1.READY_TRUE, + ) + + +def _observed_failed(*, message: str) -> fnv1.Resource: + """An observed managed resource whose provider reports its last operation failed with message.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "conditions": [ + { + "type": "LastAsyncOperation", + "status": "False", + "reason": "AsyncCreateFailure", + "message": message, + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + ), + ) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -1153,15 +1291,16 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l4", "state": "Provisioning"}]), resources={ - "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "subnet-0": _subnet( name="test-cluster-subnet-us-west-2a-952dc", az="us-west-2a", cidr="10.0.0.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-1": _subnet( name="test-cluster-subnet-us-west-2b-2b80f", @@ -1169,6 +1308,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.16.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-2": _subnet( name="test-cluster-subnet-us-west-2c-03273", @@ -1176,6 +1316,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.32.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-0": _private_subnet( name="test-cluster-private-subnet-us-west-2a-6a89f", @@ -1183,6 +1324,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.48.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-1": _private_subnet( name="test-cluster-private-subnet-us-west-2b-b7832", @@ -1190,6 +1332,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.64.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-2": _private_subnet( name="test-cluster-private-subnet-us-west-2c-ef57d", @@ -1197,59 +1340,98 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.80.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-eip": _nat_eip( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "internet-gateway": _internet_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-gateway": _nat_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-table": _route_table(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-default": _route_default(cred_kind="ClusterProviderConfig", cred_name="default"), - "private-route-table": _private_route_table(cred_kind="ClusterProviderConfig", cred_name="default"), "private-route-default": _private_route_default( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "route-table-association-0": _route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-1": _route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-2": _route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-0": _private_route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-1": _private_route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-2": _private_route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-cluster": _cluster_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-cluster-policy": _role_policy_attachment( role="cluster", arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-node": _node_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-node-worker": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-cni": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-ecr": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "cluster": _eks_cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -1257,70 +1439,119 @@ def _to_dict(msg: message.Message) -> dict: "cluster-auth": _cluster_auth( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "nodegroup-system": _nodegroup_system(cred_kind="ClusterProviderConfig", cred_name="default"), - "nodegroup-gpu-l4": _nodegroup_gpu(cred_kind="ClusterProviderConfig", cred_name="default"), - "addon-vpc-cni": _addon(name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default"), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodegroup-gpu-l4": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "addon-vpc-cni": _addon( + name="vpc-cni", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), "addon-kube-proxy": _addon( - name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default" + name="kube-proxy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "addon-coredns": _addon( + name="coredns", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), - "addon-coredns": _addon(name="coredns", cred_kind="ClusterProviderConfig", cred_name="default"), "efs-filesystem": _efs_filesystem( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "efs-security-group": _efs_security_group(cred_kind="ClusterProviderConfig", cred_name="default"), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "efs-security-group-ingress": _efs_security_group_ingress( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "efs-mount-target-0": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-1": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2b-b7832", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-2": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-role-efs-csi": _pod_identity_role( - role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default" + role="efs-csi", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-efs-csi": _role_policy_attachment( role="efs-csi", arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "addon-eks-pod-identity-agent": _addon( - name="eks-pod-identity-agent", cred_kind="ClusterProviderConfig", cred_name="default" + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "pod-identity-efs-csi": _pod_identity_association( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "addon-aws-efs-csi-driver": _addon( - name="aws-efs-csi-driver", cred_kind="ClusterProviderConfig", cred_name="default" + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-policy-cluster-autoscaler": _autoscaler_policy( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "iam-role-cluster-autoscaler": _pod_identity_role( - role="cluster-autoscaler", cred_kind="ClusterProviderConfig", cred_name="default" + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-cluster-autoscaler": _autoscaler_attachment( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 49 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -1346,15 +1577,16 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "cpu-pool", "state": "Provisioning"}]), resources={ - "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "subnet-0": _subnet( name="test-cluster-subnet-us-west-2a-952dc", az="us-west-2a", cidr="10.0.0.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-1": _subnet( name="test-cluster-subnet-us-west-2b-2b80f", @@ -1362,6 +1594,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.16.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-2": _subnet( name="test-cluster-subnet-us-west-2c-03273", @@ -1369,6 +1602,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.32.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-0": _private_subnet( name="test-cluster-private-subnet-us-west-2a-6a89f", @@ -1376,6 +1610,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.48.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-1": _private_subnet( name="test-cluster-private-subnet-us-west-2b-b7832", @@ -1383,6 +1618,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.64.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-2": _private_subnet( name="test-cluster-private-subnet-us-west-2c-ef57d", @@ -1390,59 +1626,98 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.80.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-eip": _nat_eip( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "internet-gateway": _internet_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-gateway": _nat_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-table": _route_table(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-default": _route_default(cred_kind="ClusterProviderConfig", cred_name="default"), - "private-route-table": _private_route_table(cred_kind="ClusterProviderConfig", cred_name="default"), "private-route-default": _private_route_default( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "route-table-association-0": _route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-1": _route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-2": _route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-0": _private_route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-1": _private_route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-2": _private_route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-cluster": _cluster_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-cluster-policy": _role_policy_attachment( role="cluster", arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-node": _node_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-node-worker": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-cni": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-ecr": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "cluster": _eks_cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -1450,70 +1725,117 @@ def _to_dict(msg: message.Message) -> dict: "cluster-auth": _cluster_auth( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "nodegroup-system": _nodegroup_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "nodegroup-cpu-pool": _nodegroup_cpu(cred_kind="ClusterProviderConfig", cred_name="default"), - "addon-vpc-cni": _addon(name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default"), + "addon-vpc-cni": _addon( + name="vpc-cni", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), "addon-kube-proxy": _addon( - name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default" + name="kube-proxy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "addon-coredns": _addon( + name="coredns", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), - "addon-coredns": _addon(name="coredns", cred_kind="ClusterProviderConfig", cred_name="default"), "efs-filesystem": _efs_filesystem( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "efs-security-group": _efs_security_group(cred_kind="ClusterProviderConfig", cred_name="default"), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "efs-security-group-ingress": _efs_security_group_ingress( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "efs-mount-target-0": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-1": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2b-b7832", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-2": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-role-efs-csi": _pod_identity_role( - role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default" + role="efs-csi", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-efs-csi": _role_policy_attachment( role="efs-csi", arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "addon-eks-pod-identity-agent": _addon( - name="eks-pod-identity-agent", cred_kind="ClusterProviderConfig", cred_name="default" + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "pod-identity-efs-csi": _pod_identity_association( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "addon-aws-efs-csi-driver": _addon( - name="aws-efs-csi-driver", cred_kind="ClusterProviderConfig", cred_name="default" + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-policy-cluster-autoscaler": _autoscaler_policy( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "iam-role-cluster-autoscaler": _pod_identity_role( - role="cluster-autoscaler", cred_kind="ClusterProviderConfig", cred_name="default" + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-cluster-autoscaler": _autoscaler_attachment( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 49 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The observed filesystem id lets the function compose the StorageClass @@ -1543,74 +1865,24 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "cluster": _observed_eks_cluster(cluster_security_group_id=None, ready_condition=True), - "cluster-auth": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "eks.aws.m.upbound.io/v1beta1", - "kind": "ClusterAuth", - "spec": { - "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, - "forProvider": { - "region": "us-west-2", - "clusterNameSelector": {"matchControllerRef": True}, - }, - "writeConnectionSecretToRef": {"name": "test-cluster-kubeconfig-55b57"}, - }, - "status": { - "conditions": [ - { - "type": "Ready", - "status": "True", - "reason": "Available", - "lastTransitionTime": "2024-01-01T00:00:00Z", - }, - ], - }, - } - ), - ), - "efs-filesystem": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "efs.aws.m.upbound.io/v1beta1", - "kind": "FileSystem", - "metadata": {"annotations": {"crossplane.io/external-name": "fs-0abc123"}}, - "spec": { - "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, - "forProvider": { - "region": "us-west-2", - "throughputMode": "elastic", - "encrypted": True, - }, - }, - "status": { - "conditions": [ - { - "type": "Ready", - "status": "True", - "reason": "Available", - "lastTransitionTime": "2024-01-01T00:00:00Z", - }, - ], - }, - } - ), - ), + "cluster-auth": _observed_resource(ready=True), + "efs-filesystem": _observed_efs_filesystem(ready=True), }, ), ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l4", "state": "Provisioning"}]), resources={ - "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "subnet-0": _subnet( name="test-cluster-subnet-us-west-2a-952dc", az="us-west-2a", cidr="10.0.0.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-1": _subnet( name="test-cluster-subnet-us-west-2b-2b80f", @@ -1618,6 +1890,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.16.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-2": _subnet( name="test-cluster-subnet-us-west-2c-03273", @@ -1625,6 +1898,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.32.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-0": _private_subnet( name="test-cluster-private-subnet-us-west-2a-6a89f", @@ -1632,6 +1906,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.48.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-1": _private_subnet( name="test-cluster-private-subnet-us-west-2b-b7832", @@ -1639,6 +1914,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.64.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-2": _private_subnet( name="test-cluster-private-subnet-us-west-2c-ef57d", @@ -1646,59 +1922,98 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.80.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), - "internet-gateway": _internet_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-gateway": _nat_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-table": _route_table(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-default": _route_default(cred_kind="ClusterProviderConfig", cred_name="default"), - "private-route-table": _private_route_table(cred_kind="ClusterProviderConfig", cred_name="default"), - "private-route-default": _private_route_default( - cred_kind="ClusterProviderConfig", cred_name="default" + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "route-table-association-0": _route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + "nat-eip": _nat_eip( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "route-table-association-1": _route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "route-table-association-2": _route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "private-route-table-association-0": _private_route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "private-route-default": _private_route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-table-association-0": _route_table_association( + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "route-table-association-1": _route_table_association( + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "route-table-association-2": _route_table_association( + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "private-route-table-association-0": _private_route_table_association( + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-1": _private_route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-2": _private_route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-cluster": _cluster_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-cluster-policy": _role_policy_attachment( role="cluster", arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-node": _node_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-node-worker": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-cni": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-ecr": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "cluster": _eks_cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE @@ -1706,103 +2021,121 @@ def _to_dict(msg: message.Message) -> dict: "cluster-auth": _cluster_auth( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE ), - "nodegroup-system": _nodegroup_system(cred_kind="ClusterProviderConfig", cred_name="default"), - "nodegroup-gpu-l4": _nodegroup_gpu(cred_kind="ClusterProviderConfig", cred_name="default"), - "addon-vpc-cni": _addon(name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default"), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodegroup-gpu-l4": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "addon-vpc-cni": _addon( + name="vpc-cni", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), "addon-kube-proxy": _addon( - name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default" + name="kube-proxy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "addon-coredns": _addon( + name="coredns", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), - "addon-coredns": _addon(name="coredns", cred_kind="ClusterProviderConfig", cred_name="default"), "efs-filesystem": _efs_filesystem( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE ), - "efs-security-group": _efs_security_group(cred_kind="ClusterProviderConfig", cred_name="default"), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "efs-security-group-ingress": _efs_security_group_ingress( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "efs-mount-target-0": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-1": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2b-b7832", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-2": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-role-efs-csi": _pod_identity_role( - role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default" + role="efs-csi", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-efs-csi": _role_policy_attachment( role="efs-csi", arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "addon-eks-pod-identity-agent": _addon( - name="eks-pod-identity-agent", cred_kind="ClusterProviderConfig", cred_name="default" + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "pod-identity-efs-csi": _pod_identity_association( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "addon-aws-efs-csi-driver": _addon( - name="aws-efs-csi-driver", cred_kind="ClusterProviderConfig", cred_name="default" + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-policy-cluster-autoscaler": _autoscaler_policy( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "iam-role-cluster-autoscaler": _pod_identity_role( - role="cluster-autoscaler", cred_kind="ClusterProviderConfig", cred_name="default" + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-cluster-autoscaler": _autoscaler_attachment( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( - cred_kind="ClusterProviderConfig", cred_name="default" - ), - "storage-class-rwx-efs": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", - "kind": "Object", - "metadata": {"namespace": "modelplane-system"}, - "spec": { - "managementPolicies": ["Observe", "Create", "Update"], - "providerConfigRef": { - "kind": "ProviderConfig", - "name": "test-cluster-kubeconfig-55b57", - }, - "readiness": {"policy": "SuccessfulCreate"}, - "forProvider": { - "manifest": { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "metadata": {"name": "modelplane-rwx-efs"}, - "provisioner": "efs.csi.aws.com", - "parameters": { - "provisioningMode": "efs-ap", - "fileSystemId": "fs-0abc123", - "directoryPerms": "700", - }, - "volumeBindingMode": "Immediate", - }, - }, - }, - } - ), - ready=fnv1.READY_TRUE, + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "release-cluster-autoscaler": _autoscaler_release(), + "storage-class-rwx-efs": _storage_class_rwx_efs(), + "release-cluster-autoscaler": _autoscaler_release(ready=fnv1.READY_UNSPECIFIED), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="6 of 51 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The launch template targets the reservation through the capacity-block @@ -1837,15 +2170,16 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-h200", "state": "Provisioning"}]), resources={ - "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "subnet-0": _subnet( name="test-cluster-subnet-us-west-2a-952dc", az="us-west-2a", cidr="10.0.0.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-1": _subnet( name="test-cluster-subnet-us-west-2b-2b80f", @@ -1853,6 +2187,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.16.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-2": _subnet( name="test-cluster-subnet-us-west-2c-03273", @@ -1860,6 +2195,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.32.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-0": _private_subnet( name="test-cluster-private-subnet-us-west-2a-6a89f", @@ -1867,6 +2203,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.48.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-1": _private_subnet( name="test-cluster-private-subnet-us-west-2b-b7832", @@ -1874,6 +2211,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.64.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-2": _private_subnet( name="test-cluster-private-subnet-us-west-2c-ef57d", @@ -1881,59 +2219,98 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.80.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-eip": _nat_eip( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "internet-gateway": _internet_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-gateway": _nat_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-table": _route_table(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-default": _route_default(cred_kind="ClusterProviderConfig", cred_name="default"), - "private-route-table": _private_route_table(cred_kind="ClusterProviderConfig", cred_name="default"), "private-route-default": _private_route_default( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "route-table-association-0": _route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-1": _route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-2": _route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-0": _private_route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-1": _private_route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-2": _private_route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-cluster": _cluster_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-cluster-policy": _role_policy_attachment( role="cluster", arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-node": _node_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-node-worker": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-cni": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-ecr": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "cluster": _eks_cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -1941,7 +2318,9 @@ def _to_dict(msg: message.Message) -> dict: "cluster-auth": _cluster_auth( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "nodegroup-system": _nodegroup_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "launch-template-gpu-h200": fnv1.Resource( resource=resource.dict_to_struct( { @@ -2008,68 +2387,113 @@ def _to_dict(msg: message.Message) -> dict: } ), ), - "addon-vpc-cni": _addon(name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default"), + "addon-vpc-cni": _addon( + name="vpc-cni", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), "addon-kube-proxy": _addon( - name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default" + name="kube-proxy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "addon-coredns": _addon( + name="coredns", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), - "addon-coredns": _addon(name="coredns", cred_kind="ClusterProviderConfig", cred_name="default"), "efs-filesystem": _efs_filesystem( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "efs-security-group": _efs_security_group(cred_kind="ClusterProviderConfig", cred_name="default"), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "efs-security-group-ingress": _efs_security_group_ingress( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "efs-mount-target-0": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-1": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2b-b7832", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-2": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-role-efs-csi": _pod_identity_role( - role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default" + role="efs-csi", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-efs-csi": _role_policy_attachment( role="efs-csi", arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "addon-eks-pod-identity-agent": _addon( - name="eks-pod-identity-agent", cred_kind="ClusterProviderConfig", cred_name="default" + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "pod-identity-efs-csi": _pod_identity_association( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "addon-aws-efs-csi-driver": _addon( - name="aws-efs-csi-driver", cred_kind="ClusterProviderConfig", cred_name="default" + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-policy-cluster-autoscaler": _autoscaler_policy( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "iam-role-cluster-autoscaler": _pod_identity_role( - role="cluster-autoscaler", cred_kind="ClusterProviderConfig", cred_name="default" + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-cluster-autoscaler": _autoscaler_attachment( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 49 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # An On-Demand Capacity Reservation needs no market type, and its node group @@ -2103,15 +2527,16 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-h200", "state": "Provisioning"}]), resources={ - "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "subnet-0": _subnet( name="test-cluster-subnet-us-west-2a-952dc", az="us-west-2a", cidr="10.0.0.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-1": _subnet( name="test-cluster-subnet-us-west-2b-2b80f", @@ -2119,6 +2544,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.16.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-2": _subnet( name="test-cluster-subnet-us-west-2c-03273", @@ -2126,6 +2552,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.32.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-0": _private_subnet( name="test-cluster-private-subnet-us-west-2a-6a89f", @@ -2133,6 +2560,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.48.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-1": _private_subnet( name="test-cluster-private-subnet-us-west-2b-b7832", @@ -2140,6 +2568,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.64.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-2": _private_subnet( name="test-cluster-private-subnet-us-west-2c-ef57d", @@ -2147,59 +2576,98 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.80.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-eip": _nat_eip( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "internet-gateway": _internet_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-gateway": _nat_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-table": _route_table(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-default": _route_default(cred_kind="ClusterProviderConfig", cred_name="default"), - "private-route-table": _private_route_table(cred_kind="ClusterProviderConfig", cred_name="default"), "private-route-default": _private_route_default( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "route-table-association-0": _route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-1": _route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-2": _route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-0": _private_route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-1": _private_route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-2": _private_route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-cluster": _cluster_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-cluster-policy": _role_policy_attachment( role="cluster", arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-node": _node_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-node-worker": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-cni": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-ecr": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "cluster": _eks_cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -2207,7 +2675,9 @@ def _to_dict(msg: message.Message) -> dict: "cluster-auth": _cluster_auth( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "nodegroup-system": _nodegroup_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "launch-template-gpu-h200": fnv1.Resource( resource=resource.dict_to_struct( { @@ -2272,68 +2742,113 @@ def _to_dict(msg: message.Message) -> dict: } ), ), - "addon-vpc-cni": _addon(name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default"), + "addon-vpc-cni": _addon( + name="vpc-cni", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), "addon-kube-proxy": _addon( - name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default" + name="kube-proxy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "addon-coredns": _addon( + name="coredns", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), - "addon-coredns": _addon(name="coredns", cred_kind="ClusterProviderConfig", cred_name="default"), "efs-filesystem": _efs_filesystem( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "efs-security-group": _efs_security_group(cred_kind="ClusterProviderConfig", cred_name="default"), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "efs-security-group-ingress": _efs_security_group_ingress( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "efs-mount-target-0": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-1": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2b-b7832", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-2": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-role-efs-csi": _pod_identity_role( - role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default" + role="efs-csi", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-efs-csi": _role_policy_attachment( role="efs-csi", arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "addon-eks-pod-identity-agent": _addon( - name="eks-pod-identity-agent", cred_kind="ClusterProviderConfig", cred_name="default" + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "pod-identity-efs-csi": _pod_identity_association( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "addon-aws-efs-csi-driver": _addon( - name="aws-efs-csi-driver", cred_kind="ClusterProviderConfig", cred_name="default" + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-policy-cluster-autoscaler": _autoscaler_policy( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "iam-role-cluster-autoscaler": _pod_identity_role( - role="cluster-autoscaler", cred_kind="ClusterProviderConfig", cred_name="default" + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-cluster-autoscaler": _autoscaler_attachment( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 49 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The node group's launch template carries one EFA interface per network card @@ -2368,15 +2883,16 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-h200", "state": "Provisioning"}]), resources={ - "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "subnet-0": _subnet( name="test-cluster-subnet-us-west-2a-952dc", az="us-west-2a", cidr="10.0.0.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-1": _subnet( name="test-cluster-subnet-us-west-2b-2b80f", @@ -2384,6 +2900,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.16.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-2": _subnet( name="test-cluster-subnet-us-west-2c-03273", @@ -2391,6 +2908,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.32.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-0": _private_subnet( name="test-cluster-private-subnet-us-west-2a-6a89f", @@ -2398,6 +2916,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.48.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-1": _private_subnet( name="test-cluster-private-subnet-us-west-2b-b7832", @@ -2405,6 +2924,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.64.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-2": _private_subnet( name="test-cluster-private-subnet-us-west-2c-ef57d", @@ -2412,59 +2932,98 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.80.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-eip": _nat_eip( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "internet-gateway": _internet_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-gateway": _nat_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-table": _route_table(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-default": _route_default(cred_kind="ClusterProviderConfig", cred_name="default"), - "private-route-table": _private_route_table(cred_kind="ClusterProviderConfig", cred_name="default"), "private-route-default": _private_route_default( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "route-table-association-0": _route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-1": _route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-2": _route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-0": _private_route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-1": _private_route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-2": _private_route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-cluster": _cluster_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-cluster-policy": _role_policy_attachment( role="cluster", arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-node": _node_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-node-worker": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-cni": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-ecr": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "cluster": _eks_cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -2472,74 +3031,121 @@ def _to_dict(msg: message.Message) -> dict: "cluster-auth": _cluster_auth( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "nodegroup-system": _nodegroup_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "launch-template-gpu-h200": _launch_template_efa(security_groups=None), "efa-security-group": _efa_security_group(), "efa-security-group-ingress": _efa_security_group_ingress(), "efa-security-group-egress": _efa_security_group_egress(), "nodegroup-gpu-h200": _nodegroup_gpu_efa(), - "addon-vpc-cni": _addon(name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default"), + "addon-vpc-cni": _addon( + name="vpc-cni", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), "addon-kube-proxy": _addon( - name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default" + name="kube-proxy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "addon-coredns": _addon( + name="coredns", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), - "addon-coredns": _addon(name="coredns", cred_kind="ClusterProviderConfig", cred_name="default"), "efs-filesystem": _efs_filesystem( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "efs-security-group": _efs_security_group(cred_kind="ClusterProviderConfig", cred_name="default"), - "efs-security-group-ingress": _efs_security_group_ingress( - cred_kind="ClusterProviderConfig", cred_name="default" + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "efs-security-group-ingress": _efs_security_group_ingress( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "efs-mount-target-0": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-1": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2b-b7832", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-2": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-role-efs-csi": _pod_identity_role( - role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default" + role="efs-csi", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-efs-csi": _role_policy_attachment( role="efs-csi", arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "addon-eks-pod-identity-agent": _addon( - name="eks-pod-identity-agent", cred_kind="ClusterProviderConfig", cred_name="default" + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "pod-identity-efs-csi": _pod_identity_association( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "addon-aws-efs-csi-driver": _addon( - name="aws-efs-csi-driver", cred_kind="ClusterProviderConfig", cred_name="default" + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-policy-cluster-autoscaler": _autoscaler_policy( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "iam-role-cluster-autoscaler": _pod_identity_role( - role="cluster-autoscaler", cred_kind="ClusterProviderConfig", cred_name="default" + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-cluster-autoscaler": _autoscaler_attachment( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 52 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # Both security group IDs are observed, the EFA group's in its external name @@ -2599,15 +3205,16 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-h200", "state": "Provisioning"}]), resources={ - "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "subnet-0": _subnet( name="test-cluster-subnet-us-west-2a-952dc", az="us-west-2a", cidr="10.0.0.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-1": _subnet( name="test-cluster-subnet-us-west-2b-2b80f", @@ -2615,6 +3222,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.16.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-2": _subnet( name="test-cluster-subnet-us-west-2c-03273", @@ -2622,6 +3230,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.32.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-0": _private_subnet( name="test-cluster-private-subnet-us-west-2a-6a89f", @@ -2629,6 +3238,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.48.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-1": _private_subnet( name="test-cluster-private-subnet-us-west-2b-b7832", @@ -2636,6 +3246,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.64.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-2": _private_subnet( name="test-cluster-private-subnet-us-west-2c-ef57d", @@ -2643,59 +3254,98 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.80.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-eip": _nat_eip( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "internet-gateway": _internet_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-gateway": _nat_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-table": _route_table(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-default": _route_default(cred_kind="ClusterProviderConfig", cred_name="default"), - "private-route-table": _private_route_table(cred_kind="ClusterProviderConfig", cred_name="default"), "private-route-default": _private_route_default( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "route-table-association-0": _route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-1": _route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-2": _route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-0": _private_route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-1": _private_route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-2": _private_route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-cluster": _cluster_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-cluster-policy": _role_policy_attachment( role="cluster", arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-node": _node_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-node-worker": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-cni": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-ecr": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "cluster": _eks_cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -2703,70 +3353,103 @@ def _to_dict(msg: message.Message) -> dict: "cluster-auth": _cluster_auth( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "nodegroup-system": _nodegroup_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "launch-template-gpu-h200": _launch_template_efa(security_groups=["sg-0efa", "sg-0cluster"]), "efa-security-group": _efa_security_group(), "efa-security-group-ingress": _efa_security_group_ingress(), "efa-security-group-egress": _efa_security_group_egress(), "nodegroup-gpu-h200": _nodegroup_gpu_efa(), - "addon-vpc-cni": _addon(name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default"), + "addon-vpc-cni": _addon( + name="vpc-cni", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), "addon-kube-proxy": _addon( - name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default" + name="kube-proxy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "addon-coredns": _addon( + name="coredns", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), - "addon-coredns": _addon(name="coredns", cred_kind="ClusterProviderConfig", cred_name="default"), "efs-filesystem": _efs_filesystem( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "efs-security-group": _efs_security_group(cred_kind="ClusterProviderConfig", cred_name="default"), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "efs-security-group-ingress": _efs_security_group_ingress( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "efs-mount-target-0": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-1": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2b-b7832", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-2": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-role-efs-csi": _pod_identity_role( - role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default" + role="efs-csi", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-efs-csi": _role_policy_attachment( role="efs-csi", arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "addon-eks-pod-identity-agent": _addon( - name="eks-pod-identity-agent", cred_kind="ClusterProviderConfig", cred_name="default" + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "pod-identity-efs-csi": _pod_identity_association( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "addon-aws-efs-csi-driver": _addon( - name="aws-efs-csi-driver", cred_kind="ClusterProviderConfig", cred_name="default" + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-policy-cluster-autoscaler": _autoscaler_policy( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "iam-role-cluster-autoscaler": _pod_identity_role( - role="cluster-autoscaler", cred_kind="ClusterProviderConfig", cred_name="default" + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-cluster-autoscaler": _autoscaler_attachment( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "release-cluster-autoscaler": _autoscaler_release(), + "release-cluster-autoscaler": _autoscaler_release(ready=fnv1.READY_UNSPECIFIED), "release-efa-dra-driver": fnv1.Resource( resource=resource.dict_to_struct( { @@ -2804,6 +3487,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 54 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # Like the autoscaler, the EFA DRA driver release is gated on the cluster being @@ -2838,15 +3535,16 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-h200", "state": "Provisioning"}]), resources={ - "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "subnet-0": _subnet( name="test-cluster-subnet-us-west-2a-952dc", az="us-west-2a", cidr="10.0.0.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-1": _subnet( name="test-cluster-subnet-us-west-2b-2b80f", @@ -2854,6 +3552,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.16.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-2": _subnet( name="test-cluster-subnet-us-west-2c-03273", @@ -2861,6 +3560,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.32.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-0": _private_subnet( name="test-cluster-private-subnet-us-west-2a-6a89f", @@ -2868,6 +3568,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.48.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-1": _private_subnet( name="test-cluster-private-subnet-us-west-2b-b7832", @@ -2875,6 +3576,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.64.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-2": _private_subnet( name="test-cluster-private-subnet-us-west-2c-ef57d", @@ -2882,59 +3584,98 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.80.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-eip": _nat_eip( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "internet-gateway": _internet_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-gateway": _nat_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-table": _route_table(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-default": _route_default(cred_kind="ClusterProviderConfig", cred_name="default"), - "private-route-table": _private_route_table(cred_kind="ClusterProviderConfig", cred_name="default"), "private-route-default": _private_route_default( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "route-table-association-0": _route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-1": _route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-2": _route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-0": _private_route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-1": _private_route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-2": _private_route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-cluster": _cluster_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-cluster-policy": _role_policy_attachment( role="cluster", arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-node": _node_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-node-worker": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-cni": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-ecr": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "cluster": _eks_cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE @@ -2942,70 +3683,103 @@ def _to_dict(msg: message.Message) -> dict: "cluster-auth": _cluster_auth( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "nodegroup-system": _nodegroup_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "launch-template-gpu-h200": _launch_template_efa(security_groups=None), "efa-security-group": _efa_security_group(), "efa-security-group-ingress": _efa_security_group_ingress(), "efa-security-group-egress": _efa_security_group_egress(), "nodegroup-gpu-h200": _nodegroup_gpu_efa(), - "addon-vpc-cni": _addon(name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default"), + "addon-vpc-cni": _addon( + name="vpc-cni", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), "addon-kube-proxy": _addon( - name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default" + name="kube-proxy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "addon-coredns": _addon( + name="coredns", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), - "addon-coredns": _addon(name="coredns", cred_kind="ClusterProviderConfig", cred_name="default"), "efs-filesystem": _efs_filesystem( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "efs-security-group": _efs_security_group(cred_kind="ClusterProviderConfig", cred_name="default"), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "efs-security-group-ingress": _efs_security_group_ingress( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "efs-mount-target-0": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-1": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2b-b7832", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-2": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-role-efs-csi": _pod_identity_role( - role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default" + role="efs-csi", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-efs-csi": _role_policy_attachment( role="efs-csi", arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "addon-eks-pod-identity-agent": _addon( - name="eks-pod-identity-agent", cred_kind="ClusterProviderConfig", cred_name="default" + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "pod-identity-efs-csi": _pod_identity_association( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "addon-aws-efs-csi-driver": _addon( - name="aws-efs-csi-driver", cred_kind="ClusterProviderConfig", cred_name="default" + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-policy-cluster-autoscaler": _autoscaler_policy( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "iam-role-cluster-autoscaler": _pod_identity_role( - role="cluster-autoscaler", cred_kind="ClusterProviderConfig", cred_name="default" + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-cluster-autoscaler": _autoscaler_attachment( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "release-cluster-autoscaler": _autoscaler_release(), + "release-cluster-autoscaler": _autoscaler_release(ready=fnv1.READY_UNSPECIFIED), "release-efa-dra-driver": fnv1.Resource( resource=resource.dict_to_struct( { @@ -3043,6 +3817,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 54 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -3073,15 +3861,16 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l4", "state": "Provisioning"}]), resources={ - "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default"), + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED), "subnet-0": _subnet( name="test-cluster-subnet-us-west-2a-952dc", az="us-west-2a", cidr="10.0.0.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-1": _subnet( name="test-cluster-subnet-us-west-2b-2b80f", @@ -3089,6 +3878,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.16.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-2": _subnet( name="test-cluster-subnet-us-west-2c-03273", @@ -3096,6 +3886,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.32.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-0": _private_subnet( name="test-cluster-private-subnet-us-west-2a-6a89f", @@ -3103,6 +3894,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.48.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-1": _private_subnet( name="test-cluster-private-subnet-us-west-2b-b7832", @@ -3110,6 +3902,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.64.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-2": _private_subnet( name="test-cluster-private-subnet-us-west-2c-ef57d", @@ -3117,59 +3910,98 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.80.0/20", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-eip": _nat_eip( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "internet-gateway": _internet_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default"), - "nat-gateway": _nat_gateway(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-table": _route_table(cred_kind="ClusterProviderConfig", cred_name="default"), - "route-default": _route_default(cred_kind="ClusterProviderConfig", cred_name="default"), - "private-route-table": _private_route_table(cred_kind="ClusterProviderConfig", cred_name="default"), "private-route-default": _private_route_default( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "route-table-association-0": _route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-1": _route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-2": _route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-0": _private_route_table_association( - az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2a", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-1": _private_route_table_association( - az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2b", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-2": _private_route_table_association( - az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default" + az="us-west-2c", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-cluster": _cluster_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-cluster-policy": _role_policy_attachment( role="cluster", arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-node": _node_role(cred_kind="ClusterProviderConfig", cred_name="default"), "iam-attach-node-worker": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-cni": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-ecr": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "cluster": _eks_cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE @@ -3177,71 +4009,120 @@ def _to_dict(msg: message.Message) -> dict: "cluster-auth": _cluster_auth( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "nodegroup-system": _nodegroup_system(cred_kind="ClusterProviderConfig", cred_name="default"), - "nodegroup-gpu-l4": _nodegroup_gpu(cred_kind="ClusterProviderConfig", cred_name="default"), - "addon-vpc-cni": _addon(name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default"), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodegroup-gpu-l4": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "addon-vpc-cni": _addon( + name="vpc-cni", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), "addon-kube-proxy": _addon( - name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default" + name="kube-proxy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "addon-coredns": _addon( + name="coredns", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), - "addon-coredns": _addon(name="coredns", cred_kind="ClusterProviderConfig", cred_name="default"), "efs-filesystem": _efs_filesystem( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "efs-security-group": _efs_security_group(cred_kind="ClusterProviderConfig", cred_name="default"), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "efs-security-group-ingress": _efs_security_group_ingress( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "efs-mount-target-0": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-1": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2b-b7832", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "efs-mount-target-2": _efs_mount_target( subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-role-efs-csi": _pod_identity_role( - role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default" + role="efs-csi", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-efs-csi": _role_policy_attachment( role="efs-csi", arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", cred_kind="ClusterProviderConfig", cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "addon-eks-pod-identity-agent": _addon( - name="eks-pod-identity-agent", cred_kind="ClusterProviderConfig", cred_name="default" + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "pod-identity-efs-csi": _pod_identity_association( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "addon-aws-efs-csi-driver": _addon( - name="aws-efs-csi-driver", cred_kind="ClusterProviderConfig", cred_name="default" + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-policy-cluster-autoscaler": _autoscaler_policy( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "iam-role-cluster-autoscaler": _pod_identity_role( - role="cluster-autoscaler", cred_kind="ClusterProviderConfig", cred_name="default" + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-cluster-autoscaler": _autoscaler_attachment( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( - cred_kind="ClusterProviderConfig", cred_name="default" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "release-cluster-autoscaler": _autoscaler_release(), + "release-cluster-autoscaler": _autoscaler_release(ready=fnv1.READY_UNSPECIFIED), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 50 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The ProviderConfigs reach the cluster through its kubeconfig, so they don't @@ -3271,15 +4152,16 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l4", "state": "Provisioning"}]), resources={ - "vpc": _vpc(cred_kind="ProviderConfig", cred_name="my-aws-account"), + "vpc": _vpc(cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED), "subnet-0": _subnet( name="test-cluster-subnet-us-west-2a-952dc", az="us-west-2a", cidr="10.0.0.0/20", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-1": _subnet( name="test-cluster-subnet-us-west-2b-2b80f", @@ -3287,6 +4169,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.16.0/20", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "subnet-2": _subnet( name="test-cluster-subnet-us-west-2c-03273", @@ -3294,6 +4177,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.32.0/20", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-0": _private_subnet( name="test-cluster-private-subnet-us-west-2a-6a89f", @@ -3301,6 +4185,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.48.0/20", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-1": _private_subnet( name="test-cluster-private-subnet-us-west-2b-b7832", @@ -3308,6 +4193,7 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.64.0/20", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "private-subnet-2": _private_subnet( name="test-cluster-private-subnet-us-west-2c-ef57d", @@ -3315,59 +4201,98 @@ def _to_dict(msg: message.Message) -> dict: cidr="10.0.80.0/20", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, + ), + "internet-gateway": _internet_gateway( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED + ), + "nat-eip": _nat_eip( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED + ), + "nat-gateway": _nat_gateway( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED + ), + "route-table": _route_table( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED + ), + "route-default": _route_default( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED + ), + "private-route-table": _private_route_table( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED ), - "internet-gateway": _internet_gateway(cred_kind="ProviderConfig", cred_name="my-aws-account"), - "nat-eip": _nat_eip(cred_kind="ProviderConfig", cred_name="my-aws-account"), - "nat-gateway": _nat_gateway(cred_kind="ProviderConfig", cred_name="my-aws-account"), - "route-table": _route_table(cred_kind="ProviderConfig", cred_name="my-aws-account"), - "route-default": _route_default(cred_kind="ProviderConfig", cred_name="my-aws-account"), - "private-route-table": _private_route_table(cred_kind="ProviderConfig", cred_name="my-aws-account"), "private-route-default": _private_route_default( - cred_kind="ProviderConfig", cred_name="my-aws-account" + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED ), "route-table-association-0": _route_table_association( - az="us-west-2a", cred_kind="ProviderConfig", cred_name="my-aws-account" + az="us-west-2a", + cred_kind="ProviderConfig", + cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-1": _route_table_association( - az="us-west-2b", cred_kind="ProviderConfig", cred_name="my-aws-account" + az="us-west-2b", + cred_kind="ProviderConfig", + cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "route-table-association-2": _route_table_association( - az="us-west-2c", cred_kind="ProviderConfig", cred_name="my-aws-account" + az="us-west-2c", + cred_kind="ProviderConfig", + cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-0": _private_route_table_association( - az="us-west-2a", cred_kind="ProviderConfig", cred_name="my-aws-account" + az="us-west-2a", + cred_kind="ProviderConfig", + cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-1": _private_route_table_association( - az="us-west-2b", cred_kind="ProviderConfig", cred_name="my-aws-account" + az="us-west-2b", + cred_kind="ProviderConfig", + cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "private-route-table-association-2": _private_route_table_association( - az="us-west-2c", cred_kind="ProviderConfig", cred_name="my-aws-account" + az="us-west-2c", + cred_kind="ProviderConfig", + cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-cluster": _cluster_role( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-cluster": _cluster_role(cred_kind="ProviderConfig", cred_name="my-aws-account"), "iam-attach-cluster-policy": _role_policy_attachment( role="cluster", arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-role-node": _node_role( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED ), - "iam-role-node": _node_role(cred_kind="ProviderConfig", cred_name="my-aws-account"), "iam-attach-node-worker": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-cni": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-node-ecr": _role_policy_attachment( role="node", arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "cluster": _eks_cluster( cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED @@ -3375,70 +4300,2060 @@ def _to_dict(msg: message.Message) -> dict: "cluster-auth": _cluster_auth( cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED ), - "nodegroup-system": _nodegroup_system(cred_kind="ProviderConfig", cred_name="my-aws-account"), - "nodegroup-gpu-l4": _nodegroup_gpu(cred_kind="ProviderConfig", cred_name="my-aws-account"), - "addon-vpc-cni": _addon(name="vpc-cni", cred_kind="ProviderConfig", cred_name="my-aws-account"), - "addon-kube-proxy": _addon( - name="kube-proxy", cred_kind="ProviderConfig", cred_name="my-aws-account" - ), - "addon-coredns": _addon(name="coredns", cred_kind="ProviderConfig", cred_name="my-aws-account"), - "efs-filesystem": _efs_filesystem( + "nodegroup-system": _nodegroup_system( cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED ), - "efs-security-group": _efs_security_group(cred_kind="ProviderConfig", cred_name="my-aws-account"), - "efs-security-group-ingress": _efs_security_group_ingress( - cred_kind="ProviderConfig", cred_name="my-aws-account" + "nodegroup-gpu-l4": _nodegroup_gpu( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED ), - "efs-mount-target-0": _efs_mount_target( - subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", + "addon-vpc-cni": _addon( + name="vpc-cni", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), - "efs-mount-target-1": _efs_mount_target( - subnet_name="test-cluster-private-subnet-us-west-2b-b7832", + "addon-kube-proxy": _addon( + name="kube-proxy", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), - "efs-mount-target-2": _efs_mount_target( - subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", + "addon-coredns": _addon( + name="coredns", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, + ), + "efs-filesystem": _efs_filesystem( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED + ), + "efs-security-group": _efs_security_group( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED + ), + "efs-security-group-ingress": _efs_security_group_ingress( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED + ), + "efs-mount-target-0": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", + cred_kind="ProviderConfig", + cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, + ), + "efs-mount-target-1": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2b-b7832", + cred_kind="ProviderConfig", + cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, + ), + "efs-mount-target-2": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", + cred_kind="ProviderConfig", + cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "iam-role-efs-csi": _pod_identity_role( - role="efs-csi", cred_kind="ProviderConfig", cred_name="my-aws-account" + role="efs-csi", + cred_kind="ProviderConfig", + cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, ), "iam-attach-efs-csi": _role_policy_attachment( role="efs-csi", arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", cred_kind="ProviderConfig", cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, + ), + "addon-eks-pod-identity-agent": _addon( + name="eks-pod-identity-agent", + cred_kind="ProviderConfig", + cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, + ), + "pod-identity-efs-csi": _pod_identity_association( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED + ), + "addon-aws-efs-csi-driver": _addon( + name="aws-efs-csi-driver", + cred_kind="ProviderConfig", + cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-policy-cluster-autoscaler": _autoscaler_policy( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED + ), + "iam-role-cluster-autoscaler": _pod_identity_role( + role="cluster-autoscaler", + cred_kind="ProviderConfig", + cred_name="my-aws-account", + ready=fnv1.READY_UNSPECIFIED, + ), + "iam-attach-cluster-autoscaler": _autoscaler_attachment( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED + ), + "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( + cred_kind="ProviderConfig", cred_name="my-aws-account", ready=fnv1.READY_UNSPECIFIED + ), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 49 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + Case( + name="PoolsProvisioning", + reason="With every cluster resource observed Ready and the gpu-l4 node group unready, ClusterReady is Available, NodePoolsReady is Provisioning and gpu-l4 is Provisioning.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l4", + role="GPU", + instanceType="g6.xlarge", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l4"), + zones=[v1alpha1.Zone("us-west-2a"), v1alpha1.Zone("us-west-2b")], + ), + ], + ), + resources={ + "vpc": _observed_resource(ready=True), + "subnet-0": _observed_resource(ready=True), + "subnet-1": _observed_resource(ready=True), + "subnet-2": _observed_resource(ready=True), + "private-subnet-0": _observed_resource(ready=True), + "private-subnet-1": _observed_resource(ready=True), + "private-subnet-2": _observed_resource(ready=True), + "internet-gateway": _observed_resource(ready=True), + "nat-eip": _observed_resource(ready=True), + "nat-gateway": _observed_resource(ready=True), + "route-table": _observed_resource(ready=True), + "route-default": _observed_resource(ready=True), + "private-route-table": _observed_resource(ready=True), + "private-route-default": _observed_resource(ready=True), + "route-table-association-0": _observed_resource(ready=True), + "route-table-association-1": _observed_resource(ready=True), + "route-table-association-2": _observed_resource(ready=True), + "private-route-table-association-0": _observed_resource(ready=True), + "private-route-table-association-1": _observed_resource(ready=True), + "private-route-table-association-2": _observed_resource(ready=True), + "iam-role-cluster": _observed_resource(ready=True), + "iam-attach-cluster-policy": _observed_resource(ready=True), + "iam-role-node": _observed_resource(ready=True), + "iam-attach-node-worker": _observed_resource(ready=True), + "iam-attach-node-cni": _observed_resource(ready=True), + "iam-attach-node-ecr": _observed_resource(ready=True), + "cluster": _observed_eks_cluster(cluster_security_group_id=None, ready_condition=True), + "cluster-auth": _observed_resource(ready=True), + "nodegroup-system": _observed_resource(ready=True), + "addon-vpc-cni": _observed_resource(ready=True), + "addon-kube-proxy": _observed_resource(ready=True), + "addon-coredns": _observed_resource(ready=True), + "efs-filesystem": _observed_efs_filesystem(ready=True), + "efs-security-group": _observed_resource(ready=True), + "efs-security-group-ingress": _observed_resource(ready=True), + "efs-mount-target-0": _observed_resource(ready=True), + "efs-mount-target-1": _observed_resource(ready=True), + "efs-mount-target-2": _observed_resource(ready=True), + "iam-role-efs-csi": _observed_resource(ready=True), + "iam-attach-efs-csi": _observed_resource(ready=True), + "addon-eks-pod-identity-agent": _observed_resource(ready=True), + "pod-identity-efs-csi": _observed_resource(ready=True), + "addon-aws-efs-csi-driver": _observed_resource(ready=True), + "iam-policy-cluster-autoscaler": _observed_resource(ready=True), + "iam-role-cluster-autoscaler": _observed_resource(ready=True), + "iam-attach-cluster-autoscaler": _observed_resource(ready=True), + "pod-identity-cluster-autoscaler": _observed_resource(ready=True), + "release-cluster-autoscaler": _observed_resource(ready=True), + "nodegroup-gpu-l4": _observed_resource(ready=False), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(node_pools=[{"name": "gpu-l4", "state": "Provisioning"}]), + resources={ + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "subnet-0": _subnet( + name="test-cluster-subnet-us-west-2a-952dc", + az="us-west-2a", + cidr="10.0.0.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "subnet-1": _subnet( + name="test-cluster-subnet-us-west-2b-2b80f", + az="us-west-2b", + cidr="10.0.16.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "subnet-2": _subnet( + name="test-cluster-subnet-us-west-2c-03273", + az="us-west-2c", + cidr="10.0.32.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-0": _private_subnet( + name="test-cluster-private-subnet-us-west-2a-6a89f", + az="us-west-2a", + cidr="10.0.48.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-1": _private_subnet( + name="test-cluster-private-subnet-us-west-2b-b7832", + az="us-west-2b", + cidr="10.0.64.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-2": _private_subnet( + name="test-cluster-private-subnet-us-west-2c-ef57d", + az="us-west-2c", + cidr="10.0.80.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-default": _private_route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-0": _route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-1": _route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-2": _route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-0": _private_route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-1": _private_route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-2": _private_route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-cluster-policy": _role_policy_attachment( + role="cluster", + arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-node-worker": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-node-cni": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-node-ecr": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "cluster": _eks_cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cluster-auth": _cluster_auth( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-gpu-l4": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "addon-vpc-cni": _addon( + name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-kube-proxy": _addon( + name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-coredns": _addon( + name="coredns", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-filesystem": _efs_filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-security-group-ingress": _efs_security_group_ingress( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-mount-target-0": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "efs-mount-target-1": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2b-b7832", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "efs-mount-target-2": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-role-efs-csi": _pod_identity_role( + role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-efs-csi": _role_policy_attachment( + role="efs-csi", + arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, ), "addon-eks-pod-identity-agent": _addon( - name="eks-pod-identity-agent", cred_kind="ProviderConfig", cred_name="my-aws-account" + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, ), "pod-identity-efs-csi": _pod_identity_association( - cred_kind="ProviderConfig", cred_name="my-aws-account" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE ), "addon-aws-efs-csi-driver": _addon( - name="aws-efs-csi-driver", cred_kind="ProviderConfig", cred_name="my-aws-account" + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, ), "iam-policy-cluster-autoscaler": _autoscaler_policy( - cred_kind="ProviderConfig", cred_name="my-aws-account" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE ), "iam-role-cluster-autoscaler": _pod_identity_role( - role="cluster-autoscaler", cred_kind="ProviderConfig", cred_name="my-aws-account" + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, ), "iam-attach-cluster-autoscaler": _autoscaler_attachment( - cred_kind="ProviderConfig", cred_name="my-aws-account" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE ), "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( - cred_kind="ProviderConfig", cred_name="my-aws-account" + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE ), + "storage-class-rwx-efs": _storage_class_rwx_efs(), + "release-cluster-autoscaler": _autoscaler_release(ready=fnv1.READY_TRUE), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 node pools ready", + ), + ], + ), + ), + Case( + name="AllReady", + reason="With every composed resource observed Ready, ClusterReady and NodePoolsReady are Available and gpu-l4 is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l4", + role="GPU", + instanceType="g6.xlarge", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l4"), + zones=[v1alpha1.Zone("us-west-2a"), v1alpha1.Zone("us-west-2b")], + ), + ], + ), + resources={ + "vpc": _observed_resource(ready=True), + "subnet-0": _observed_resource(ready=True), + "subnet-1": _observed_resource(ready=True), + "subnet-2": _observed_resource(ready=True), + "private-subnet-0": _observed_resource(ready=True), + "private-subnet-1": _observed_resource(ready=True), + "private-subnet-2": _observed_resource(ready=True), + "internet-gateway": _observed_resource(ready=True), + "nat-eip": _observed_resource(ready=True), + "nat-gateway": _observed_resource(ready=True), + "route-table": _observed_resource(ready=True), + "route-default": _observed_resource(ready=True), + "private-route-table": _observed_resource(ready=True), + "private-route-default": _observed_resource(ready=True), + "route-table-association-0": _observed_resource(ready=True), + "route-table-association-1": _observed_resource(ready=True), + "route-table-association-2": _observed_resource(ready=True), + "private-route-table-association-0": _observed_resource(ready=True), + "private-route-table-association-1": _observed_resource(ready=True), + "private-route-table-association-2": _observed_resource(ready=True), + "iam-role-cluster": _observed_resource(ready=True), + "iam-attach-cluster-policy": _observed_resource(ready=True), + "iam-role-node": _observed_resource(ready=True), + "iam-attach-node-worker": _observed_resource(ready=True), + "iam-attach-node-cni": _observed_resource(ready=True), + "iam-attach-node-ecr": _observed_resource(ready=True), + "cluster": _observed_eks_cluster(cluster_security_group_id=None, ready_condition=True), + "cluster-auth": _observed_resource(ready=True), + "nodegroup-system": _observed_resource(ready=True), + "addon-vpc-cni": _observed_resource(ready=True), + "addon-kube-proxy": _observed_resource(ready=True), + "addon-coredns": _observed_resource(ready=True), + "efs-filesystem": _observed_efs_filesystem(ready=True), + "efs-security-group": _observed_resource(ready=True), + "efs-security-group-ingress": _observed_resource(ready=True), + "efs-mount-target-0": _observed_resource(ready=True), + "efs-mount-target-1": _observed_resource(ready=True), + "efs-mount-target-2": _observed_resource(ready=True), + "iam-role-efs-csi": _observed_resource(ready=True), + "iam-attach-efs-csi": _observed_resource(ready=True), + "addon-eks-pod-identity-agent": _observed_resource(ready=True), + "pod-identity-efs-csi": _observed_resource(ready=True), + "addon-aws-efs-csi-driver": _observed_resource(ready=True), + "iam-policy-cluster-autoscaler": _observed_resource(ready=True), + "iam-role-cluster-autoscaler": _observed_resource(ready=True), + "iam-attach-cluster-autoscaler": _observed_resource(ready=True), + "pod-identity-cluster-autoscaler": _observed_resource(ready=True), + "release-cluster-autoscaler": _observed_resource(ready=True), + "nodegroup-gpu-l4": _observed_resource(ready=True), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(node_pools=[{"name": "gpu-l4", "state": "Ready"}]), + resources={ + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "subnet-0": _subnet( + name="test-cluster-subnet-us-west-2a-952dc", + az="us-west-2a", + cidr="10.0.0.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "subnet-1": _subnet( + name="test-cluster-subnet-us-west-2b-2b80f", + az="us-west-2b", + cidr="10.0.16.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "subnet-2": _subnet( + name="test-cluster-subnet-us-west-2c-03273", + az="us-west-2c", + cidr="10.0.32.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-0": _private_subnet( + name="test-cluster-private-subnet-us-west-2a-6a89f", + az="us-west-2a", + cidr="10.0.48.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-1": _private_subnet( + name="test-cluster-private-subnet-us-west-2b-b7832", + az="us-west-2b", + cidr="10.0.64.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-2": _private_subnet( + name="test-cluster-private-subnet-us-west-2c-ef57d", + az="us-west-2c", + cidr="10.0.80.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-default": _private_route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-0": _route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-1": _route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-2": _route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-0": _private_route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-1": _private_route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-2": _private_route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-cluster-policy": _role_policy_attachment( + role="cluster", + arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-node-worker": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-node-cni": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-node-ecr": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "cluster": _eks_cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cluster-auth": _cluster_auth( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-gpu-l4": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-vpc-cni": _addon( + name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-kube-proxy": _addon( + name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-coredns": _addon( + name="coredns", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-filesystem": _efs_filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-security-group-ingress": _efs_security_group_ingress( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-mount-target-0": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "efs-mount-target-1": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2b-b7832", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "efs-mount-target-2": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-role-efs-csi": _pod_identity_role( + role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-efs-csi": _role_policy_attachment( + role="efs-csi", + arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "addon-eks-pod-identity-agent": _addon( + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "pod-identity-efs-csi": _pod_identity_association( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-aws-efs-csi-driver": _addon( + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-policy-cluster-autoscaler": _autoscaler_policy( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-role-cluster-autoscaler": _pod_identity_role( + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-cluster-autoscaler": _autoscaler_attachment( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "storage-class-rwx-efs": _storage_class_rwx_efs(), + "release-cluster-autoscaler": _autoscaler_release(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + ), + ), + Case( + name="PoolFailed", + reason="With the gpu-l4 node group's LastAsyncOperation False, gpu-l4 is Failed with the provider's error prefixed by the resource name, and NodePoolsReady is Failed naming the pool.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l4", + role="GPU", + instanceType="g6.xlarge", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l4"), + zones=[v1alpha1.Zone("us-west-2a"), v1alpha1.Zone("us-west-2b")], + ), + ], + ), + resources={ + "vpc": _observed_resource(ready=True), + "subnet-0": _observed_resource(ready=True), + "subnet-1": _observed_resource(ready=True), + "subnet-2": _observed_resource(ready=True), + "private-subnet-0": _observed_resource(ready=True), + "private-subnet-1": _observed_resource(ready=True), + "private-subnet-2": _observed_resource(ready=True), + "internet-gateway": _observed_resource(ready=True), + "nat-eip": _observed_resource(ready=True), + "nat-gateway": _observed_resource(ready=True), + "route-table": _observed_resource(ready=True), + "route-default": _observed_resource(ready=True), + "private-route-table": _observed_resource(ready=True), + "private-route-default": _observed_resource(ready=True), + "route-table-association-0": _observed_resource(ready=True), + "route-table-association-1": _observed_resource(ready=True), + "route-table-association-2": _observed_resource(ready=True), + "private-route-table-association-0": _observed_resource(ready=True), + "private-route-table-association-1": _observed_resource(ready=True), + "private-route-table-association-2": _observed_resource(ready=True), + "iam-role-cluster": _observed_resource(ready=True), + "iam-attach-cluster-policy": _observed_resource(ready=True), + "iam-role-node": _observed_resource(ready=True), + "iam-attach-node-worker": _observed_resource(ready=True), + "iam-attach-node-cni": _observed_resource(ready=True), + "iam-attach-node-ecr": _observed_resource(ready=True), + "cluster": _observed_eks_cluster(cluster_security_group_id=None, ready_condition=True), + "cluster-auth": _observed_resource(ready=True), + "nodegroup-system": _observed_resource(ready=True), + "addon-vpc-cni": _observed_resource(ready=True), + "addon-kube-proxy": _observed_resource(ready=True), + "addon-coredns": _observed_resource(ready=True), + "efs-filesystem": _observed_efs_filesystem(ready=True), + "efs-security-group": _observed_resource(ready=True), + "efs-security-group-ingress": _observed_resource(ready=True), + "efs-mount-target-0": _observed_resource(ready=True), + "efs-mount-target-1": _observed_resource(ready=True), + "efs-mount-target-2": _observed_resource(ready=True), + "iam-role-efs-csi": _observed_resource(ready=True), + "iam-attach-efs-csi": _observed_resource(ready=True), + "addon-eks-pod-identity-agent": _observed_resource(ready=True), + "pod-identity-efs-csi": _observed_resource(ready=True), + "addon-aws-efs-csi-driver": _observed_resource(ready=True), + "iam-policy-cluster-autoscaler": _observed_resource(ready=True), + "iam-role-cluster-autoscaler": _observed_resource(ready=True), + "iam-attach-cluster-autoscaler": _observed_resource(ready=True), + "pod-identity-cluster-autoscaler": _observed_resource(ready=True), + "release-cluster-autoscaler": _observed_resource(ready=True), + "nodegroup-gpu-l4": _observed_failed( + message="async create failed: waiting for EKS Node Group (test-cluster:gpu-l4) create: unexpected state CREATE_FAILED, AsgInstanceLaunchFailures: InsufficientInstanceCapacity" + ), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[ + { + "name": "gpu-l4", + "state": "Failed", + "message": "nodegroup-gpu-l4: async create failed: waiting for EKS Node Group (test-cluster:gpu-l4) create: unexpected state CREATE_FAILED, AsgInstanceLaunchFailures: InsufficientInstanceCapacity", + } + ] + ), + resources={ + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "subnet-0": _subnet( + name="test-cluster-subnet-us-west-2a-952dc", + az="us-west-2a", + cidr="10.0.0.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "subnet-1": _subnet( + name="test-cluster-subnet-us-west-2b-2b80f", + az="us-west-2b", + cidr="10.0.16.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "subnet-2": _subnet( + name="test-cluster-subnet-us-west-2c-03273", + az="us-west-2c", + cidr="10.0.32.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-0": _private_subnet( + name="test-cluster-private-subnet-us-west-2a-6a89f", + az="us-west-2a", + cidr="10.0.48.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-1": _private_subnet( + name="test-cluster-private-subnet-us-west-2b-b7832", + az="us-west-2b", + cidr="10.0.64.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-2": _private_subnet( + name="test-cluster-private-subnet-us-west-2c-ef57d", + az="us-west-2c", + cidr="10.0.80.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-default": _private_route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-0": _route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-1": _route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-2": _route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-0": _private_route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-1": _private_route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-2": _private_route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-cluster-policy": _role_policy_attachment( + role="cluster", + arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-node-worker": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-node-cni": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-node-ecr": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "cluster": _eks_cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cluster-auth": _cluster_auth( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-gpu-l4": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "addon-vpc-cni": _addon( + name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-kube-proxy": _addon( + name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-coredns": _addon( + name="coredns", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-filesystem": _efs_filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-security-group-ingress": _efs_security_group_ingress( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-mount-target-0": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "efs-mount-target-1": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2b-b7832", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "efs-mount-target-2": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-role-efs-csi": _pod_identity_role( + role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-efs-csi": _role_policy_attachment( + role="efs-csi", + arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "addon-eks-pod-identity-agent": _addon( + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "pod-identity-efs-csi": _pod_identity_association( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-aws-efs-csi-driver": _addon( + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-policy-cluster-autoscaler": _autoscaler_policy( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-role-cluster-autoscaler": _pod_identity_role( + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-cluster-autoscaler": _autoscaler_attachment( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "storage-class-rwx-efs": _storage_class_rwx_efs(), + "release-cluster-autoscaler": _autoscaler_release(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 1 node pools failed: gpu-l4", + ), + ], + ), + ), + Case( + name="LaunchTemplateFailed", + reason="With a Capacity Block pool's launch template's LastAsyncOperation False, the launch template counts toward gpu-h200, so gpu-h200 is Failed with its error and NodePoolsReady is Failed while ClusterReady is Available.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-h200", + role="GPU", + instanceType="p5en.48xlarge", + nodeCount=2, + minNodeCount=0, + maxNodeCount=2, + diskSizeGb=1024, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h200"), + capacityReservation=v1alpha1.CapacityReservation( + type="CapacityBlock", id="cr-0123456789abcdef0" + ), + zones=[v1alpha1.Zone("us-west-2a")], + ), + ], + ), + resources={ + "vpc": _observed_resource(ready=True), + "subnet-0": _observed_resource(ready=True), + "subnet-1": _observed_resource(ready=True), + "subnet-2": _observed_resource(ready=True), + "private-subnet-0": _observed_resource(ready=True), + "private-subnet-1": _observed_resource(ready=True), + "private-subnet-2": _observed_resource(ready=True), + "internet-gateway": _observed_resource(ready=True), + "nat-eip": _observed_resource(ready=True), + "nat-gateway": _observed_resource(ready=True), + "route-table": _observed_resource(ready=True), + "route-default": _observed_resource(ready=True), + "private-route-table": _observed_resource(ready=True), + "private-route-default": _observed_resource(ready=True), + "route-table-association-0": _observed_resource(ready=True), + "route-table-association-1": _observed_resource(ready=True), + "route-table-association-2": _observed_resource(ready=True), + "private-route-table-association-0": _observed_resource(ready=True), + "private-route-table-association-1": _observed_resource(ready=True), + "private-route-table-association-2": _observed_resource(ready=True), + "iam-role-cluster": _observed_resource(ready=True), + "iam-attach-cluster-policy": _observed_resource(ready=True), + "iam-role-node": _observed_resource(ready=True), + "iam-attach-node-worker": _observed_resource(ready=True), + "iam-attach-node-cni": _observed_resource(ready=True), + "iam-attach-node-ecr": _observed_resource(ready=True), + "cluster": _observed_eks_cluster(cluster_security_group_id=None, ready_condition=True), + "cluster-auth": _observed_resource(ready=True), + "nodegroup-system": _observed_resource(ready=True), + "addon-vpc-cni": _observed_resource(ready=True), + "addon-kube-proxy": _observed_resource(ready=True), + "addon-coredns": _observed_resource(ready=True), + "efs-filesystem": _observed_efs_filesystem(ready=True), + "efs-security-group": _observed_resource(ready=True), + "efs-security-group-ingress": _observed_resource(ready=True), + "efs-mount-target-0": _observed_resource(ready=True), + "efs-mount-target-1": _observed_resource(ready=True), + "efs-mount-target-2": _observed_resource(ready=True), + "iam-role-efs-csi": _observed_resource(ready=True), + "iam-attach-efs-csi": _observed_resource(ready=True), + "addon-eks-pod-identity-agent": _observed_resource(ready=True), + "pod-identity-efs-csi": _observed_resource(ready=True), + "addon-aws-efs-csi-driver": _observed_resource(ready=True), + "iam-policy-cluster-autoscaler": _observed_resource(ready=True), + "iam-role-cluster-autoscaler": _observed_resource(ready=True), + "iam-attach-cluster-autoscaler": _observed_resource(ready=True), + "pod-identity-cluster-autoscaler": _observed_resource(ready=True), + "release-cluster-autoscaler": _observed_resource(ready=True), + "launch-template-gpu-h200": _observed_failed( + message="async create failed: creating EC2 Launch Template (test-cluster-gpu-h200): operation error EC2: CreateLaunchTemplate, InvalidCapacityReservationId.NotFound" + ), + "nodegroup-gpu-h200": _observed_resource(ready=False), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[ + { + "name": "gpu-h200", + "state": "Failed", + "message": "launch-template-gpu-h200: async create failed: creating EC2 Launch Template (test-cluster-gpu-h200): operation error EC2: CreateLaunchTemplate, InvalidCapacityReservationId.NotFound", + } + ] + ), + resources={ + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "subnet-0": _subnet( + name="test-cluster-subnet-us-west-2a-952dc", + az="us-west-2a", + cidr="10.0.0.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "subnet-1": _subnet( + name="test-cluster-subnet-us-west-2b-2b80f", + az="us-west-2b", + cidr="10.0.16.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "subnet-2": _subnet( + name="test-cluster-subnet-us-west-2c-03273", + az="us-west-2c", + cidr="10.0.32.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-0": _private_subnet( + name="test-cluster-private-subnet-us-west-2a-6a89f", + az="us-west-2a", + cidr="10.0.48.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-1": _private_subnet( + name="test-cluster-private-subnet-us-west-2b-b7832", + az="us-west-2b", + cidr="10.0.64.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-2": _private_subnet( + name="test-cluster-private-subnet-us-west-2c-ef57d", + az="us-west-2c", + cidr="10.0.80.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-default": _private_route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-0": _route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-1": _route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-2": _route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-0": _private_route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-1": _private_route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-2": _private_route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-cluster-policy": _role_policy_attachment( + role="cluster", + arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-node-worker": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-node-cni": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-node-ecr": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "cluster": _eks_cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cluster-auth": _cluster_auth( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "launch-template-gpu-h200": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "ec2.aws.m.upbound.io/v1beta1", + "kind": "LaunchTemplate", + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, + "forProvider": { + "region": "us-west-2", + "name": "test-cluster-lt-gpu-h200-83c00", + "instanceType": "p5en.48xlarge", + "blockDeviceMappings": [ + {"deviceName": "/dev/xvda", "ebs": {"volumeSize": 1024}}, + ], + "instanceMarketOptions": {"marketType": "capacity-block"}, + "capacityReservationSpecification": { + "capacityReservationPreference": "capacity-reservations-only", + "capacityReservationTarget": { + "capacityReservationId": "cr-0123456789abcdef0", + }, + }, + }, + }, + } + ), + ), + "nodegroup-gpu-h200": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "eks.aws.m.upbound.io/v1beta1", + "kind": "NodeGroup", + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, + "managementPolicies": ["Observe", "Create", "Update", "Delete"], + "initProvider": {"scalingConfig": {"desiredSize": 2}}, + "forProvider": { + "region": "us-west-2", + "amiType": "AL2023_x86_64_NVIDIA", + "clusterNameSelector": {"matchControllerRef": True}, + "nodeRoleArnSelector": { + "matchControllerRef": True, + "matchLabels": {"modelplane.ai/iam-role": "node"}, + }, + "capacityType": "CAPACITY_BLOCK", + "launchTemplate": { + "name": "test-cluster-lt-gpu-h200-83c00", + "version": "$Latest", + }, + "subnetIdRefs": [{"name": "test-cluster-private-subnet-us-west-2a-6a89f"}], + "scalingConfig": {"minSize": 0, "maxSize": 2}, + "labels": { + "modelplane.ai/gpu": "nvidia-h200", + "modelplane.ai/pool": "gpu-h200", + }, + "taint": [ + { + "key": "nvidia.com/gpu", + "value": "true", + "effect": "NO_SCHEDULE", + }, + ], + }, + }, + } + ), + ), + "addon-vpc-cni": _addon( + name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-kube-proxy": _addon( + name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-coredns": _addon( + name="coredns", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-filesystem": _efs_filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-security-group-ingress": _efs_security_group_ingress( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-mount-target-0": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "efs-mount-target-1": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2b-b7832", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "efs-mount-target-2": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-role-efs-csi": _pod_identity_role( + role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-efs-csi": _role_policy_attachment( + role="efs-csi", + arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "addon-eks-pod-identity-agent": _addon( + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "pod-identity-efs-csi": _pod_identity_association( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-aws-efs-csi-driver": _addon( + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-policy-cluster-autoscaler": _autoscaler_policy( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-role-cluster-autoscaler": _pod_identity_role( + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-cluster-autoscaler": _autoscaler_attachment( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "storage-class-rwx-efs": _storage_class_rwx_efs(), + "release-cluster-autoscaler": _autoscaler_release(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 1 node pools failed: gpu-h200", + ), + ], + ), + ), + Case( + name="ClusterFailed", + reason="With the NAT gateway's LastAsyncOperation False, ClusterReady is Failed with the provider's error prefixed by the resource name, and NodePoolsReady is Pending on it while no pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l4", + role="GPU", + instanceType="g6.xlarge", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l4"), + zones=[v1alpha1.Zone("us-west-2a"), v1alpha1.Zone("us-west-2b")], + ), + ], + ), + resources={ + "vpc": _observed_resource(ready=True), + "subnet-0": _observed_resource(ready=True), + "subnet-1": _observed_resource(ready=True), + "subnet-2": _observed_resource(ready=True), + "private-subnet-0": _observed_resource(ready=True), + "private-subnet-1": _observed_resource(ready=True), + "private-subnet-2": _observed_resource(ready=True), + "internet-gateway": _observed_resource(ready=True), + "nat-eip": _observed_resource(ready=True), + "nat-gateway": _observed_failed( + message="async create failed: creating EC2 NAT Gateway: operation error EC2: CreateNatGateway, NatGatewayLimitExceeded" + ), + "route-table": _observed_resource(ready=True), + "route-default": _observed_resource(ready=True), + "private-route-table": _observed_resource(ready=True), + "private-route-default": _observed_resource(ready=True), + "route-table-association-0": _observed_resource(ready=True), + "route-table-association-1": _observed_resource(ready=True), + "route-table-association-2": _observed_resource(ready=True), + "private-route-table-association-0": _observed_resource(ready=True), + "private-route-table-association-1": _observed_resource(ready=True), + "private-route-table-association-2": _observed_resource(ready=True), + "iam-role-cluster": _observed_resource(ready=True), + "iam-attach-cluster-policy": _observed_resource(ready=True), + "iam-role-node": _observed_resource(ready=True), + "iam-attach-node-worker": _observed_resource(ready=True), + "iam-attach-node-cni": _observed_resource(ready=True), + "iam-attach-node-ecr": _observed_resource(ready=True), + "cluster": _observed_eks_cluster(cluster_security_group_id=None, ready_condition=True), + "cluster-auth": _observed_resource(ready=True), + "nodegroup-system": _observed_resource(ready=True), + "addon-vpc-cni": _observed_resource(ready=True), + "addon-kube-proxy": _observed_resource(ready=True), + "addon-coredns": _observed_resource(ready=True), + "efs-filesystem": _observed_efs_filesystem(ready=True), + "efs-security-group": _observed_resource(ready=True), + "efs-security-group-ingress": _observed_resource(ready=True), + "efs-mount-target-0": _observed_resource(ready=True), + "efs-mount-target-1": _observed_resource(ready=True), + "efs-mount-target-2": _observed_resource(ready=True), + "iam-role-efs-csi": _observed_resource(ready=True), + "iam-attach-efs-csi": _observed_resource(ready=True), + "addon-eks-pod-identity-agent": _observed_resource(ready=True), + "pod-identity-efs-csi": _observed_resource(ready=True), + "addon-aws-efs-csi-driver": _observed_resource(ready=True), + "iam-policy-cluster-autoscaler": _observed_resource(ready=True), + "iam-role-cluster-autoscaler": _observed_resource(ready=True), + "iam-attach-cluster-autoscaler": _observed_resource(ready=True), + "pod-identity-cluster-autoscaler": _observed_resource(ready=True), + "release-cluster-autoscaler": _observed_resource(ready=True), + "nodegroup-gpu-l4": _observed_resource(ready=False), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(node_pools=[{"name": "gpu-l4", "state": "Provisioning"}]), + resources={ + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "subnet-0": _subnet( + name="test-cluster-subnet-us-west-2a-952dc", + az="us-west-2a", + cidr="10.0.0.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "subnet-1": _subnet( + name="test-cluster-subnet-us-west-2b-2b80f", + az="us-west-2b", + cidr="10.0.16.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "subnet-2": _subnet( + name="test-cluster-subnet-us-west-2c-03273", + az="us-west-2c", + cidr="10.0.32.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-0": _private_subnet( + name="test-cluster-private-subnet-us-west-2a-6a89f", + az="us-west-2a", + cidr="10.0.48.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-1": _private_subnet( + name="test-cluster-private-subnet-us-west-2b-b7832", + az="us-west-2b", + cidr="10.0.64.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-2": _private_subnet( + name="test-cluster-private-subnet-us-west-2c-ef57d", + az="us-west-2c", + cidr="10.0.80.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-default": _private_route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-0": _route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-1": _route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-2": _route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-0": _private_route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-1": _private_route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-2": _private_route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-cluster-policy": _role_policy_attachment( + role="cluster", + arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-node-worker": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-node-cni": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-node-ecr": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "cluster": _eks_cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cluster-auth": _cluster_auth( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-gpu-l4": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "addon-vpc-cni": _addon( + name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-kube-proxy": _addon( + name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-coredns": _addon( + name="coredns", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-filesystem": _efs_filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-security-group-ingress": _efs_security_group_ingress( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-mount-target-0": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "efs-mount-target-1": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2b-b7832", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "efs-mount-target-2": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-role-efs-csi": _pod_identity_role( + role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-efs-csi": _role_policy_attachment( + role="efs-csi", + arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "addon-eks-pod-identity-agent": _addon( + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "pod-identity-efs-csi": _pod_identity_association( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-aws-efs-csi-driver": _addon( + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-policy-cluster-autoscaler": _autoscaler_policy( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-role-cluster-autoscaler": _pod_identity_role( + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-cluster-autoscaler": _autoscaler_attachment( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "storage-class-rwx-efs": _storage_class_rwx_efs(), + "release-cluster-autoscaler": _autoscaler_release(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="nat-gateway: async create failed: creating EC2 NAT Gateway: operation error EC2: CreateNatGateway, NatGatewayLimitExceeded", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + Case( + name="ClusterUnreadyPoolReady", + reason="With the coredns addon unready and the gpu-l4 node group Ready, ClusterReady is Provisioning and NodePoolsReady reports gpu-l4's own state, Available, rather than Pending.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l4", + role="GPU", + instanceType="g6.xlarge", + nodeCount=1, + minNodeCount=0, + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l4"), + zones=[v1alpha1.Zone("us-west-2a"), v1alpha1.Zone("us-west-2b")], + ), + ], + ), + resources={ + "vpc": _observed_resource(ready=True), + "subnet-0": _observed_resource(ready=True), + "subnet-1": _observed_resource(ready=True), + "subnet-2": _observed_resource(ready=True), + "private-subnet-0": _observed_resource(ready=True), + "private-subnet-1": _observed_resource(ready=True), + "private-subnet-2": _observed_resource(ready=True), + "internet-gateway": _observed_resource(ready=True), + "nat-eip": _observed_resource(ready=True), + "nat-gateway": _observed_resource(ready=True), + "route-table": _observed_resource(ready=True), + "route-default": _observed_resource(ready=True), + "private-route-table": _observed_resource(ready=True), + "private-route-default": _observed_resource(ready=True), + "route-table-association-0": _observed_resource(ready=True), + "route-table-association-1": _observed_resource(ready=True), + "route-table-association-2": _observed_resource(ready=True), + "private-route-table-association-0": _observed_resource(ready=True), + "private-route-table-association-1": _observed_resource(ready=True), + "private-route-table-association-2": _observed_resource(ready=True), + "iam-role-cluster": _observed_resource(ready=True), + "iam-attach-cluster-policy": _observed_resource(ready=True), + "iam-role-node": _observed_resource(ready=True), + "iam-attach-node-worker": _observed_resource(ready=True), + "iam-attach-node-cni": _observed_resource(ready=True), + "iam-attach-node-ecr": _observed_resource(ready=True), + "cluster": _observed_eks_cluster(cluster_security_group_id=None, ready_condition=True), + "cluster-auth": _observed_resource(ready=True), + "nodegroup-system": _observed_resource(ready=True), + "addon-vpc-cni": _observed_resource(ready=True), + "addon-kube-proxy": _observed_resource(ready=True), + "addon-coredns": _observed_resource(ready=False), + "efs-filesystem": _observed_efs_filesystem(ready=True), + "efs-security-group": _observed_resource(ready=True), + "efs-security-group-ingress": _observed_resource(ready=True), + "efs-mount-target-0": _observed_resource(ready=True), + "efs-mount-target-1": _observed_resource(ready=True), + "efs-mount-target-2": _observed_resource(ready=True), + "iam-role-efs-csi": _observed_resource(ready=True), + "iam-attach-efs-csi": _observed_resource(ready=True), + "addon-eks-pod-identity-agent": _observed_resource(ready=True), + "pod-identity-efs-csi": _observed_resource(ready=True), + "addon-aws-efs-csi-driver": _observed_resource(ready=True), + "iam-policy-cluster-autoscaler": _observed_resource(ready=True), + "iam-role-cluster-autoscaler": _observed_resource(ready=True), + "iam-attach-cluster-autoscaler": _observed_resource(ready=True), + "pod-identity-cluster-autoscaler": _observed_resource(ready=True), + "release-cluster-autoscaler": _observed_resource(ready=True), + "nodegroup-gpu-l4": _observed_resource(ready=True), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(node_pools=[{"name": "gpu-l4", "state": "Ready"}]), + resources={ + "vpc": _vpc(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "subnet-0": _subnet( + name="test-cluster-subnet-us-west-2a-952dc", + az="us-west-2a", + cidr="10.0.0.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "subnet-1": _subnet( + name="test-cluster-subnet-us-west-2b-2b80f", + az="us-west-2b", + cidr="10.0.16.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "subnet-2": _subnet( + name="test-cluster-subnet-us-west-2c-03273", + az="us-west-2c", + cidr="10.0.32.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-0": _private_subnet( + name="test-cluster-private-subnet-us-west-2a-6a89f", + az="us-west-2a", + cidr="10.0.48.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-1": _private_subnet( + name="test-cluster-private-subnet-us-west-2b-b7832", + az="us-west-2b", + cidr="10.0.64.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "private-subnet-2": _private_subnet( + name="test-cluster-private-subnet-us-west-2c-ef57d", + az="us-west-2c", + cidr="10.0.80.0/20", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "internet-gateway": _internet_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nat-eip": _nat_eip(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "nat-gateway": _nat_gateway( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table": _route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-default": _route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table": _private_route_table( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-default": _private_route_default( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-0": _route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-1": _route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "route-table-association-2": _route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-0": _private_route_table_association( + az="us-west-2a", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-1": _private_route_table_association( + az="us-west-2b", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "private-route-table-association-2": _private_route_table_association( + az="us-west-2c", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-role-cluster": _cluster_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-cluster-policy": _role_policy_attachment( + role="cluster", + arn="arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-role-node": _node_role( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-node-worker": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-node-cni": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-node-ecr": _role_policy_attachment( + role="node", + arn="arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "cluster": _eks_cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cluster-auth": _cluster_auth( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-gpu-l4": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-vpc-cni": _addon( + name="vpc-cni", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-kube-proxy": _addon( + name="kube-proxy", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-coredns": _addon( + name="coredns", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_UNSPECIFIED, + ), + "efs-filesystem": _efs_filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-security-group": _efs_security_group( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-security-group-ingress": _efs_security_group_ingress( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "efs-mount-target-0": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2a-6a89f", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "efs-mount-target-1": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2b-b7832", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "efs-mount-target-2": _efs_mount_target( + subnet_name="test-cluster-private-subnet-us-west-2c-ef57d", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-role-efs-csi": _pod_identity_role( + role="efs-csi", cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-attach-efs-csi": _role_policy_attachment( + role="efs-csi", + arn="arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "addon-eks-pod-identity-agent": _addon( + name="eks-pod-identity-agent", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "pod-identity-efs-csi": _pod_identity_association( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "addon-aws-efs-csi-driver": _addon( + name="aws-efs-csi-driver", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-policy-cluster-autoscaler": _autoscaler_policy( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-role-cluster-autoscaler": _pod_identity_role( + role="cluster-autoscaler", + cred_kind="ClusterProviderConfig", + cred_name="default", + ready=fnv1.READY_TRUE, + ), + "iam-attach-cluster-autoscaler": _autoscaler_attachment( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "pod-identity-cluster-autoscaler": _autoscaler_pod_identity( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "storage-class-rwx-efs": _storage_class_rwx_efs(), + "release-cluster-autoscaler": _autoscaler_release(ready=fnv1.READY_TRUE), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="50 of 51 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], ), ), ] diff --git a/functions/compose-gke-cluster/function/fn.py b/functions/compose-gke-cluster/function/fn.py index a5ee3816f..9c624911e 100644 --- a/functions/compose-gke-cluster/function/fn.py +++ b/functions/compose-gke-cluster/function/fn.py @@ -138,6 +138,38 @@ _GKE_IMAGE_TYPE = "COS_CONTAINERD" _GKE_OAUTH_SCOPE = "https://www.googleapis.com/auth/cloud-platform" +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: network, Filestore +# API, service account and IAM, control plane, system node pool, ProviderConfigs +# and the RWX StorageClass. compose-inference-cluster relays both conditions +# onto the InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + + +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + def _name(meta: metav1.ObjectMeta | None) -> str: """The object's name, always set on resources read from the API server.""" @@ -231,6 +263,21 @@ def resolve_project(self) -> str | None: ) return project response.normal(self.rsp, f"Waiting for GCP {cred_kind} {cred_name}") + response.set_conditions( + self.rsp, + resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"Waiting for GCP {cred_kind} {cred_name}", + ), + resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ), + ) # Nothing is composed while waiting for the project, and an XR with # no composed resources would otherwise be trivially ready. self.rsp.desired.composite.ready = fnv1.READY_FALSE @@ -271,8 +318,10 @@ def compose(self) -> None: self.compose_iam_binding() self.compose_provider_configs() self.compose_storage_class() - self.write_status() self.mark_readiness() + node_pools = self.node_pool_statuses() + self.derive_conditions(node_pools) + self.write_status(node_pools) def compose_network(self) -> None: resource.update( @@ -659,8 +708,9 @@ def compose_storage_class(self) -> None: ) self.rsp.desired.resources["storage-class-rwx"].ready = fnv1.READY_TRUE - def write_status(self) -> None: + def write_status(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: status = v1alpha1.Status( + nodePools=node_pools, secrets=[ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -714,6 +764,110 @@ def mark_readiness(self) -> None: self.rsp.desired.resources["provider-config-kubernetes"].ready = fnv1.READY_TRUE self.rsp.desired.resources["provider-config-helm"].ready = fnv1.READY_TRUE + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool.""" + return [f"nodepool-{pool.name}"] + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def _stage_pools(self) -> list[v1alpha1.NodePool]: + """The node pools NodePoolsReady and status.nodePools report. A System + pool runs the cluster's controllers and serving stack, so it's part of + ClusterReady.""" + return [pool for pool in self.xr.spec.nodePools if pool.role != "System"] + + def node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each node pool, from its resources.""" + statuses = [] + for pool in self._stage_pools(): + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: + """Derive ClusterReady from the readiness mark_readiness set, and + NodePoolsReady from node_pools. + + NodePoolsReady waits on ClusterReady only while no pool is Ready, so on + a running cluster the pools report their own state while the cluster + or a new pool provisions. A failed pool is reported Failed even while + the cluster is still provisioning. + """ + in_pools = {r for pool in self._stage_pools() for r in self.pool_resources(pool)} + cluster = sorted(r for r in self.rsp.desired.resources if r not in in_pools) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True" and not ready: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) + def observed_sa_email(self) -> str | None: """Read the service account email from observed state.""" observed_sa = self.req.observed.resources.get("service-account") diff --git a/functions/compose-gke-cluster/tests/test_fn.py b/functions/compose-gke-cluster/tests/test_fn.py index d3508202a..ed7ab5982 100644 --- a/functions/compose-gke-cluster/tests/test_fn.py +++ b/functions/compose-gke-cluster/tests/test_fn.py @@ -55,12 +55,13 @@ def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.N return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr() -> fnv1.Resource: - """The desired XR, publishing its connection Secrets and cache StorageClass.""" +def _desired_xr(*, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR, publishing node_pools' states, its connection Secrets and cache StorageClass.""" return fnv1.Resource( resource=resource.dict_to_struct( { "status": { + "nodePools": node_pools, "secrets": [ { "type": "Kubeconfig", @@ -126,7 +127,7 @@ def _network(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resou ) -def _projectservice_filestore(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _projectservice_filestore(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed ProjectService that enables the Filestore API.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -142,10 +143,11 @@ def _projectservice_filestore(*, cred_kind: str, cred_name: str) -> fnv1.Resourc }, } ), + ready=ready, ) -def _subnet(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _subnet(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed Subnetwork, with secondary ranges for pods and services.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -166,10 +168,11 @@ def _subnet(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _cluster(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _cluster(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed GKE Cluster.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -204,10 +207,11 @@ def _cluster(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _nodepool_system(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _nodepool_system(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed system NodePool the function adds to every cluster.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -233,10 +237,13 @@ def _nodepool_system(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) -def _nodepool_gpu(*, cred_kind: str, cred_name: str, reservation_affinity: dict | None) -> fnv1.Resource: +def _nodepool_gpu( + *, cred_kind: str, cred_name: str, reservation_affinity: dict | None, ready: fnv1.Ready +) -> fnv1.Resource: """The composed NodePool for the XR's gpu-pool, consuming a reservation unless reservation_affinity is None.""" node_config: dict = { "machineType": "a2-highgpu-8g", @@ -286,6 +293,7 @@ def _nodepool_gpu(*, cred_kind: str, cred_name: str, reservation_affinity: dict }, } ), + ready=ready, ) @@ -348,7 +356,7 @@ def _service_account(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fn ) -def _service_account_key(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _service_account_key(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed ServiceAccountKey.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -366,6 +374,7 @@ def _service_account_key(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) @@ -402,6 +411,165 @@ def _provider_config(*, api_version: str) -> fnv1.Resource: ) +def _observed_resource(*, ready: bool) -> fnv1.Resource: + """An observed managed resource, Ready or not.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Creating", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + ), + ) + + +def _observed_network(*, ready: bool) -> fnv1.Resource: + """The observed Network, whose external name is the VPC the Filestore StorageClass is pinned to.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "compute.gcp.m.upbound.io/v1beta1", + "kind": "Network", + "metadata": { + "annotations": {"crossplane.io/external-name": "test-cluster-abc12"}, + }, + "spec": { + "forProvider": { + "autoCreateSubnetworks": False, + }, + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Creating", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + ), + ) + + +def _observed_service_account(*, ready: bool) -> fnv1.Resource: + """The observed ServiceAccount, whose email the IAM binding grants roles to.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "cloudplatform.gcp.m.upbound.io/v1beta1", + "kind": "ServiceAccount", + "spec": {"forProvider": {}}, + "status": { + "atProvider": { + "email": "test-sa@my-gcp-project.iam.gserviceaccount.com", + }, + "conditions": [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Creating", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + ), + ) + + +def _iam_binding(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: + """The composed ProjectIAMMember granting the observed service account container.admin.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "cloudplatform.gcp.m.upbound.io/v1beta1", + "kind": "ProjectIAMMember", + "spec": { + "providerConfigRef": {"kind": cred_kind, "name": cred_name}, + "forProvider": { + "role": "roles/container.admin", + "member": "serviceAccount:test-sa@my-gcp-project.iam.gserviceaccount.com", + "project": "my-gcp-project", + }, + }, + } + ), + ready=ready, + ) + + +def _storage_class_rwx() -> fnv1.Resource: + """The composed Filestore StorageClass, pinned to the observed Network's VPC. + + A StorageClass has no Ready condition, hence SuccessfulCreate, and the + Object omits Delete so it dies with the cluster rather than wedging + teardown on the deleted kubeconfig Secret. + """ + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "managementPolicies": ["Observe", "Create", "Update"], + "providerConfigRef": { + "kind": "ProviderConfig", + "name": "test-cluster-kubeconfig-55b57", + }, + "readiness": {"policy": "SuccessfulCreate"}, + "forProvider": { + "manifest": { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "metadata": {"name": "modelplane-rwx"}, + "provisioner": "filestore.csi.storage.gke.io", + "parameters": { + "tier": "enterprise", + "network": "test-cluster-abc12", + }, + "volumeBindingMode": "Immediate", + "allowVolumeExpansion": True, + }, + }, + }, + } + ), + ready=fnv1.READY_TRUE, + ) + + +def _observed_failed(*, message: str) -> fnv1.Resource: + """An observed managed resource whose provider reports its last operation failed with message.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "conditions": [ + { + "type": "LastAsyncOperation", + "status": "False", + "reason": "AsyncCreateFailure", + "message": message, + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + ), + ) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -434,7 +602,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-pool", "state": "Provisioning"}]), resources={ "network": _network( cred_kind="ClusterProviderConfig", @@ -442,26 +610,50 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_UNSPECIFIED, ), "projectservice-filestore": _projectservice_filestore( - cred_kind="ClusterProviderConfig", - cred_name="default", + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "subnet": _subnet( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-system": _nodepool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default"), - "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default"), - "nodepool-system": _nodepool_system(cred_kind="ClusterProviderConfig", cred_name="default"), "nodepool-gpu-pool": _nodepool_gpu( - cred_kind="ClusterProviderConfig", cred_name="default", reservation_affinity=None + cred_kind="ClusterProviderConfig", + cred_name="default", + reservation_affinity=None, + ready=fnv1.READY_UNSPECIFIED, ), "service-account": _service_account( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED, ), - "service-account-key": _service_account_key(cred_kind="ClusterProviderConfig", cred_name="default"), + "service-account-key": _service_account_key( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 9 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "gcp-provider-config": fnv1.ResourceSelector( @@ -500,7 +692,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-pool", "state": "Provisioning"}]), resources={ "network": _network( cred_kind="ClusterProviderConfig", @@ -508,12 +700,17 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_UNSPECIFIED, ), "projectservice-filestore": _projectservice_filestore( - cred_kind="ClusterProviderConfig", - cred_name="default", + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "subnet": _subnet( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-system": _nodepool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default"), - "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default"), - "nodepool-system": _nodepool_system(cred_kind="ClusterProviderConfig", cred_name="default"), "nodepool-gpu-pool": _nodepool_gpu( cred_kind="ClusterProviderConfig", cred_name="default", @@ -522,18 +719,35 @@ def _to_dict(msg: message.Message) -> dict: "key": "compute.googleapis.com/reservation-name", "values": ["a100-calendar"], }, + ready=fnv1.READY_UNSPECIFIED, ), "service-account": _service_account( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED, ), - "service-account-key": _service_account_key(cred_kind="ClusterProviderConfig", cred_name="default"), + "service-account-key": _service_account_key( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 9 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "gcp-provider-config": fnv1.ResourceSelector( @@ -572,7 +786,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-pool", "state": "Provisioning"}]), resources={ "network": _network( cred_kind="ClusterProviderConfig", @@ -580,12 +794,17 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_UNSPECIFIED, ), "projectservice-filestore": _projectservice_filestore( - cred_kind="ClusterProviderConfig", - cred_name="default", + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "subnet": _subnet( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-system": _nodepool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default"), - "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default"), - "nodepool-system": _nodepool_system(cred_kind="ClusterProviderConfig", cred_name="default"), "nodepool-gpu-pool": _nodepool_gpu( cred_kind="ClusterProviderConfig", cred_name="default", @@ -594,18 +813,35 @@ def _to_dict(msg: message.Message) -> dict: "key": "compute.googleapis.com/reservation-name", "values": ["projects/shared-reservations/reservations/a100-calendar"], }, + ready=fnv1.READY_UNSPECIFIED, ), "service-account": _service_account( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED, ), - "service-account-key": _service_account_key(cred_kind="ClusterProviderConfig", cred_name="default"), + "service-account-key": _service_account_key( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 9 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "gcp-provider-config": fnv1.ResourceSelector( @@ -642,7 +878,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "cpu-pool", "state": "Provisioning"}]), resources={ "network": _network( cred_kind="ClusterProviderConfig", @@ -650,24 +886,45 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_UNSPECIFIED, ), "projectservice-filestore": _projectservice_filestore( - cred_kind="ClusterProviderConfig", - cred_name="default", + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "subnet": _subnet( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-system": _nodepool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default"), - "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default"), - "nodepool-system": _nodepool_system(cred_kind="ClusterProviderConfig", cred_name="default"), "nodepool-cpu-pool": _nodepool_cpu(cred_kind="ClusterProviderConfig", cred_name="default"), "service-account": _service_account( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED, ), - "service-account-key": _service_account_key(cred_kind="ClusterProviderConfig", cred_name="default"), + "service-account-key": _service_account_key( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 9 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "gcp-provider-config": fnv1.ResourceSelector( @@ -683,7 +940,7 @@ def _to_dict(msg: message.Message) -> dict: # the function marks it not ready. Case( name="ProviderConfigMissing", - reason="With the GCP provider config missing and no cluster observed to take the project from, a GKECluster composes nothing and isn't ready.", + reason="With the GCP provider config missing and no cluster observed to take the project from, a GKECluster composes nothing, isn't ready, and reports ClusterReady waiting for the provider config.", req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_xr( @@ -710,6 +967,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="Waiting for GCP ClusterProviderConfig default", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "gcp-provider-config": fnv1.ResourceSelector( @@ -738,57 +1009,8 @@ def _to_dict(msg: message.Message) -> dict: ], ), resources={ - "service-account": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "cloudplatform.gcp.m.upbound.io/v1beta1", - "kind": "ServiceAccount", - "spec": {"forProvider": {}}, - "status": { - "atProvider": { - "email": "test-sa@my-gcp-project.iam.gserviceaccount.com", - }, - "conditions": [ - { - "type": "Ready", - "status": "True", - "reason": "Available", - "lastTransitionTime": "2024-01-01T00:00:00Z", - }, - ], - }, - } - ), - ), - "network": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "compute.gcp.m.upbound.io/v1beta1", - "kind": "Network", - "metadata": { - # The external-name annotation carries the - # provider-generated VPC name, which the - # function pins the Filestore StorageClass to. - "annotations": {"crossplane.io/external-name": "test-cluster-abc12"}, - }, - "spec": { - "forProvider": { - "autoCreateSubnetworks": False, - }, - }, - "status": { - "conditions": [ - { - "type": "Ready", - "status": "True", - "reason": "Available", - "lastTransitionTime": "2024-01-01T00:00:00Z", - }, - ], - }, - } - ), - ), + "service-account": _observed_service_account(ready=True), + "network": _observed_network(ready=True), }, ), required_resources={ @@ -800,7 +1022,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-pool", "state": "Provisioning"}]), resources={ "network": _network( cred_kind="ClusterProviderConfig", @@ -808,79 +1030,54 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_TRUE, ), "projectservice-filestore": _projectservice_filestore( - cred_kind="ClusterProviderConfig", - cred_name="default", + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - # Composed once the observed Network's external name gives - # the VPC to pin Filestore to. A StorageClass has no Ready - # condition, hence SuccessfulCreate, and the Object omits - # Delete so it dies with the cluster rather than wedging - # teardown on the deleted kubeconfig Secret. - "storage-class-rwx": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", - "kind": "Object", - "metadata": {"namespace": "modelplane-system"}, - "spec": { - "managementPolicies": ["Observe", "Create", "Update"], - "providerConfigRef": { - "kind": "ProviderConfig", - "name": "test-cluster-kubeconfig-55b57", - }, - "readiness": {"policy": "SuccessfulCreate"}, - "forProvider": { - "manifest": { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "metadata": {"name": "modelplane-rwx"}, - "provisioner": "filestore.csi.storage.gke.io", - "parameters": { - "tier": "enterprise", - "network": "test-cluster-abc12", - }, - "volumeBindingMode": "Immediate", - "allowVolumeExpansion": True, - }, - }, - }, - } - ), - ready=fnv1.READY_TRUE, + "storage-class-rwx": _storage_class_rwx(), + "subnet": _subnet( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-system": _nodepool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default"), - "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default"), - "nodepool-system": _nodepool_system(cred_kind="ClusterProviderConfig", cred_name="default"), "nodepool-gpu-pool": _nodepool_gpu( - cred_kind="ClusterProviderConfig", cred_name="default", reservation_affinity=None + cred_kind="ClusterProviderConfig", + cred_name="default", + reservation_affinity=None, + ready=fnv1.READY_UNSPECIFIED, ), "service-account": _service_account( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE, ), - "service-account-key": _service_account_key(cred_kind="ClusterProviderConfig", cred_name="default"), - "iam-binding": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "cloudplatform.gcp.m.upbound.io/v1beta1", - "kind": "ProjectIAMMember", - "spec": { - "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, - "forProvider": { - "role": "roles/container.admin", - "member": "serviceAccount:test-sa@my-gcp-project.iam.gserviceaccount.com", - "project": "my-gcp-project", - }, - }, - } - ), + "service-account-key": _service_account_key( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "iam-binding": _iam_binding( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="5 of 11 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "gcp-provider-config": fnv1.ResourceSelector( @@ -911,20 +1108,7 @@ def _to_dict(msg: message.Message) -> dict: ], ), resources={ - "service-account": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "cloudplatform.gcp.m.upbound.io/v1beta1", - "kind": "ServiceAccount", - "spec": {"forProvider": {}}, - "status": { - "atProvider": { - "email": "test-sa@my-gcp-project.iam.gserviceaccount.com", - }, - }, - } - ), - ), + "service-account": _observed_service_account(ready=False), }, ), required_resources={ @@ -942,7 +1126,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-pool", "state": "Provisioning"}]), resources={ "network": _network( cred_kind="ProviderConfig", @@ -950,42 +1134,53 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_UNSPECIFIED, ), "projectservice-filestore": _projectservice_filestore( - cred_kind="ProviderConfig", - cred_name="my-gcp-account", + cred_kind="ProviderConfig", cred_name="my-gcp-account", ready=fnv1.READY_UNSPECIFIED + ), + "subnet": _subnet( + cred_kind="ProviderConfig", cred_name="my-gcp-account", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ProviderConfig", cred_name="my-gcp-account", ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-system": _nodepool_system( + cred_kind="ProviderConfig", cred_name="my-gcp-account", ready=fnv1.READY_UNSPECIFIED ), - "subnet": _subnet(cred_kind="ProviderConfig", cred_name="my-gcp-account"), - "cluster": _cluster(cred_kind="ProviderConfig", cred_name="my-gcp-account"), - "nodepool-system": _nodepool_system(cred_kind="ProviderConfig", cred_name="my-gcp-account"), "nodepool-gpu-pool": _nodepool_gpu( - cred_kind="ProviderConfig", cred_name="my-gcp-account", reservation_affinity=None + cred_kind="ProviderConfig", + cred_name="my-gcp-account", + reservation_affinity=None, + ready=fnv1.READY_UNSPECIFIED, ), "service-account": _service_account( cred_kind="ProviderConfig", cred_name="my-gcp-account", ready=fnv1.READY_UNSPECIFIED, ), - "service-account-key": _service_account_key(cred_kind="ProviderConfig", cred_name="my-gcp-account"), - "iam-binding": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "cloudplatform.gcp.m.upbound.io/v1beta1", - "kind": "ProjectIAMMember", - "spec": { - "providerConfigRef": {"kind": "ProviderConfig", "name": "my-gcp-account"}, - "forProvider": { - "role": "roles/container.admin", - "member": "serviceAccount:test-sa@my-gcp-project.iam.gserviceaccount.com", - "project": "my-gcp-project", - }, - }, - } - ), + "service-account-key": _service_account_key( + cred_kind="ProviderConfig", cred_name="my-gcp-account", ready=fnv1.READY_UNSPECIFIED + ), + "iam-binding": _iam_binding( + cred_kind="ProviderConfig", cred_name="my-gcp-account", ready=fnv1.READY_UNSPECIFIED ), "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 10 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], # A ProviderConfig, unlike a ClusterProviderConfig, is namespaced, # so the function requires it from the XR's namespace. Crossplane # wouldn't return the request's crossplane-system one for this @@ -1002,6 +1197,487 @@ def _to_dict(msg: message.Message) -> dict: ), ), ), + Case( + name="PoolsProvisioning", + reason="With every cluster resource observed Ready and the gpu-pool NodePool unready, ClusterReady is Available, NodePoolsReady is Provisioning and gpu-pool is Provisioning.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-pool", + role="GPU", + machineType="a2-highgpu-8g", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-tesla-a100", acceleratorCount=8), + ), + ], + ), + resources={ + "network": _observed_network(ready=True), + "projectservice-filestore": _observed_resource(ready=True), + "subnet": _observed_resource(ready=True), + "cluster": _observed_resource(ready=True), + "nodepool-system": _observed_resource(ready=True), + "service-account-key": _observed_resource(ready=True), + "iam-binding": _observed_resource(ready=True), + "service-account": _observed_service_account(ready=True), + "nodepool-gpu-pool": _observed_resource(ready=False), + }, + ), + required_resources={ + "gcp-provider-config": fnv1.Resources( + items=[_gcp_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(node_pools=[{"name": "gpu-pool", "state": "Provisioning"}]), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "projectservice-filestore": _projectservice_filestore( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "storage-class-rwx": _storage_class_rwx(), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "nodepool-system": _nodepool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodepool-gpu-pool": _nodepool_gpu( + cred_kind="ClusterProviderConfig", + cred_name="default", + reservation_affinity=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "service-account": _service_account( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "service-account-key": _service_account_key( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-binding": _iam_binding( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 node pools ready", + ), + ], + requirements=fnv1.Requirements( + resources={ + "gcp-provider-config": fnv1.ResourceSelector( + api_version="gcp.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), + Case( + name="AllReady", + reason="With every composed resource observed Ready, ClusterReady and NodePoolsReady are Available and gpu-pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-pool", + role="GPU", + machineType="a2-highgpu-8g", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-tesla-a100", acceleratorCount=8), + ), + ], + ), + resources={ + "network": _observed_network(ready=True), + "projectservice-filestore": _observed_resource(ready=True), + "subnet": _observed_resource(ready=True), + "cluster": _observed_resource(ready=True), + "nodepool-system": _observed_resource(ready=True), + "service-account-key": _observed_resource(ready=True), + "iam-binding": _observed_resource(ready=True), + "service-account": _observed_service_account(ready=True), + "nodepool-gpu-pool": _observed_resource(ready=True), + }, + ), + required_resources={ + "gcp-provider-config": fnv1.Resources( + items=[_gcp_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(node_pools=[{"name": "gpu-pool", "state": "Ready"}]), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "projectservice-filestore": _projectservice_filestore( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "storage-class-rwx": _storage_class_rwx(), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "nodepool-system": _nodepool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodepool-gpu-pool": _nodepool_gpu( + cred_kind="ClusterProviderConfig", + cred_name="default", + reservation_affinity=None, + ready=fnv1.READY_TRUE, + ), + "service-account": _service_account( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "service-account-key": _service_account_key( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-binding": _iam_binding( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + requirements=fnv1.Requirements( + resources={ + "gcp-provider-config": fnv1.ResourceSelector( + api_version="gcp.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), + Case( + name="PoolFailed", + reason="With the gpu-pool NodePool's LastAsyncOperation False, gpu-pool is Failed with the provider's error prefixed by the resource name, and NodePoolsReady is Failed naming the pool.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-pool", + role="GPU", + machineType="a2-highgpu-8g", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-tesla-a100", acceleratorCount=8), + ), + ], + ), + resources={ + "network": _observed_network(ready=True), + "projectservice-filestore": _observed_resource(ready=True), + "subnet": _observed_resource(ready=True), + "cluster": _observed_resource(ready=True), + "nodepool-system": _observed_resource(ready=True), + "service-account-key": _observed_resource(ready=True), + "iam-binding": _observed_resource(ready=True), + "service-account": _observed_service_account(ready=True), + "nodepool-gpu-pool": _observed_failed( + message="async create failed: googleapi: Error 403: Quota 'NVIDIA_A100_GPUS' exceeded. Limit: 0.0 in region us-central1." + ), + }, + ), + required_resources={ + "gcp-provider-config": fnv1.Resources( + items=[_gcp_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[ + { + "name": "gpu-pool", + "state": "Failed", + "message": "nodepool-gpu-pool: async create failed: googleapi: Error 403: Quota 'NVIDIA_A100_GPUS' exceeded. Limit: 0.0 in region us-central1.", + } + ] + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "projectservice-filestore": _projectservice_filestore( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "storage-class-rwx": _storage_class_rwx(), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "nodepool-system": _nodepool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodepool-gpu-pool": _nodepool_gpu( + cred_kind="ClusterProviderConfig", + cred_name="default", + reservation_affinity=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "service-account": _service_account( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "service-account-key": _service_account_key( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-binding": _iam_binding( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 1 node pools failed: gpu-pool", + ), + ], + requirements=fnv1.Requirements( + resources={ + "gcp-provider-config": fnv1.ResourceSelector( + api_version="gcp.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), + Case( + name="ClusterFailed", + reason="With the GKE Cluster's LastAsyncOperation False, ClusterReady is Failed with the provider's error prefixed by the resource name, and NodePoolsReady is Pending on it while no pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-pool", + role="GPU", + machineType="a2-highgpu-8g", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-tesla-a100", acceleratorCount=8), + ), + ], + ), + resources={ + "network": _observed_network(ready=True), + "projectservice-filestore": _observed_resource(ready=True), + "subnet": _observed_resource(ready=True), + "cluster": _observed_failed( + message="async create failed: googleapi: Error 400: Master version unsupported." + ), + "nodepool-system": _observed_resource(ready=True), + "service-account-key": _observed_resource(ready=True), + "iam-binding": _observed_resource(ready=True), + "service-account": _observed_service_account(ready=True), + "nodepool-gpu-pool": _observed_resource(ready=False), + }, + ), + required_resources={ + "gcp-provider-config": fnv1.Resources( + items=[_gcp_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(node_pools=[{"name": "gpu-pool", "state": "Provisioning"}]), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "projectservice-filestore": _projectservice_filestore( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "storage-class-rwx": _storage_class_rwx(), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-system": _nodepool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodepool-gpu-pool": _nodepool_gpu( + cred_kind="ClusterProviderConfig", + cred_name="default", + reservation_affinity=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "service-account": _service_account( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "service-account-key": _service_account_key( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-binding": _iam_binding( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="cluster: async create failed: googleapi: Error 400: Master version unsupported.", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + requirements=fnv1.Requirements( + resources={ + "gcp-provider-config": fnv1.ResourceSelector( + api_version="gcp.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), + Case( + name="ClusterUnreadyPoolReady", + reason="With the GKE Cluster unready and the gpu-pool NodePool Ready, ClusterReady is Provisioning and NodePoolsReady reports gpu-pool's own state, Available, rather than Pending.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-pool", + role="GPU", + machineType="a2-highgpu-8g", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-tesla-a100", acceleratorCount=8), + ), + ], + ), + resources={ + "network": _observed_network(ready=True), + "projectservice-filestore": _observed_resource(ready=True), + "subnet": _observed_resource(ready=True), + "cluster": _observed_resource(ready=False), + "nodepool-system": _observed_resource(ready=True), + "service-account-key": _observed_resource(ready=True), + "iam-binding": _observed_resource(ready=True), + "service-account": _observed_service_account(ready=True), + "nodepool-gpu-pool": _observed_resource(ready=True), + }, + ), + required_resources={ + "gcp-provider-config": fnv1.Resources( + items=[_gcp_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr(node_pools=[{"name": "gpu-pool", "state": "Ready"}]), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "projectservice-filestore": _projectservice_filestore( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "storage-class-rwx": _storage_class_rwx(), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodepool-system": _nodepool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodepool-gpu-pool": _nodepool_gpu( + cred_kind="ClusterProviderConfig", + cred_name="default", + reservation_affinity=None, + ready=fnv1.READY_TRUE, + ), + "service-account": _service_account( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "service-account-key": _service_account_key( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "iam-binding": _iam_binding( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="10 of 11 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + requirements=fnv1.Requirements( + resources={ + "gcp-provider-config": fnv1.ResourceSelector( + api_version="gcp.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), ] diff --git a/functions/compose-inference-cluster/function/fn.py b/functions/compose-inference-cluster/function/fn.py index f8c6812f6..0d2c9201c 100644 --- a/functions/compose-inference-cluster/function/fn.py +++ b/functions/compose-inference-cluster/function/fn.py @@ -78,20 +78,40 @@ CLUSTER_SOURCE_UPCLOUD: Final = "UpCloud" CLUSTER_SOURCE_EXISTING: Final = "Existing" -# Condition types and reasons for the InferenceCluster XR. +# Composed resource key of the cloud cluster XR for each source Modelplane provisions. +_CLUSTER_RESOURCE_KEYS: Final[dict[str, str]] = { + CLUSTER_SOURCE_GKE: "gke-cluster", + CLUSTER_SOURCE_EKS: "eks-cluster", + CLUSTER_SOURCE_AKS: "aks-cluster", + CLUSTER_SOURCE_NEBIUS: "nebius-cluster", + CLUSTER_SOURCE_VULTR: "vultr-cluster", + CLUSTER_SOURCE_CIVO: "civo-cluster", + CLUSTER_SOURCE_COREWEAVE: "coreweave-cluster", + CLUSTER_SOURCE_UPCLOUD: "upcloud-cluster", +} + +# Conditions reporting which provisioning stage the cluster is in: Accepted, +# then ClusterReady and NodePoolsReady, relayed from the cloud cluster XR, then +# ServingStackReady. A stage is Pending on an earlier one only until it's +# composed. After that it reports its own state, so a running cluster's +# ServingStackReady stays True while a new node pool provisions. +CONDITION_TYPE_ACCEPTED = "Accepted" CONDITION_TYPE_CLUSTER_READY = "ClusterReady" -CONDITION_TYPE_BACKEND_READY = "BackendReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" +CONDITION_TYPE_SERVING_STACK_READY = "ServingStackReady" -CONDITION_REASON_CLUSTER_RUNNING = "ClusterRunning" +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" CONDITION_REASON_PROVISIONING = "Provisioning" -CONDITION_REASON_WAITING_FOR_CLUSTER = "WaitingForCluster" +CONDITION_REASON_EXISTING = "Existing" CONDITION_REASON_WAITING_FOR_CLASSES = "WaitingForClasses" CONDITION_REASON_WAITING_FOR_RESERVATIONS = "WaitingForReservations" -CONDITION_REASON_BACKEND_HEALTHY = "BackendHealthy" -CONDITION_REASON_INSTALLING = "Installing" CONDITION_REASON_INVALID_NODE_POOL = "InvalidNodePool" CONDITION_REASON_UNSUPPORTED_DEVICES = "UnsupportedDevices" +# status.nodePools[].state of a pool the cloud cluster XR doesn't report. +_POOL_STATE_PENDING = "Pending" + # The DRA drivers each cloud supports. A hand-kept mirror of # supported_drivers() in compose-serving-stack's stacks package - # functions are self-contained by design - which the @@ -118,6 +138,9 @@ # Composed resource key for the backend XR. BACKEND_RESOURCE_KEY = "serving-stack" +# Composed resource key for the ClusterProviderConfig that reaches the cluster. +CLUSTER_PROVIDER_CONFIG_RESOURCE_KEY = "cluster-provider-config-kubernetes" + # Composed resource key for the ClusterUsage that blocks the InferenceCluster's # deletion while anything composes objects onto it. # @@ -274,6 +297,26 @@ def _namespace(meta: metav1.ObjectMeta | None) -> str: return meta.namespace +def _pending(typ: str, *, after: str) -> resource.Condition: + """Condition typ False and Pending, waiting on the earlier stage after.""" + return resource.Condition(typ=typ, status="False", reason=CONDITION_REASON_PENDING, message=f"Waiting for {after}") + + +def _relay(observed: fnv1.Resource | None, typ: str, *, after: resource.Condition) -> resource.Condition: + """The cloud cluster XR's condition of type typ, including an Unknown one + Crossplane sets when the cloud cluster XR's function fails. + + A condition the cloud cluster XR doesn't report yet is Pending while the + stage before it, after, isn't True, and Provisioning once it is. + """ + c = resource.get_condition(observed, typ) + if c.reason is not None: + return resource.Condition(typ=typ, status=c.status, reason=c.reason, message=c.message) + if after.status != "True": + return _pending(typ, after=after.typ) + return resource.Condition(typ=typ, status="False", reason=CONDITION_REASON_PROVISIONING) + + def _gateway_hostname(cluster_name: str) -> str: """The internal name an InferenceGateway addresses this cluster's gateway by. @@ -840,17 +883,23 @@ def check_devices_supported(self, source: Cloud) -> bool: return False def fail(self, reason: str, msg: str) -> None: - """Stop composing the cluster, reporting why on its ClusterReady condition. + """Stop composing the cluster, reporting why on its Accepted condition. The result is fatal, so Crossplane applies none of this reconcile's desired state. Its caller returns before composing the cluster XR, and Crossplane would otherwise delete an existing cluster XR that's missing from desired state, for example because someone deleted an InferenceClass or edited a node pool. + + Stages that are already composed keep reporting their own state, so a + running cluster's ServingStackReady stays True while Accepted is False. """ + accepted = resource.Condition(typ=CONDITION_TYPE_ACCEPTED, status="False", reason=reason, message=msg) + source = self.xr.spec.cluster.source if self.xr.spec.cluster else CLUSTER_SOURCE_EXISTING response.set_conditions( self.rsp, - resource.Condition(typ=CONDITION_TYPE_CLUSTER_READY, status="False", reason=reason, message=msg), + accepted, + *self.stage_conditions(accepted, cluster=_CLUSTER_RESOURCE_KEYS.get(source)), ) response.fatal(self.rsp, msg) @@ -873,7 +922,7 @@ def compose_gke(self, gke: v1alpha1.Gke | None) -> None: sa_key = self.observed_gke_secret(_IDENTITY_TYPE_GCP) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if gke_ready and kubeconfig_secret: + if kubeconfig_secret and self.keep_cluster_provider_config(cluster_ready=gke_ready): self.compose_cluster_provider_config( kubeconfig_secret.name, kubeconfig_secret.key, @@ -892,8 +941,8 @@ def compose_gke(self, gke: v1alpha1.Gke | None) -> None: if not backend_exists: response.normal(self.rsp, "GKE cluster ready, composing backend") - self.write_status(self.schedulable_pools()) - self.derive_conditions(cluster_ready=gke_ready) + self.write_status(self.schedulable_pools(), self.relayed_node_pools("gke-cluster")) + self.derive_conditions(cluster="gke-cluster") def compose_eks(self, eks: v1alpha1.Eks | None) -> None: """Compose an InferenceCluster backed by a Modelplane-provisioned @@ -920,7 +969,7 @@ def compose_eks(self, eks: v1alpha1.Eks | None) -> None: kubeconfig = self.observed_eks_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if eks_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=eks_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_eks_backend_secrets(eks_ready=eks_ready, backend_exists=backend_exists) @@ -934,8 +983,8 @@ def compose_eks(self, eks: v1alpha1.Eks | None) -> None: if not backend_exists: response.normal(self.rsp, "EKS cluster ready, composing backend") - self.write_status(self.schedulable_pools()) - self.derive_conditions(cluster_ready=eks_ready) + self.write_status(self.schedulable_pools(), self.relayed_node_pools("eks-cluster")) + self.derive_conditions(cluster="eks-cluster") def compose_aks(self, aks: v1alpha1.Aks | None) -> None: """Compose an InferenceCluster backed by a Modelplane-provisioned @@ -960,7 +1009,7 @@ def compose_aks(self, aks: v1alpha1.Aks | None) -> None: kubeconfig = self.observed_aks_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if aks_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=aks_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_aks_backend_secrets(aks_ready=aks_ready, backend_exists=backend_exists) @@ -974,8 +1023,8 @@ def compose_aks(self, aks: v1alpha1.Aks | None) -> None: if not backend_exists: response.normal(self.rsp, "AKS cluster ready, composing backend") - self.write_status(self.schedulable_pools()) - self.derive_conditions(cluster_ready=aks_ready) + self.write_status(self.schedulable_pools(), self.relayed_node_pools("aks-cluster")) + self.derive_conditions(cluster="aks-cluster") def compose_nebius(self, nebius: v1alpha1.Nebius | None) -> None: """Compose an InferenceCluster backed by a Modelplane-provisioned @@ -1005,7 +1054,7 @@ def compose_nebius(self, nebius: v1alpha1.Nebius | None) -> None: credentials = self.observed_nebius_secret(_IDENTITY_TYPE_NEBIUS) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if nebius_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=nebius_ready): self.compose_cluster_provider_config( kubeconfig.name, kubeconfig.key, @@ -1024,8 +1073,8 @@ def compose_nebius(self, nebius: v1alpha1.Nebius | None) -> None: if not backend_exists: response.normal(self.rsp, "Nebius cluster ready, composing backend") - self.write_status(self.schedulable_pools()) - self.derive_conditions(cluster_ready=nebius_ready) + self.write_status(self.schedulable_pools(), self.relayed_node_pools("nebius-cluster")) + self.derive_conditions(cluster="nebius-cluster") def compose_vultr(self, vultr: v1alpha1.Vultr | None) -> None: """Compose an InferenceCluster backed by a Modelplane-provisioned @@ -1050,7 +1099,7 @@ def compose_vultr(self, vultr: v1alpha1.Vultr | None) -> None: kubeconfig = self.observed_vultr_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if vultr_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=vultr_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_vultr_backend_secrets(vultr_ready=vultr_ready, backend_exists=backend_exists) @@ -1064,8 +1113,8 @@ def compose_vultr(self, vultr: v1alpha1.Vultr | None) -> None: if not backend_exists: response.normal(self.rsp, "Vultr cluster ready, composing backend") - self.write_status(self.schedulable_pools()) - self.derive_conditions(cluster_ready=vultr_ready) + self.write_status(self.schedulable_pools(), self.relayed_node_pools("vultr-cluster")) + self.derive_conditions(cluster="vultr-cluster") def compose_civo(self, civo: v1alpha1.Civo | None) -> None: """Compose an InferenceCluster backed by a Modelplane-provisioned @@ -1090,7 +1139,7 @@ def compose_civo(self, civo: v1alpha1.Civo | None) -> None: kubeconfig = self.observed_civo_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if civo_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=civo_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_civo_backend_secrets(civo_ready=civo_ready, backend_exists=backend_exists) @@ -1110,8 +1159,8 @@ def compose_civo(self, civo: v1alpha1.Civo | None) -> None: if not backend_exists: response.normal(self.rsp, "Civo cluster ready, composing backend") - self.write_status(self.schedulable_pools()) - self.derive_conditions(cluster_ready=civo_ready) + self.write_status(self.schedulable_pools(), self.relayed_node_pools("civo-cluster")) + self.derive_conditions(cluster="civo-cluster") def compose_coreweave(self, coreweave: v1alpha1.Coreweave | None) -> None: """Compose an InferenceCluster backed by a Modelplane-provisioned @@ -1135,7 +1184,7 @@ def compose_coreweave(self, coreweave: v1alpha1.Coreweave | None) -> None: kubeconfig = self.observed_coreweave_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if coreweave_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=coreweave_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_coreweave_backend_secrets( @@ -1151,8 +1200,8 @@ def compose_coreweave(self, coreweave: v1alpha1.Coreweave | None) -> None: if not backend_exists: response.normal(self.rsp, "CoreWeave cluster ready, composing backend") - self.write_status(self.schedulable_pools()) - self.derive_conditions(cluster_ready=coreweave_ready) + self.write_status(self.schedulable_pools(), self.relayed_node_pools("coreweave-cluster")) + self.derive_conditions(cluster="coreweave-cluster") def compose_upcloud(self, upcloud: v1alpha1.Upcloud | None) -> None: """Compose an InferenceCluster backed by a Modelplane-provisioned @@ -1179,7 +1228,7 @@ def compose_upcloud(self, upcloud: v1alpha1.Upcloud | None) -> None: kubeconfig = self.observed_upcloud_secret(_SECRET_TYPE_KUBECONFIG) backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources - if upcloud_ready and kubeconfig: + if kubeconfig and self.keep_cluster_provider_config(cluster_ready=upcloud_ready): self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) backend_secrets = self.resolve_upcloud_backend_secrets( @@ -1195,8 +1244,8 @@ def compose_upcloud(self, upcloud: v1alpha1.Upcloud | None) -> None: if not backend_exists: response.normal(self.rsp, "UpCloud cluster ready, composing backend") - self.write_status(self.schedulable_pools()) - self.derive_conditions(cluster_ready=upcloud_ready) + self.write_status(self.schedulable_pools(), self.relayed_node_pools("upcloud-cluster")) + self.derive_conditions(cluster="upcloud-cluster") def compose_existing(self, existing: v1alpha1.Existing | None) -> None: """Compose an InferenceCluster backed by a user-supplied cluster. @@ -1224,8 +1273,8 @@ def compose_existing(self, existing: v1alpha1.Existing | None) -> None: ) self.compose_serving_stack(backend_secrets, CLUSTER_SOURCE_EXISTING) - self.write_status(self.schedulable_pools()) - self.derive_conditions(cluster_ready=True) + self.write_status(self.schedulable_pools(), None) + self.derive_conditions(cluster=None) def compose_serving_stack( self, @@ -1275,6 +1324,18 @@ def compose_serving_stack( ), ) + def keep_cluster_provider_config(self, *, cluster_ready: bool) -> bool: + """Whether to compose the ClusterProviderConfig reaching a provisioned cluster. + + It's first composed once the cloud cluster XR is Ready, then kept while + it's observed. The cloud cluster XR goes unready whenever any resource it + composes does, and every Object on the cluster - its namespaces, + PoolObservations, and every ModelReplica's and ModelRoute's - reaches it + through this ProviderConfig, so dropping it then would have Crossplane + delete a ProviderConfig they all still use. + """ + return cluster_ready or CLUSTER_PROVIDER_CONFIG_RESOURCE_KEY in self.req.observed.resources + def compose_cluster_provider_config( self, kubeconfig_name: str, @@ -1317,13 +1378,17 @@ def compose_cluster_provider_config( ), ) resource.update( - self.rsp.desired.resources["cluster-provider-config-kubernetes"], + self.rsp.desired.resources[CLUSTER_PROVIDER_CONFIG_RESOURCE_KEY], cpc, ) - self.rsp.desired.resources["cluster-provider-config-kubernetes"].ready = fnv1.READY_TRUE + self.rsp.desired.resources[CLUSTER_PROVIDER_CONFIG_RESOURCE_KEY].ready = fnv1.READY_TRUE - def write_status(self, schedulable_pools: list[v1alpha1.SchedulablePool]) -> None: - """Write the InferenceCluster status.""" + def write_status( + self, + schedulable_pools: list[v1alpha1.SchedulablePool], + node_pools: list[v1alpha1.NodePoolModel] | None, + ) -> None: + """Write the InferenceCluster status, with status.nodePools unless node_pools is None.""" status = v1alpha1.Status( providerConfigRef=v1alpha1.ProviderConfigRef( name=resource.child_name(_name(self.xr.metadata), "cluster-kubeconfig"), @@ -1331,6 +1396,8 @@ def write_status(self, schedulable_pools: list[v1alpha1.SchedulablePool]) -> Non namespace=_NAMESPACE_SYSTEM, schedulablePools=schedulable_pools, ) + if node_pools is not None: + status.nodePools = node_pools cache_storage_class = self.observed_cache_storage_class() if cache_storage_class: status.cache = v1alpha1.CacheModel(storageClassName=cache_storage_class) @@ -1370,38 +1437,68 @@ def write_status(self, schedulable_pools: list[v1alpha1.SchedulablePool]) -> Non status.model_dump(mode="json", exclude_unset=True, by_alias=True, warnings=False), ) - def derive_conditions(self, *, cluster_ready: bool) -> None: - """Derive ClusterReady and BackendReady conditions.""" - backend_ready = ( - resource.get_condition(self.req.observed.resources.get(BACKEND_RESOURCE_KEY), "Ready").status == "True" - ) - if BACKEND_RESOURCE_KEY in self.rsp.desired.resources and backend_ready: + def relayed_node_pools(self, cluster: str) -> list[v1alpha1.NodePoolModel]: + """Each node pool's state, as the cloud cluster XR composed as cluster reports it. + + A pool the cloud cluster XR doesn't report is Pending, as every pool is + before the cloud cluster XR exists. + """ + observed = self.req.observed.resources.get(cluster) + status = (resource.struct_to_dict(observed.resource).get("status") or {}) if observed else {} + reported = {p["name"]: p for p in status.get("nodePools") or []} + return [ + v1alpha1.NodePoolModel.model_validate(reported[pool.name]) + if pool.name in reported + else v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING) + for pool in self.xr.spec.nodePools or [] + ] + + def derive_conditions(self, *, cluster: str | None) -> None: + """Derive the stage conditions of a cluster Accepted for composing.""" + if ( + BACKEND_RESOURCE_KEY in self.rsp.desired.resources + and resource.get_condition(self.req.observed.resources.get(BACKEND_RESOURCE_KEY), "Ready").status == "True" + ): self.rsp.desired.resources[BACKEND_RESOURCE_KEY].ready = fnv1.READY_TRUE - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_CLUSTER_READY, - status="True" if cluster_ready else "False", - reason=CONDITION_REASON_CLUSTER_RUNNING if cluster_ready else CONDITION_REASON_PROVISIONING, - ), - ) + accepted = resource.Condition(typ=CONDITION_TYPE_ACCEPTED, status="True", reason=CONDITION_REASON_AVAILABLE) + response.set_conditions(self.rsp, accepted, *self.stage_conditions(accepted, cluster=cluster)) - if not cluster_ready: - backend_reason = CONDITION_REASON_WAITING_FOR_CLUSTER - elif backend_ready: - backend_reason = CONDITION_REASON_BACKEND_HEALTHY - else: - backend_reason = CONDITION_REASON_INSTALLING + def stage_conditions( + self, accepted: resource.Condition, *, cluster: str | None + ) -> tuple[resource.Condition, resource.Condition, resource.Condition]: + """ClusterReady, NodePoolsReady and ServingStackReady, relaying the first + two from the cloud cluster XR composed as cluster. cluster is None for an + Existing cluster, which Modelplane doesn't provision. - response.set_conditions( - self.rsp, - resource.Condition( - typ=CONDITION_TYPE_BACKEND_READY, - status="True" if backend_ready else "False", - reason=backend_reason, - ), - ) + A stage whose resources aren't observed yet is Pending on the stage + before it, Accepted included. + """ + if cluster is None: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_EXISTING + ) + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_EXISTING + ) + else: + observed = self.req.observed.resources.get(cluster) + cluster_ready = _relay(observed, CONDITION_TYPE_CLUSTER_READY, after=accepted) + pools_ready = _relay(observed, CONDITION_TYPE_NODE_POOLS_READY, after=cluster_ready) + + backend = self.req.observed.resources.get(BACKEND_RESOURCE_KEY) + waiting_on = next((c.typ for c in (pools_ready, accepted) if c.status != "True"), None) + if resource.get_condition(backend, "Ready").status == "True": + serving_stack = resource.Condition( + typ=CONDITION_TYPE_SERVING_STACK_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + elif backend is None and waiting_on is not None: + serving_stack = _pending(CONDITION_TYPE_SERVING_STACK_READY, after=waiting_on) + else: + serving_stack = resource.Condition( + typ=CONDITION_TYPE_SERVING_STACK_READY, status="False", reason=CONDITION_REASON_PROVISIONING + ) + return cluster_ready, pools_ready, serving_stack def compose_gke_cluster(self, gke: v1alpha1.Gke) -> bool: """Compose a GKECluster XR. diff --git a/functions/compose-inference-cluster/tests/test_fn.py b/functions/compose-inference-cluster/tests/test_fn.py index 5d5785d24..509869e6a 100644 --- a/functions/compose-inference-cluster/tests/test_fn.py +++ b/functions/compose-inference-cluster/tests/test_fn.py @@ -62,14 +62,16 @@ def _inference_cluster(*, cluster: v1alpha1.Cluster, node_pools: list[v1alpha1.N def _desired_inference_cluster( - *, schedulable_pools: list[dict], cache: dict | None, gateway: dict | None + *, schedulable_pools: list[dict], node_pools: list[dict] | None, cache: dict | None, gateway: dict | None ) -> fnv1.Resource: - """The desired InferenceCluster XR's status, with its cache and gateway unless they're None.""" + """The desired InferenceCluster XR's status, with its node pools, cache and gateway unless they're None.""" status: dict = { "providerConfigRef": {"name": "test-cluster-cluster-kubeconfig-d0f89"}, "namespace": "modelplane-system", "schedulablePools": schedulable_pools, } + if node_pools is not None: + status["nodePools"] = node_pools if cache is not None: status["cache"] = cache if gateway is not None: @@ -281,6 +283,41 @@ def _eks_cluster( ) +def _observed_eks_cluster( + *, ready: bool, conditions: list[dict], node_pools: list[dict], cache: dict | None +) -> fnv1.Resource: + """The observed EKSCluster with l4-pool, reporting conditions and node_pools, and its cache unless it's None.""" + status: dict = { + "conditions": [ + {**c, "lastTransitionTime": "2024-01-01T00:00:00Z"} + for c in [ + {"type": "Ready", "status": "True", "reason": "Available"} + if ready + else {"type": "Ready", "status": "False", "reason": "Creating"}, + *conditions, + ] + ], + "nodePools": node_pools, + "secrets": [{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], + } + if cache is not None: + status["cache"] = cache + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "EKSCluster", + "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "spec": { + "region": "us-west-2", + "nodePools": [{"name": "l4-pool", "role": "GPU", "instanceType": "g6.xlarge", "nodeCount": 2}], + }, + "status": status, + } + ) + ) + + def _vultr_cluster(*, credentials: dict | None, ready: fnv1.Ready) -> fnv1.Resource: """The composed VultrCluster with l40s-pool, using credentials unless they're None.""" spec: dict = { @@ -690,6 +727,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "l4-pool", @@ -745,8 +783,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -796,6 +852,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "l4-pool", @@ -867,8 +924,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -928,6 +1003,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Pending"}], schedulable_pools=[ { "name": "l4-pool", @@ -986,8 +1062,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -1088,11 +1184,29 @@ def _to_dict(msg: message.Message) -> dict: ], conditions=[ fnv1.Condition( - type="ClusterReady", + type="Accepted", status=fnv1.STATUS_CONDITION_FALSE, reason="InvalidNodePool", message="InferenceClass gpu-l4 has no GKE provisioning block", ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for Accepted", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -1164,6 +1278,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Pending"}], schedulable_pools=[ { "name": "l4-pool", @@ -1232,8 +1347,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -1355,11 +1490,29 @@ def _to_dict(msg: message.Message) -> dict: ], conditions=[ fnv1.Condition( - type="ClusterReady", + type="Accepted", status=fnv1.STATUS_CONDITION_FALSE, reason="InvalidNodePool", message="Node pool l4-pool must set zones to [us-central1-a] to use CapacityReservation l4-calendar", ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for Accepted", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -1479,11 +1632,29 @@ def _to_dict(msg: message.Message) -> dict: ], conditions=[ fnv1.Condition( - type="ClusterReady", + type="Accepted", status=fnv1.STATUS_CONDITION_FALSE, reason="InvalidNodePool", message="CapacityReservation l4-block is for EKS, not GKE", ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for Accepted", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -1561,11 +1732,29 @@ def _to_dict(msg: message.Message) -> dict: ], conditions=[ fnv1.Condition( - type="ClusterReady", + type="Accepted", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForReservations", message="Waiting for CapacityReservations: l4-calendar", ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for Accepted", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -1698,6 +1887,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Pending"}], schedulable_pools=[ { "name": "l4-pool", @@ -1760,14 +1950,34 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), ComposeCase( - name="ExistingBackendReady", - reason="Once its ServingStack is observed Ready with an address, an Existing cluster relays the address, reports its backend healthy, and composes a PoolObservation for each pool.", + name="ExistingServingStackReady", + reason="Once its ServingStack is observed Ready with an address, an Existing cluster relays the address, reports ServingStackReady, and composes a PoolObservation for each pool.", req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_inference_cluster( @@ -1807,6 +2017,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "l4-pool", @@ -1863,8 +2074,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), ], ), ), @@ -1944,6 +2173,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "l4-pool", @@ -2021,8 +2251,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), ], ), ), @@ -2101,6 +2349,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "gpu-l4x4-a", @@ -2160,8 +2409,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), ], ), ), @@ -2234,6 +2501,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "gpu-l4x4-a", @@ -2290,8 +2558,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), ], ), ), @@ -2370,6 +2656,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "gpu-l4x4-a", @@ -2429,8 +2716,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), ], ), ), @@ -2508,6 +2813,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "gpu-l4x4-a", @@ -2571,8 +2877,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), ], ), ), @@ -2662,6 +2986,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "gpu-l4x4-a", @@ -2720,8 +3045,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -2795,6 +3138,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Pending"}], schedulable_pools=[ { "name": "l4-pool", @@ -2872,17 +3216,34 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), - # The ClusterProviderConfig is built only from the kubeconfig, so it's - # recreated once the kubeconfig is observed again. It is never emitted with - # an empty secretRef. ComposeCase( - name="EKSObservedCPC", - reason="With a ClusterProviderConfig observed but no EKSCluster yet, an EKS cluster leaves the ClusterProviderConfig out of desired state.", + name="EKSPoolsProvisioning", + reason="With its EKSCluster reporting the cluster ready and its pool provisioning, an EKS cluster relays both stages and the pool's state, and ServingStackReady waits on NodePoolsReady.", req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_inference_cluster( @@ -2898,24 +3259,21 @@ def _to_dict(msg: message.Message) -> dict: ], ), resources={ - "cluster-provider-config-kubernetes": fnv1.Resource( - resource=resource.dict_to_struct( + "eks-cluster": _observed_eks_cluster( + ready=False, + conditions=[ + {"type": "ClusterReady", "status": "True", "reason": "Available"}, { - "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", - "kind": "ClusterProviderConfig", - "metadata": {"name": "test-cluster-cluster-kubeconfig-d0f89"}, - "spec": { - "credentials": { - "source": "Secret", - "secretRef": { - "namespace": "modelplane-system", - "name": "test-cluster-kubeconfig-abcde", - "key": "kubeconfig", - }, - }, - }, - } - ), + "type": "NodePoolsReady", + "status": "False", + "reason": "Provisioning", + "message": "0 of 1 node pools ready", + }, + ], + node_pools=[ + {"name": "l4-pool", "state": "Provisioning"}, + ], + cache=None, ), "activation": _observed_activation_policy( activated=[ @@ -2969,6 +3327,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Provisioning"}], schedulable_pools=[ { "name": "l4-pool", @@ -3046,61 +3405,1142 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 node pools ready", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), - # The GKE kubeconfig has no embedded credentials, hence the service account - # identity on the ClusterProviderConfig. ComposeCase( - name="GKEReady", - reason="Once its GKECluster is Ready, a GKE cluster composes the ClusterProviderConfig, ServingStack and Usage, and relays its RWX StorageClass.", + name="EKSPoolFailed", + reason="With its EKSCluster reporting a failed pool while the cluster provisions, an EKS cluster relays the failure and the provider's error.", req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_inference_cluster( - cluster=v1alpha1.Cluster(source="GKE", gke=v1alpha1.Gke(region="us-central1")), + cluster=v1alpha1.Cluster(source="EKS", eks=v1alpha1.Eks(region="us-west-2")), node_pools=[ v1alpha1.NodePool( name="l4-pool", - className="gpu-l4", + className="gpu-l4-eks", nodeCount=2, maxNodeCount=4, - zones=["us-central1-a"], + zones=["us-west-2a", "us-west-2b"], ), ], ), resources={ - "gke-cluster": fnv1.Resource( - resource=resource.dict_to_struct( + "eks-cluster": _observed_eks_cluster( + ready=False, + conditions=[ { - "apiVersion": "infrastructure.modelplane.ai/v1alpha1", - "kind": "GKECluster", - "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "type": "ClusterReady", + "status": "False", + "reason": "Provisioning", + "message": "12 of 52 resources ready", + }, + { + "type": "NodePoolsReady", + "status": "False", + "reason": "Failed", + "message": "1 of 1 node pools failed: l4-pool", + }, + ], + node_pools=[ + { + "name": "l4-pool", + "state": "Failed", + "message": "nodegroup-l4-pool: async create failed: InsufficientInstanceCapacity", + }, + ], + cache=None, + ), + "activation": _observed_activation_policy( + activated=[ + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", + ] + ), + }, + ), + required_resources={ + "class-gpu-l4-eks": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4-eks", + count=1, + memory="24Gi", + provisioning={ + "provider": "EKS", + "eks": { + "instanceType": "g6.xlarge", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 1}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + node_pools=[ + { + "name": "l4-pool", + "state": "Failed", + "message": "nodegroup-l4-pool: async create failed: InsufficientInstanceCapacity", + }, + ], + schedulable_pools=[ + { + "name": "l4-pool", + "nodes": 4, + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 1, + "capacity": {"memory": {"value": "24Gi"}}, + }, + ], + }, + ], + cache=None, + gateway=None, + ), + resources={ + "usage-inferenceclass-gpu-l4-eks": _input_usage(kind="InferenceClass", name="gpu-l4-eks"), + "activation": _activation_policy( + activate=[ + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", + ], + ready=fnv1.READY_TRUE, + ), + "eks-cluster": _eks_cluster( + zones=["us-west-2a", "us-west-2b"], + capacity_reservation=None, + fabric=None, + ready=fnv1.READY_UNSPECIFIED, + ), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4-eks": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4-eks" + ), + }, + ), + conditions=[ + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="12 of 52 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 1 node pools failed: l4-pool", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), + ], + ), + ), + # The ClusterProviderConfig is built only from the kubeconfig, so it's + # recreated once the kubeconfig is observed again. It is never emitted with + # an empty secretRef. + ComposeCase( + name="EKSObservedCPC", + reason="With a ClusterProviderConfig observed but no EKSCluster yet, an EKS cluster leaves the ClusterProviderConfig out of desired state.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster(source="EKS", eks=v1alpha1.Eks(region="us-west-2")), + node_pools=[ + v1alpha1.NodePool( + name="l4-pool", + className="gpu-l4-eks", + nodeCount=2, + maxNodeCount=4, + zones=["us-west-2a", "us-west-2b"], + ), + ], + ), + resources={ + "cluster-provider-config-kubernetes": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "ClusterProviderConfig", + "metadata": {"name": "test-cluster-cluster-kubeconfig-d0f89"}, + "spec": { + "credentials": { + "source": "Secret", + "secretRef": { + "namespace": "modelplane-system", + "name": "test-cluster-kubeconfig-abcde", + "key": "kubeconfig", + }, + }, + }, + } + ), + ), + "activation": _observed_activation_policy( + activated=[ + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", + ] + ), + }, + ), + required_resources={ + "class-gpu-l4-eks": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4-eks", + count=1, + memory="24Gi", + provisioning={ + "provider": "EKS", + "eks": { + "instanceType": "g6.xlarge", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 1}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Pending"}], + schedulable_pools=[ + { + "name": "l4-pool", + "nodes": 4, + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 1, + "capacity": {"memory": {"value": "24Gi"}}, + }, + ], + }, + ], + cache=None, + gateway=None, + ), + resources={ + "usage-inferenceclass-gpu-l4-eks": _input_usage(kind="InferenceClass", name="gpu-l4-eks"), + "activation": _activation_policy( + activate=[ + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", + ], + ready=fnv1.READY_TRUE, + ), + "eks-cluster": _eks_cluster( + zones=["us-west-2a", "us-west-2b"], + capacity_reservation=None, + fabric=None, + ready=fnv1.READY_UNSPECIFIED, + ), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4-eks": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4-eks" + ), + }, + ), + conditions=[ + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), + ], + ), + ), + # The GKE kubeconfig has no embedded credentials, hence the service account + # identity on the ClusterProviderConfig. + ComposeCase( + name="GKEReady", + reason="Once its GKECluster is Ready, a GKE cluster composes the ClusterProviderConfig, ServingStack and Usage, and relays its RWX StorageClass.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster(source="GKE", gke=v1alpha1.Gke(region="us-central1")), + node_pools=[ + v1alpha1.NodePool( + name="l4-pool", + className="gpu-l4", + nodeCount=2, + maxNodeCount=4, + zones=["us-central1-a"], + ), + ], + ), + resources={ + "gke-cluster": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "GKECluster", + "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "spec": { + "region": "us-central1", + "nodePools": [{"name": "system", "role": "System", "machineType": "e2-standard-4"}], + }, + "status": { + "nodePools": [{"name": "l4-pool", "state": "Ready"}], + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2026-06-08T00:00:00Z", + }, + { + "type": "ClusterReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + { + "type": "NodePoolsReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + "cache": {"storageClassName": "modelplane-rwx"}, + "secrets": [ + { + "type": "Kubeconfig", + "name": "test-cluster-kubeconfig-abcde", + "key": "kubeconfig", + }, + { + "type": "GoogleApplicationCredentials", + "name": "test-cluster-sa-key-fghij", + "key": "credentials.json", + }, + ], + }, + } + ), + ), + "activation": _observed_activation_policy( + activated=[ + "projectiammembers.cloudplatform.gcp.m.upbound.io", + "projectservices.cloudplatform.gcp.m.upbound.io", + "serviceaccounts.cloudplatform.gcp.m.upbound.io", + "serviceaccountkeys.cloudplatform.gcp.m.upbound.io", + "networks.compute.gcp.m.upbound.io", + "subnetworks.compute.gcp.m.upbound.io", + "clusters.container.gcp.m.upbound.io", + "nodepools.container.gcp.m.upbound.io", + ] + ), + }, + ), + required_resources={ + "class-gpu-l4": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4", + count=1, + memory="24Gi", + provisioning={ + "provider": "GKE", + "gke": { + "machineType": "g2-standard-48", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 1}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Ready"}], + schedulable_pools=[ + { + "name": "l4-pool", + "nodes": 4, + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 1, + "capacity": {"memory": {"value": "24Gi"}}, + }, + ], + }, + ], + cache={"storageClassName": "modelplane-rwx"}, + gateway=None, + ), + resources={ + "usage-inferenceclass-gpu-l4": _input_usage(kind="InferenceClass", name="gpu-l4"), + "activation": _activation_policy( + activate=[ + "projectiammembers.cloudplatform.gcp.m.upbound.io", + "projectservices.cloudplatform.gcp.m.upbound.io", + "serviceaccounts.cloudplatform.gcp.m.upbound.io", + "serviceaccountkeys.cloudplatform.gcp.m.upbound.io", + "networks.compute.gcp.m.upbound.io", + "subnetworks.compute.gcp.m.upbound.io", + "clusters.container.gcp.m.upbound.io", + "nodepools.container.gcp.m.upbound.io", + ], + ready=fnv1.READY_TRUE, + ), + "gke-cluster": _gke_cluster(credentials=None, reservation=None, ready=fnv1.READY_TRUE), + "cluster-provider-config-kubernetes": _cluster_provider_config( + kubeconfig="test-cluster-kubeconfig-abcde", + identity={ + "type": "GoogleApplicationCredentials", + "source": "Secret", + "secretRef": { + "namespace": "modelplane-system", + "name": "test-cluster-sa-key-fghij", + "key": "credentials.json", + }, + }, + ), + "serving-stack": _serving_stack( + cloud="GKE", + dra_drivers=("gpu.nvidia.com",), + secrets=[ + {"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}, + { + "type": "GoogleApplicationCredentials", + "name": "test-cluster-sa-key-fghij", + "key": "credentials.json", + }, + ], + client_cas=None, + gpu=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "usage-gke-by-backend": _backend_usage(cluster_kind="GKECluster"), + }, + ), + results=[fnv1.Result(severity=fnv1.SEVERITY_NORMAL, message="GKE cluster ready, composing backend")], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4" + ), + }, + ), + conditions=[ + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + ], + ), + ), + ComposeCase( + name="EKSReady", + reason="Once its EKSCluster is Ready, an EKS cluster composes the ClusterProviderConfig, ServingStack and Usage, and relays its status.cache.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster(source="EKS", eks=v1alpha1.Eks(region="us-west-2")), + node_pools=[ + v1alpha1.NodePool( + name="l4-pool", + className="gpu-l4-eks", + nodeCount=2, + maxNodeCount=4, + zones=["us-west-2a", "us-west-2b"], + ), + ], + ), + resources={ + "eks-cluster": _observed_eks_cluster( + ready=True, + conditions=[ + {"type": "ClusterReady", "status": "True", "reason": "Available"}, + {"type": "NodePoolsReady", "status": "True", "reason": "Available"}, + ], + node_pools=[ + {"name": "l4-pool", "state": "Ready"}, + ], + cache={"storageClassName": "modelplane-rwx-efs"}, + ), + "activation": _observed_activation_policy( + activated=[ + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", + ] + ), + }, + ), + required_resources={ + "class-gpu-l4-eks": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4-eks", + count=1, + memory="24Gi", + provisioning={ + "provider": "EKS", + "eks": { + "instanceType": "g6.xlarge", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 1}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Ready"}], + schedulable_pools=[ + { + "name": "l4-pool", + "nodes": 4, + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 1, + "capacity": {"memory": {"value": "24Gi"}}, + }, + ], + }, + ], + cache={"storageClassName": "modelplane-rwx-efs"}, + gateway=None, + ), + resources={ + "usage-inferenceclass-gpu-l4-eks": _input_usage(kind="InferenceClass", name="gpu-l4-eks"), + "activation": _activation_policy( + activate=[ + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", + ], + ready=fnv1.READY_TRUE, + ), + "eks-cluster": _eks_cluster( + zones=["us-west-2a", "us-west-2b"], + capacity_reservation=None, + fabric=None, + ready=fnv1.READY_TRUE, + ), + "cluster-provider-config-kubernetes": _cluster_provider_config( + kubeconfig="test-cluster-kubeconfig-abcde", identity=None + ), + "serving-stack": _serving_stack( + cloud="EKS", + dra_drivers=("gpu.nvidia.com",), + secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], + client_cas=None, + gpu=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "usage-eks-by-backend": _backend_usage(cluster_kind="EKSCluster"), + }, + ), + results=[fnv1.Result(severity=fnv1.SEVERITY_NORMAL, message="EKS cluster ready, composing backend")], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4-eks": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4-eks" + ), + }, + ), + conditions=[ + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + ], + ), + ), + # An EKSCluster that was Ready goes unready whenever one of its resources + # does. Every Object on the cluster still reaches it through the + # ClusterProviderConfig, so dropping it would have Crossplane delete it from + # under them. + ComposeCase( + name="EKSUnreadyKeepsCPC", + reason="With its EKSCluster gone unready after composing the ClusterProviderConfig and ServingStack, an EKS cluster keeps both.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster(source="EKS", eks=v1alpha1.Eks(region="us-west-2")), + node_pools=[ + v1alpha1.NodePool( + name="l4-pool", + className="gpu-l4-eks", + nodeCount=2, + maxNodeCount=4, + zones=["us-west-2a", "us-west-2b"], + ), + ], + ), + resources={ + "eks-cluster": _observed_eks_cluster( + ready=False, + conditions=[ + {"type": "ClusterReady", "status": "True", "reason": "Available"}, + { + "type": "NodePoolsReady", + "status": "False", + "reason": "Provisioning", + "message": "0 of 1 node pools ready", + }, + ], + node_pools=[ + {"name": "l4-pool", "state": "Provisioning"}, + ], + cache={"storageClassName": "modelplane-rwx-efs"}, + ), + "cluster-provider-config-kubernetes": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "ClusterProviderConfig", + "metadata": {"name": "test-cluster-cluster-kubeconfig-d0f89"}, + } + ), + ), + "serving-stack": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "ServingStack", + "metadata": {"name": "test-cluster-serving-stack-fd00b"}, + "spec": { + "secrets": [ + { + "type": "Kubeconfig", + "name": "test-cluster-kubeconfig-abcde", + "key": "kubeconfig", + }, + ], + }, + } + ), + ), + "activation": _observed_activation_policy( + activated=[ + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", + ] + ), + }, + ), + required_resources={ + "class-gpu-l4-eks": fnv1.Resources( + items=[ + _inference_class( + name="gpu-l4-eks", + count=1, + memory="24Gi", + provisioning={ + "provider": "EKS", + "eks": { + "instanceType": "g6.xlarge", + "diskSizeGb": 100, + "accelerator": {"type": "nvidia-l4", "count": 1}, + }, + }, + ), + ], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Provisioning"}], + schedulable_pools=[ + { + "name": "l4-pool", + "nodes": 4, + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 1, + "capacity": {"memory": {"value": "24Gi"}}, + }, + ], + }, + ], + cache={"storageClassName": "modelplane-rwx-efs"}, + gateway=None, + ), + resources={ + "usage-inferenceclass-gpu-l4-eks": _input_usage(kind="InferenceClass", name="gpu-l4-eks"), + "activation": _activation_policy( + activate=[ + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", + ], + ready=fnv1.READY_TRUE, + ), + "eks-cluster": _eks_cluster( + zones=["us-west-2a", "us-west-2b"], + capacity_reservation=None, + fabric=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "cluster-provider-config-kubernetes": _cluster_provider_config( + kubeconfig="test-cluster-kubeconfig-abcde", identity=None + ), + "serving-stack": _serving_stack( + cloud="EKS", + dra_drivers=("gpu.nvidia.com",), + secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], + client_cas=None, + gpu=None, + ready=fnv1.READY_UNSPECIFIED, + ), + "usage-eks-by-backend": _backend_usage(cluster_kind="EKSCluster"), + }, + ), + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "model-replicas": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelReplica", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-routes": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="ModelRoute", + match_labels=fnv1.MatchLabels(labels={"modelplane.ai/cluster": "test-cluster"}), + ), + "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), + "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), + "class-gpu-l4-eks": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4-eks" + ), + }, + ), + conditions=[ + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 node pools ready", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + ], + ), + ), + # compose-eks-cluster turns the pool's capacityReservation into a + # CAPACITY_BLOCK node group. + ComposeCase( + name="PoolAddedStackReady", + reason="With a node pool provisioning on a running cluster whose ServingStack is Ready, ServingStackReady stays True while NodePoolsReady is Provisioning.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_inference_cluster( + cluster=v1alpha1.Cluster(source="EKS", eks=v1alpha1.Eks(region="us-west-2")), + node_pools=[ + v1alpha1.NodePool( + name="l4-pool", + className="gpu-l4-eks", + nodeCount=2, + maxNodeCount=4, + zones=["us-west-2a", "us-west-2b"], + ), + ], + ), + resources={ + "eks-cluster": _observed_eks_cluster( + ready=False, + conditions=[ + {"type": "ClusterReady", "status": "True", "reason": "Available"}, + { + "type": "NodePoolsReady", + "status": "False", + "reason": "Provisioning", + "message": "0 of 1 node pools ready", + }, + ], + node_pools=[ + {"name": "l4-pool", "state": "Provisioning"}, + ], + cache={"storageClassName": "modelplane-rwx-efs"}, + ), + "cluster-provider-config-kubernetes": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "ClusterProviderConfig", + "metadata": {"name": "test-cluster-cluster-kubeconfig-d0f89"}, + } + ), + ), + "serving-stack": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "ServingStack", + "metadata": {"name": "test-cluster-serving-stack-fd00b"}, "spec": { - "region": "us-central1", - "nodePools": [{"name": "system", "role": "System", "machineType": "e2-standard-4"}], - }, - "status": { - "conditions": [ - { - "type": "Ready", - "status": "True", - "reason": "Available", - "lastTransitionTime": "2026-06-08T00:00:00Z", - }, - ], - "cache": {"storageClassName": "modelplane-rwx"}, "secrets": [ { "type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig", }, + ], + }, + "status": { + "conditions": [ { - "type": "GoogleApplicationCredentials", - "name": "test-cluster-sa-key-fghij", - "key": "credentials.json", + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", }, ], }, @@ -3109,29 +4549,43 @@ def _to_dict(msg: message.Message) -> dict: ), "activation": _observed_activation_policy( activated=[ - "projectiammembers.cloudplatform.gcp.m.upbound.io", - "projectservices.cloudplatform.gcp.m.upbound.io", - "serviceaccounts.cloudplatform.gcp.m.upbound.io", - "serviceaccountkeys.cloudplatform.gcp.m.upbound.io", - "networks.compute.gcp.m.upbound.io", - "subnetworks.compute.gcp.m.upbound.io", - "clusters.container.gcp.m.upbound.io", - "nodepools.container.gcp.m.upbound.io", + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", ] ), }, ), required_resources={ - "class-gpu-l4": fnv1.Resources( + "class-gpu-l4-eks": fnv1.Resources( items=[ _inference_class( - name="gpu-l4", + name="gpu-l4-eks", count=1, memory="24Gi", provisioning={ - "provider": "GKE", - "gke": { - "machineType": "g2-standard-48", + "provider": "EKS", + "eks": { + "instanceType": "g6.xlarge", "diskSizeGb": 100, "accelerator": {"type": "nvidia-l4", "count": 1}, }, @@ -3145,6 +4599,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Provisioning"}], schedulable_pools=[ { "name": "l4-pool", @@ -3161,56 +4616,59 @@ def _to_dict(msg: message.Message) -> dict: ], }, ], - cache={"storageClassName": "modelplane-rwx"}, + cache={"storageClassName": "modelplane-rwx-efs"}, gateway=None, ), resources={ - "usage-inferenceclass-gpu-l4": _input_usage(kind="InferenceClass", name="gpu-l4"), + "usage-inferenceclass-gpu-l4-eks": _input_usage(kind="InferenceClass", name="gpu-l4-eks"), "activation": _activation_policy( activate=[ - "projectiammembers.cloudplatform.gcp.m.upbound.io", - "projectservices.cloudplatform.gcp.m.upbound.io", - "serviceaccounts.cloudplatform.gcp.m.upbound.io", - "serviceaccountkeys.cloudplatform.gcp.m.upbound.io", - "networks.compute.gcp.m.upbound.io", - "subnetworks.compute.gcp.m.upbound.io", - "clusters.container.gcp.m.upbound.io", - "nodepools.container.gcp.m.upbound.io", + "eips.ec2.aws.m.upbound.io", + "internetgateways.ec2.aws.m.upbound.io", + "launchtemplates.ec2.aws.m.upbound.io", + "natgateways.ec2.aws.m.upbound.io", + "routes.ec2.aws.m.upbound.io", + "routetables.ec2.aws.m.upbound.io", + "routetableassociations.ec2.aws.m.upbound.io", + "securitygroups.ec2.aws.m.upbound.io", + "securitygroupegressrules.ec2.aws.m.upbound.io", + "securitygroupingressrules.ec2.aws.m.upbound.io", + "subnets.ec2.aws.m.upbound.io", + "vpcs.ec2.aws.m.upbound.io", + "filesystems.efs.aws.m.upbound.io", + "mounttargets.efs.aws.m.upbound.io", + "addons.eks.aws.m.upbound.io", + "clusters.eks.aws.m.upbound.io", + "clusterauths.eks.aws.m.upbound.io", + "nodegroups.eks.aws.m.upbound.io", + "podidentityassociations.eks.aws.m.upbound.io", + "policies.iam.aws.m.upbound.io", + "roles.iam.aws.m.upbound.io", + "rolepolicyattachments.iam.aws.m.upbound.io", ], ready=fnv1.READY_TRUE, ), - "gke-cluster": _gke_cluster(credentials=None, reservation=None, ready=fnv1.READY_TRUE), + "eks-cluster": _eks_cluster( + zones=["us-west-2a", "us-west-2b"], + capacity_reservation=None, + fabric=None, + ready=fnv1.READY_UNSPECIFIED, + ), "cluster-provider-config-kubernetes": _cluster_provider_config( - kubeconfig="test-cluster-kubeconfig-abcde", - identity={ - "type": "GoogleApplicationCredentials", - "source": "Secret", - "secretRef": { - "namespace": "modelplane-system", - "name": "test-cluster-sa-key-fghij", - "key": "credentials.json", - }, - }, + kubeconfig="test-cluster-kubeconfig-abcde", identity=None ), "serving-stack": _serving_stack( - cloud="GKE", + cloud="EKS", dra_drivers=("gpu.nvidia.com",), - secrets=[ - {"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}, - { - "type": "GoogleApplicationCredentials", - "name": "test-cluster-sa-key-fghij", - "key": "credentials.json", - }, - ], + secrets=[{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-abcde", "key": "kubeconfig"}], client_cas=None, gpu=None, - ready=fnv1.READY_UNSPECIFIED, + ready=fnv1.READY_TRUE, ), - "usage-gke-by-backend": _backend_usage(cluster_kind="GKECluster"), + "pool-observation-l4-pool": _pool_observation(pool="l4-pool"), + "usage-eks-by-backend": _backend_usage(cluster_kind="EKSCluster"), }, ), - results=[fnv1.Result(severity=fnv1.SEVERITY_NORMAL, message="GKE cluster ready, composing backend")], context=structpb.Struct(), requirements=fnv1.Requirements( resources={ @@ -3226,20 +4684,41 @@ def _to_dict(msg: message.Message) -> dict: ), "model-caches": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelCache"), "gateways": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceGateway"), - "class-gpu-l4": fnv1.ResourceSelector( - api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4" + "class-gpu-l4-eks": fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", kind="InferenceClass", match_name="gpu-l4-eks" ), }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 node pools ready", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), ], ), ), + # compose-eks-cluster turns the pool's capacityReservation into a + # CAPACITY_BLOCK node group. ComposeCase( - name="EKSReady", - reason="Once its EKSCluster is Ready, an EKS cluster composes the ClusterProviderConfig, ServingStack and Usage, and relays its status.cache.", + name="EKSClusterFunctionFailed", + reason="With its EKSCluster still Ready but its stage conditions Unknown after a fatal error in its function, an EKS cluster relays ClusterReady and NodePoolsReady as Unknown rather than True.", req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_inference_cluster( @@ -3255,27 +4734,43 @@ def _to_dict(msg: message.Message) -> dict: ], ), resources={ - "eks-cluster": fnv1.Resource( + "eks-cluster": _observed_eks_cluster( + ready=True, + conditions=[ + { + "type": "ClusterReady", + "status": "Unknown", + "reason": "FatalError", + "message": "A fatal error occurred before the status of this condition could be determined.", + }, + { + "type": "NodePoolsReady", + "status": "Unknown", + "reason": "FatalError", + "message": "A fatal error occurred before the status of this condition could be determined.", + }, + ], + node_pools=[ + {"name": "l4-pool", "state": "Ready"}, + ], + cache={"storageClassName": "modelplane-rwx-efs"}, + ), + "cluster-provider-config-kubernetes": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "ClusterProviderConfig", + "metadata": {"name": "test-cluster-cluster-kubeconfig-d0f89"}, + } + ), + ), + "serving-stack": fnv1.Resource( resource=resource.dict_to_struct( { "apiVersion": "infrastructure.modelplane.ai/v1alpha1", - "kind": "EKSCluster", - "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "kind": "ServingStack", + "metadata": {"name": "test-cluster-serving-stack-fd00b"}, "spec": { - "region": "us-west-2", - "nodePools": [ - {"name": "l4-pool", "role": "GPU", "instanceType": "g6.xlarge", "nodeCount": 2}, - ], - }, - "status": { - "conditions": [ - { - "type": "Ready", - "status": "True", - "reason": "Available", - "lastTransitionTime": "2024-01-01T00:00:00Z", - }, - ], "secrets": [ { "type": "Kubeconfig", @@ -3283,7 +4778,6 @@ def _to_dict(msg: message.Message) -> dict: "key": "kubeconfig", }, ], - "cache": {"storageClassName": "modelplane-rwx-efs"}, }, } ), @@ -3340,6 +4834,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Ready"}], schedulable_pools=[ { "name": "l4-pool", @@ -3408,7 +4903,6 @@ def _to_dict(msg: message.Message) -> dict: "usage-eks-by-backend": _backend_usage(cluster_kind="EKSCluster"), }, ), - results=[fnv1.Result(severity=fnv1.SEVERITY_NORMAL, message="EKS cluster ready, composing backend")], context=structpb.Struct(), requirements=fnv1.Requirements( resources={ @@ -3430,8 +4924,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_UNKNOWN, + reason="FatalError", + message="A fatal error occurred before the status of this condition could be determined.", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_UNKNOWN, + reason="FatalError", + message="A fatal error occurred before the status of this condition could be determined.", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -3519,6 +5033,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Pending"}], schedulable_pools=[ { "name": "l4-pool", @@ -3600,8 +5115,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -3751,11 +5286,29 @@ def _to_dict(msg: message.Message) -> dict: ], conditions=[ fnv1.Condition( - type="ClusterReady", + type="Accepted", status=fnv1.STATUS_CONDITION_FALSE, reason="InvalidNodePool", message="CapacityReservation l4-block is in region us-east-2, but the cluster is in us-west-2", ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for Accepted", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -3832,6 +5385,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Pending"}], schedulable_pools=[ { "name": "l4-pool", @@ -3906,8 +5460,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -3971,6 +5545,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "h100-pool", "state": "Pending"}], schedulable_pools=[ { "name": "h100-pool", @@ -4051,8 +5626,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -4098,6 +5693,7 @@ def _to_dict(msg: message.Message) -> dict: ], }, "status": { + "nodePools": [{"name": "h100-pool", "state": "Ready"}], "conditions": [ { "type": "Ready", @@ -4105,6 +5701,18 @@ def _to_dict(msg: message.Message) -> dict: "reason": "Available", "lastTransitionTime": "2024-01-01T00:00:00Z", }, + { + "type": "ClusterReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + { + "type": "NodePoolsReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, ], "secrets": [ { @@ -4160,6 +5768,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "h100-pool", "state": "Ready"}], schedulable_pools=[ { "name": "h100-pool", @@ -4271,8 +5880,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -4333,6 +5960,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "h100pool", "state": "Pending"}], schedulable_pools=[ { "name": "h100pool", @@ -4413,8 +6041,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -4457,6 +6105,7 @@ def _to_dict(msg: message.Message) -> dict: ], }, "status": { + "nodePools": [{"name": "h100pool", "state": "Ready"}], "conditions": [ { "type": "Ready", @@ -4464,6 +6113,18 @@ def _to_dict(msg: message.Message) -> dict: "reason": "Available", "lastTransitionTime": "2024-01-01T00:00:00Z", }, + { + "type": "ClusterReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + { + "type": "NodePoolsReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, ], "secrets": [ { @@ -4512,6 +6173,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "h100pool", "state": "Ready"}], schedulable_pools=[ { "name": "h100pool", @@ -4606,8 +6268,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -4671,6 +6351,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Pending"}], schedulable_pools=[ { "name": "l4-pool", @@ -4728,8 +6409,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -4765,6 +6466,7 @@ def _to_dict(msg: message.Message) -> dict: "nodePools": [{"name": "system", "role": "System", "machineType": "e2-standard-4"}], }, "status": { + "nodePools": [{"name": "l4-pool", "state": "Ready"}], "conditions": [ { "type": "Ready", @@ -4772,6 +6474,18 @@ def _to_dict(msg: message.Message) -> dict: "reason": "Available", "lastTransitionTime": "2026-06-08T00:00:00Z", }, + { + "type": "ClusterReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + { + "type": "NodePoolsReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, ], "cache": {"storageClassName": "modelplane-rwx"}, "secrets": [ @@ -4816,6 +6530,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l4-pool", "state": "Ready"}], schedulable_pools=[ { "name": "l4-pool", @@ -4903,8 +6618,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -4952,6 +6685,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l40s-pool", "state": "Pending"}], schedulable_pools=[ { "name": "l40s-pool", @@ -5001,8 +6735,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -5053,6 +6807,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l40s-pool", "state": "Pending"}], schedulable_pools=[ { "name": "l40s-pool", @@ -5104,8 +6859,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -5145,6 +6920,7 @@ def _to_dict(msg: message.Message) -> dict: ], }, "status": { + "nodePools": [{"name": "l40s-pool", "state": "Ready"}], "conditions": [ { "type": "Ready", @@ -5152,6 +6928,18 @@ def _to_dict(msg: message.Message) -> dict: "reason": "Available", "lastTransitionTime": "2024-01-01T00:00:00Z", }, + { + "type": "ClusterReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + { + "type": "NodePoolsReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, ], "secrets": [ { @@ -5192,6 +6980,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l40s-pool", "state": "Ready"}], schedulable_pools=[ { "name": "l40s-pool", @@ -5254,8 +7043,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -5295,6 +7102,7 @@ def _to_dict(msg: message.Message) -> dict: ], }, "status": { + "nodePools": [{"name": "cpu-pool", "state": "Ready"}], "conditions": [ { "type": "Ready", @@ -5302,6 +7110,18 @@ def _to_dict(msg: message.Message) -> dict: "reason": "Available", "lastTransitionTime": "2024-01-01T00:00:00Z", }, + { + "type": "ClusterReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + { + "type": "NodePoolsReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, ], "secrets": [ { @@ -5338,6 +7158,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "cpu-pool", "state": "Ready"}], schedulable_pools=[ { "name": "cpu-pool", @@ -5400,8 +7221,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -5452,6 +7291,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l40s-pool", "state": "Pending"}], schedulable_pools=[ { "name": "l40s-pool", @@ -5519,8 +7359,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -5575,6 +7435,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l40s-pool", "state": "Pending"}], schedulable_pools=[ { "name": "l40s-pool", @@ -5642,8 +7503,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -5680,6 +7561,7 @@ def _to_dict(msg: message.Message) -> dict: ], }, "status": { + "nodePools": [{"name": "l40s-pool", "state": "Ready"}], "conditions": [ { "type": "Ready", @@ -5687,6 +7569,18 @@ def _to_dict(msg: message.Message) -> dict: "reason": "Available", "lastTransitionTime": "2024-01-01T00:00:00Z", }, + { + "type": "ClusterReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + { + "type": "NodePoolsReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, ], "secrets": [ { @@ -5732,6 +7626,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l40s-pool", "state": "Ready"}], schedulable_pools=[ { "name": "l40s-pool", @@ -5812,8 +7707,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -5854,6 +7767,7 @@ def _to_dict(msg: message.Message) -> dict: ], }, "status": { + "nodePools": [{"name": "h100-pool", "state": "Ready"}], "conditions": [ { "type": "Ready", @@ -5861,6 +7775,18 @@ def _to_dict(msg: message.Message) -> dict: "reason": "Available", "lastTransitionTime": "2024-01-01T00:00:00Z", }, + { + "type": "ClusterReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + { + "type": "NodePoolsReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, ], "secrets": [ { @@ -5922,6 +7848,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "h100-pool", "state": "Ready"}], schedulable_pools=[ { "name": "h100-pool", @@ -6001,8 +7928,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -6046,6 +7991,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "h100-pool", "state": "Pending"}], schedulable_pools=[ { "name": "h100-pool", @@ -6109,8 +8055,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -6170,6 +8136,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "h100-pool", "state": "Pending"}], schedulable_pools=[ { "name": "h100-pool", @@ -6245,8 +8212,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -6295,7 +8282,20 @@ def _to_dict(msg: message.Message) -> dict: "reason": "Available", "lastTransitionTime": "2024-01-01T00:00:00Z", }, + { + "type": "ClusterReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + { + "type": "NodePoolsReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, ], + "nodePools": [{"name": "h100-pool", "state": "Ready"}], "secrets": [ { "type": "Kubeconfig", @@ -6336,6 +8336,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "h100-pool", "state": "Ready"}], schedulable_pools=[ { "name": "h100-pool", @@ -6412,8 +8413,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -6460,7 +8479,20 @@ def _to_dict(msg: message.Message) -> dict: "reason": "Available", "lastTransitionTime": "2024-01-01T00:00:00Z", }, + { + "type": "ClusterReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + { + "type": "NodePoolsReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, ], + "nodePools": [{"name": "cpu-pool", "state": "Ready"}], "secrets": [ { "type": "Kubeconfig", @@ -6497,6 +8529,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "cpu-pool", "state": "Ready"}], schedulable_pools=[ { "name": "cpu-pool", @@ -6570,8 +8603,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -6625,6 +8676,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l40s-pool", "state": "Pending"}], schedulable_pools=[ { "name": "l40s-pool", @@ -6695,8 +8747,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -6751,6 +8823,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l40s-pool", "state": "Pending"}], schedulable_pools=[ { "name": "l40s-pool", @@ -6821,8 +8894,28 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Provisioning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForCluster"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for NodePoolsReady", + ), ], ), ), @@ -6873,7 +8966,20 @@ def _to_dict(msg: message.Message) -> dict: "reason": "Available", "lastTransitionTime": "2024-01-01T00:00:00Z", }, + { + "type": "ClusterReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + { + "type": "NodePoolsReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, ], + "nodePools": [{"name": "l40s-pool", "state": "Ready"}], "secrets": [ { "type": "Kubeconfig", @@ -6917,6 +9023,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "l40s-pool", "state": "Ready"}], schedulable_pools=[ { "name": "l40s-pool", @@ -7000,8 +9107,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -7045,7 +9170,20 @@ def _to_dict(msg: message.Message) -> dict: "reason": "Available", "lastTransitionTime": "2024-01-01T00:00:00Z", }, + { + "type": "ClusterReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + { + "type": "NodePoolsReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, ], + "nodePools": [{"name": "cpu-pool", "state": "Ready"}], "secrets": [ { "type": "Kubeconfig", @@ -7085,6 +9223,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=[{"name": "cpu-pool", "state": "Ready"}], schedulable_pools=[ { "name": "cpu-pool", @@ -7164,8 +9303,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -7283,6 +9440,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "l4-pool", @@ -7345,8 +9503,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -7408,6 +9584,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "l4-pool", @@ -7465,8 +9642,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -7514,6 +9709,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "l4-pool", @@ -7571,8 +9767,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -7621,6 +9835,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "l4-pool", @@ -7677,8 +9892,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -7732,6 +9965,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[ { "name": "l4-pool", @@ -7787,8 +10021,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_FALSE, reason="Installing"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), ], ), ), @@ -7797,7 +10049,7 @@ def _to_dict(msg: message.Message) -> dict: # missing from it. ComposeCase( name="ClassUnresolved", - reason="With its class unresolved, the function fails the reconcile, reporting the missing class on ClusterReady.", + reason="With its class unresolved, the function fails the reconcile, reporting the missing class on Accepted.", req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_inference_cluster( @@ -7863,11 +10115,27 @@ def _to_dict(msg: message.Message) -> dict: ), conditions=[ fnv1.Condition( - type="ClusterReady", + type="Accepted", status=fnv1.STATUS_CONDITION_FALSE, reason="WaitingForClasses", message="Waiting for InferenceClasses: gpu-l4", ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for Accepted", + ), ], ), ), @@ -7956,7 +10224,7 @@ def _to_dict(msg: message.Message) -> dict: # gate goes back to returning without a fatal result. ComposeCase( name="UnsupportedDriverHolds", - reason="A running cluster whose class turns unsupported gets a fatal result, so Crossplane holds its resources instead of collecting them.", + reason="A running cluster whose class turns unsupported gets a fatal result, so Crossplane holds its resources instead of collecting them, and its stages keep reporting the running cluster.", req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_inference_cluster( @@ -7980,11 +10248,25 @@ def _to_dict(msg: message.Message) -> dict: "reason": "Available", "lastTransitionTime": "2026-06-08T00:00:00Z", }, + { + "type": "ClusterReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2026-06-08T00:00:00Z", + }, + { + "type": "NodePoolsReady", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2026-06-08T00:00:00Z", + }, ], + "nodePools": [{"name": "l4-pool", "state": "Ready"}], }, } ), ), + "serving-stack": _observed_serving_stack(gateway={"address": "34.55.100.10"}), }, ), required_resources={ @@ -8058,11 +10340,26 @@ def _to_dict(msg: message.Message) -> dict: ), conditions=[ fnv1.Condition( - type="ClusterReady", + type="Accepted", status=fnv1.STATUS_CONDITION_FALSE, reason="UnsupportedDevices", message="class gpu-l4 claims gpu.amd.com; GKE clusters support dra.cpu, gpu.nvidia.com", ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), ], ), ), @@ -8108,6 +10405,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[], cache=None, gateway={ @@ -8149,8 +10447,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), ], ), ), @@ -8182,7 +10498,10 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - schedulable_pools=[], cache=None, gateway={"address": "34.55.100.10", "caCertificate": "cluster-ca"} + node_pools=None, + schedulable_pools=[], + cache=None, + gateway={"address": "34.55.100.10", "caCertificate": "cluster-ca"}, ), resources={ "cluster-provider-config-kubernetes": _cluster_provider_config( @@ -8216,8 +10535,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), ], ), ), @@ -8254,7 +10591,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( - schedulable_pools=[], cache=None, gateway={"address": "34.55.100.10"} + node_pools=None, schedulable_pools=[], cache=None, gateway={"address": "34.55.100.10"} ), resources={ "usage-replicas": _guard_cluster_usage(reason="InferenceGateways use this InferenceCluster"), @@ -8289,8 +10626,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), ], ), ), @@ -8325,7 +10680,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_inference_cluster(schedulable_pools=[], cache=None, gateway=None), + composite=_desired_inference_cluster(schedulable_pools=[], node_pools=None, cache=None, gateway=None), resources={ "usage-replicas": _guard_cluster_usage(reason="InferenceGateways use this InferenceCluster"), "cluster-provider-config-kubernetes": _cluster_provider_config( @@ -8359,8 +10714,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), ], ), ), @@ -8405,6 +10778,7 @@ def _to_dict(msg: message.Message) -> dict: meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( composite=_desired_inference_cluster( + node_pools=None, schedulable_pools=[], cache=None, gateway={ @@ -8450,8 +10824,26 @@ def _to_dict(msg: message.Message) -> dict: }, ), conditions=[ - fnv1.Condition(type="ClusterReady", status=fnv1.STATUS_CONDITION_TRUE, reason="ClusterRunning"), - fnv1.Condition(type="BackendReady", status=fnv1.STATUS_CONDITION_TRUE, reason="BackendHealthy"), + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Existing", + ), + fnv1.Condition( + type="ServingStackReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), ], ), ), @@ -8550,7 +10942,7 @@ def test_dra_drivers_synthetic() -> None: def test_unsupported_devices() -> None: - """Classes claiming drivers their cloud can't serve gate ClusterReady + """Classes claiming drivers their cloud can't serve gate Accepted with UnsupportedDevices, naming every offending class, and stop with a fatal result so a running cluster holds its existing resources rather than being garbage-collected.""" @@ -8570,9 +10962,9 @@ def test_unsupported_devices() -> None: ) assert composer.check_devices_supported("Vultr") is False conditions = {c.type: c for c in composer.rsp.conditions} - assert conditions["ClusterReady"].status == fnv1.STATUS_CONDITION_FALSE - assert conditions["ClusterReady"].reason == "UnsupportedDevices" - assert conditions["ClusterReady"].message == ( + assert conditions["Accepted"].status == fnv1.STATUS_CONDITION_FALSE + assert conditions["Accepted"].reason == "UnsupportedDevices" + assert conditions["Accepted"].message == ( "class amd-mi355x-8x claims gpu.amd.com; class efa claims dra.net; " "Vultr clusters support dra.cpu, gpu.nvidia.com" ) diff --git a/functions/compose-nebius-cluster/function/fn.py b/functions/compose-nebius-cluster/function/fn.py index 254d76c49..33c426a0e 100644 --- a/functions/compose-nebius-cluster/function/fn.py +++ b/functions/compose-nebius-cluster/function/fn.py @@ -171,6 +171,38 @@ # modelplane-rwx-efs on EKS. _MANAGED_STORAGE_CLASS = "modelplane-rwx-fs" +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: network, control +# plane, system node group, cache filesystem and its CSI driver and +# StorageClass, and ProviderConfigs. compose-inference-cluster relays both +# conditions onto the InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + + +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + def _name(meta: metav1.ObjectMeta | None) -> str: """The object's name, always set on resources read from the API server.""" @@ -266,8 +298,10 @@ def compose(self) -> None: self.compose_provider_configs(creds) self.compose_csi_driver() self.compose_storage_class() - self.write_status(creds) self.mark_readiness(provider_configs_composed=creds is not None) + node_pools = self.node_pool_statuses() + self.derive_conditions(node_pools) + self.write_status(creds, node_pools) def resolve_credentials(self) -> Credentials | None: """Resolve the Nebius service account credentials Secret consumers @@ -833,7 +867,7 @@ def compose_provider_configs(self, creds: Credentials) -> None: ), ) - def write_status(self, creds: Credentials | None) -> None: + def write_status(self, creds: Credentials | None, node_pools: list[v1alpha1.NodePoolModel]) -> None: secrets = [ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -853,6 +887,7 @@ def write_status(self, creds: Credentials | None) -> None: credential.namespace = creds.namespace secrets.append(credential) status = v1alpha1.Status( + nodePools=node_pools, secrets=secrets, # The RWX StorageClass Modelplane composes for ModelCache. # Published immediately so ModelCache can target it; the class may @@ -888,6 +923,115 @@ def mark_readiness(self, *, provider_configs_composed: bool) -> None: self.rsp.desired.resources["provider-config-kubernetes"].ready = fnv1.READY_TRUE self.rsp.desired.resources["provider-config-helm"].ready = fnv1.READY_TRUE + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool. A GPU cluster is shared by + every pool on its InfiniBand fabric, so it belongs to each of them.""" + resources = [f"nodegroup-{pool.name}"] + fabric = _pool_fabric(pool) + if fabric: + resources.append(f"gpu-cluster-{fabric}") + return resources + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def _stage_pools(self) -> list[v1alpha1.NodePool]: + """The node pools NodePoolsReady and status.nodePools report. A System + pool runs the cluster's controllers and serving stack, so it's part of + ClusterReady.""" + return [pool for pool in self.xr.spec.nodePools if pool.role != "System"] + + def node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each node pool, from its resources.""" + statuses = [] + for pool in self._stage_pools(): + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: + """Derive ClusterReady from the readiness mark_readiness set, and + NodePoolsReady from node_pools. + + NodePoolsReady waits on ClusterReady only while no pool is Ready, so on + a running cluster the pools report their own state while the cluster + or a new pool provisions. A failed pool is reported Failed even while + the cluster is still provisioning. + """ + in_pools = {r for pool in self._stage_pools() for r in self.pool_resources(pool)} + cluster = sorted(r for r in self.rsp.desired.resources if r not in in_pools) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True" and not ready: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) + def _fabrics(self) -> list[str]: """The distinct InfiniBand fabrics used by the node pools, in first-use order.""" diff --git a/functions/compose-nebius-cluster/tests/test_fn.py b/functions/compose-nebius-cluster/tests/test_fn.py index 543b9529f..ecaaa000e 100644 --- a/functions/compose-nebius-cluster/tests/test_fn.py +++ b/functions/compose-nebius-cluster/tests/test_fn.py @@ -52,8 +52,8 @@ def _xr(*, credentials: v1alpha1.Credentials | None, node_pools: list[v1alpha1.N return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr(*, credentials_secret: bool) -> fnv1.Resource: - """The desired XR's status, which names the credentials Secret only if credentials_secret.""" +def _desired_xr(*, credentials_secret: bool, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR's status, publishing node_pools' states and naming the credentials Secret only if credentials_secret.""" secrets = [{"type": "Kubeconfig", "name": "test-cluster-kubeconfig-55b57", "key": "kubeconfig"}] if credentials_secret: secrets.append( @@ -66,7 +66,13 @@ def _desired_xr(*, credentials_secret: bool) -> fnv1.Resource: ) return fnv1.Resource( resource=resource.dict_to_struct( - {"status": {"secrets": secrets, "cache": {"storageClassName": "modelplane-rwx-fs"}}}, + { + "status": { + "nodePools": node_pools, + "secrets": secrets, + "cache": {"storageClassName": "modelplane-rwx-fs"}, + } + }, ), ) @@ -183,8 +189,8 @@ def _filesystem(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Re ) -def _subnet_observed(*, cred_kind: str, cred_name: str, subnet_id: str) -> fnv1.Resource: - """The observed subnet, provisioned with subnet_id but not yet Ready.""" +def _subnet_observed(*, cred_kind: str, cred_name: str, subnet_id: str, ready: bool) -> fnv1.Resource: + """The observed subnet, provisioned with subnet_id.""" return fnv1.Resource( resource=resource.dict_to_struct( { @@ -198,14 +204,24 @@ def _subnet_observed(*, cred_kind: str, cred_name: str, subnet_id: str) -> fnv1. "ipv4PrivatePools": {"useNetworkPools": True}, }, }, - "status": {"atProvider": {"id": subnet_id}}, + "status": { + "atProvider": {"id": subnet_id}, + "conditions": [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Creating", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, } ), ) -def _filesystem_observed(*, cred_kind: str, cred_name: str, filesystem_id: str) -> fnv1.Resource: - """The observed cache filesystem, provisioned with filesystem_id but not yet Ready.""" +def _filesystem_observed(*, cred_kind: str, cred_name: str, filesystem_id: str, ready: bool) -> fnv1.Resource: + """The observed cache filesystem, provisioned with filesystem_id.""" return fnv1.Resource( resource=resource.dict_to_struct( { @@ -219,7 +235,17 @@ def _filesystem_observed(*, cred_kind: str, cred_name: str, filesystem_id: str) "sizeGibibytes": 1024, }, }, - "status": {"atProvider": {"id": filesystem_id}}, + "status": { + "atProvider": {"id": filesystem_id}, + "conditions": [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Creating", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, } ), ) @@ -436,6 +462,124 @@ def _provider_config(*, api_version: str) -> fnv1.Resource: ) +def _gpu_cluster(*, ready: fnv1.Ready) -> fnv1.Resource: + """The composed GpuCluster for InfiniBand fabric-2.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "compute.nebius.m.upbound.io/v1beta1", + "kind": "GpuCluster", + "metadata": {"labels": {"modelplane.ai/fabric": "fabric-2"}}, + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, + "forProvider": { + "name": "test-cluster-fabric-2", + "infinibandFabric": "fabric-2", + }, + }, + } + ), + ready=ready, + ) + + +def _csi_release(*, ready: fnv1.Ready) -> fnv1.Resource: + """The composed csi-mounted-fs-path Helm release, tolerating the GPU taint.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "helm.m.crossplane.io/v1beta1", + "kind": "Release", + "metadata": { + "namespace": "modelplane-system", + "annotations": {"crossplane.io/external-name": "mp-csi-mounted-fs-path"}, + }, + "spec": { + "managementPolicies": ["Observe", "Create", "Update"], + "providerConfigRef": { + "kind": "ProviderConfig", + "name": "test-cluster-kubeconfig-55b57", + }, + "forProvider": { + "chart": { + "name": "csi-mounted-fs-path", + "repository": "oci://cr.eu-north1.nebius.cloud/mk8s/helm", + "version": "0.1.6", + }, + "namespace": "kube-system", + "values": { + "dataDir": "/mnt/data/csi-mounted-fs-path-data/", + "tolerations": [ + { + "key": "nvidia.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, + ], + }, + }, + }, + } + ), + ready=ready, + ) + + +def _storage_class() -> fnv1.Resource: + """The composed Object holding the RWX StorageClass, ready once created.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "managementPolicies": ["Observe", "Create", "Update"], + "providerConfigRef": { + "kind": "ProviderConfig", + "name": "test-cluster-kubeconfig-55b57", + }, + "readiness": {"policy": "SuccessfulCreate"}, + "forProvider": { + "manifest": { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "metadata": {"name": "modelplane-rwx-fs"}, + "provisioner": "mounted-fs-path.csi.nebius.ai", + "volumeBindingMode": "WaitForFirstConsumer", + }, + }, + }, + } + ), + ready=fnv1.READY_TRUE, + ) + + +def _observed_managed_resource(*, ready: bool, failure: str | None) -> fnv1.Resource: + """An observed managed resource with the given Ready status, whose provider + reports its last operation failed with failure unless it's None.""" + conditions = [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Creating", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ] + if failure is not None: + conditions.append( + { + "type": "LastAsyncOperation", + "status": "False", + "reason": "AsyncCreateFailure", + "message": failure, + "lastTransitionTime": "2024-01-01T00:00:00Z", + } + ) + return fnv1.Resource(resource=resource.dict_to_struct({"status": {"conditions": conditions}})) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -480,7 +624,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr(credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Pending"}]), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -506,6 +650,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -541,12 +699,16 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "subnet": _subnet_observed( - cred_kind="ClusterProviderConfig", cred_name="default", subnet_id="vpcsubnet-e00test" + cred_kind="ClusterProviderConfig", + cred_name="default", + subnet_id="vpcsubnet-e00test", + ready=False, ), "filesystem": _filesystem_observed( cred_kind="ClusterProviderConfig", cred_name="default", filesystem_id="computefilesystem-e00test", + ready=False, ), }, ), @@ -559,7 +721,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr( + credentials_secret=True, node_pools=[{"name": "cpu-pool", "state": "Provisioning"}] + ), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -587,6 +751,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 8 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -620,12 +798,16 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "subnet": _subnet_observed( - cred_kind="ClusterProviderConfig", cred_name="default", subnet_id="vpcsubnet-e00test" + cred_kind="ClusterProviderConfig", + cred_name="default", + subnet_id="vpcsubnet-e00test", + ready=False, ), "filesystem": _filesystem_observed( cred_kind="ClusterProviderConfig", cred_name="default", filesystem_id="computefilesystem-e00test", + ready=False, ), }, ), @@ -633,7 +815,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=False), + composite=_desired_xr( + credentials_secret=False, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -668,6 +852,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -704,12 +902,16 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "subnet": _subnet_observed( - cred_kind="ClusterProviderConfig", cred_name="default", subnet_id="vpcsubnet-e00test" + cred_kind="ClusterProviderConfig", + cred_name="default", + subnet_id="vpcsubnet-e00test", + ready=False, ), "filesystem": _filesystem_observed( cred_kind="ClusterProviderConfig", cred_name="default", filesystem_id="computefilesystem-e00test", + ready=False, ), "provider-config-kubernetes": fnv1.Resource( resource=resource.dict_to_struct( @@ -745,7 +947,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr( + credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -783,6 +987,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 8 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -819,12 +1037,16 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "subnet": _subnet_observed( - cred_kind="ClusterProviderConfig", cred_name="default", subnet_id="vpcsubnet-e00test" + cred_kind="ClusterProviderConfig", + cred_name="default", + subnet_id="vpcsubnet-e00test", + ready=False, ), "filesystem": _filesystem_observed( cred_kind="ClusterProviderConfig", cred_name="default", filesystem_id="computefilesystem-e00test", + ready=False, ), }, ), @@ -837,7 +1059,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr( + credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -852,22 +1076,7 @@ def _to_dict(msg: message.Message) -> dict: cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), "cloud-init": _cloud_init_secret(), - "gpu-cluster-fabric-2": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "compute.nebius.m.upbound.io/v1beta1", - "kind": "GpuCluster", - "metadata": {"labels": {"modelplane.ai/fabric": "fabric-2"}}, - "spec": { - "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, - "forProvider": { - "name": "test-cluster-fabric-2", - "infinibandFabric": "fabric-2", - }, - }, - } - ), - ), + "gpu-cluster-fabric-2": _gpu_cluster(ready=fnv1.READY_UNSPECIFIED), "nodegroup-system": _nodegroup_system( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), @@ -884,6 +1093,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 8 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -1180,7 +1403,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr(credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Ready"}]), resources={ "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), @@ -1189,71 +1412,8 @@ def _to_dict(msg: message.Message) -> dict: cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE ), "cloud-init": _cloud_init_secret(), - "release-csi-mounted-fs-path": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "helm.m.crossplane.io/v1beta1", - "kind": "Release", - "metadata": { - "namespace": "modelplane-system", - "annotations": {"crossplane.io/external-name": "mp-csi-mounted-fs-path"}, - }, - "spec": { - "managementPolicies": ["Observe", "Create", "Update"], - "providerConfigRef": { - "kind": "ProviderConfig", - "name": "test-cluster-kubeconfig-55b57", - }, - "forProvider": { - "chart": { - "name": "csi-mounted-fs-path", - "repository": "oci://cr.eu-north1.nebius.cloud/mk8s/helm", - "version": "0.1.6", - }, - "namespace": "kube-system", - "values": { - "dataDir": "/mnt/data/csi-mounted-fs-path-data/", - "tolerations": [ - { - "key": "nvidia.com/gpu", - "operator": "Exists", - "effect": "NoSchedule", - }, - ], - }, - }, - }, - } - ), - ready=fnv1.READY_TRUE, - ), - "storage-class-rwx-fs": fnv1.Resource( - resource=resource.dict_to_struct( - { - "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", - "kind": "Object", - "metadata": {"namespace": "modelplane-system"}, - "spec": { - "managementPolicies": ["Observe", "Create", "Update"], - "providerConfigRef": { - "kind": "ProviderConfig", - "name": "test-cluster-kubeconfig-55b57", - }, - "readiness": {"policy": "SuccessfulCreate"}, - "forProvider": { - "manifest": { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "metadata": {"name": "modelplane-rwx-fs"}, - "provisioner": "mounted-fs-path.csi.nebius.ai", - "volumeBindingMode": "WaitForFirstConsumer", - }, - }, - }, - } - ), - ready=fnv1.READY_TRUE, - ), + "release-csi-mounted-fs-path": _csi_release(ready=fnv1.READY_TRUE), + "storage-class-rwx-fs": _storage_class(), "nodegroup-system": _nodegroup_system( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE ), @@ -1270,6 +1430,18 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -1309,12 +1481,16 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "subnet": _subnet_observed( - cred_kind="ProviderConfig", cred_name="my-nebius-account", subnet_id="vpcsubnet-e00test" + cred_kind="ProviderConfig", + cred_name="my-nebius-account", + subnet_id="vpcsubnet-e00test", + ready=False, ), "filesystem": _filesystem_observed( cred_kind="ProviderConfig", cred_name="my-nebius-account", filesystem_id="computefilesystem-e00test", + ready=False, ), }, ), @@ -1331,7 +1507,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr( + credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "network": _network( cred_kind="ProviderConfig", cred_name="my-nebius-account", ready=fnv1.READY_UNSPECIFIED @@ -1362,6 +1540,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 8 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -1457,7 +1649,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr(credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Pending"}]), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -1483,6 +1675,20 @@ def _to_dict(msg: message.Message) -> dict: ), ], context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( @@ -1540,7 +1746,9 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(credentials_secret=True), + composite=_desired_xr( + credentials_secret=True, node_pools=[{"name": "gpu-h100", "state": "Provisioning"}] + ), resources={ "network": _network( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED @@ -1571,6 +1779,759 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 8 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + requirements=fnv1.Requirements( + resources={ + "nebius-provider-config": fnv1.ResourceSelector( + api_version="nebius.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), + Case( + name="StagesClusterProvisioning", + reason="With the mk8s cluster unready, ClusterReady is Provisioning with a count of its ready resources, and NodePoolsReady is Pending on it while no pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-h100", + role="GPU", + platform="gpu-h100-sxm", + preset="8gpu-128vcpu-1600gb", + diskSizeGb=200, + nodeCount=2, + fabric=v1alpha1.Fabric( + type="InfiniBand", + infiniband=v1alpha1.Infiniband(fabric="fabric-2"), + ), + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + platform="cpu-d3", + preset="8vcpu-32gb", + nodeCount=1, + maxNodeCount=4, + ), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "subnet": _subnet_observed( + cred_kind="ClusterProviderConfig", + cred_name="default", + subnet_id="vpcsubnet-e00test", + ready=True, + ), + "filesystem": _filesystem_observed( + cred_kind="ClusterProviderConfig", + cred_name="default", + filesystem_id="computefilesystem-e00test", + ready=True, + ), + "cluster": _observed_managed_resource(ready=False, failure=None), + }, + ), + required_resources={ + "nebius-provider-config": fnv1.Resources( + items=[_nebius_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + credentials_secret=True, + node_pools=[ + {"name": "gpu-h100", "state": "Provisioning"}, + {"name": "cpu-pool", "state": "Provisioning"}, + ], + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "filesystem": _filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cloud-init": _cloud_init_secret(), + "gpu-cluster-fabric-2": _gpu_cluster(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodegroup-gpu-h100": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", + cred_name="default", + autoscaling=None, + fixed_node_count=2, + fabric="fabric-2", + ready=fnv1.READY_UNSPECIFIED, + ), + "nodegroup-cpu-pool": _nodegroup_cpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "release-csi-mounted-fs-path": _csi_release(ready=fnv1.READY_UNSPECIFIED), + "storage-class-rwx-fs": _storage_class(), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="7 of 10 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + requirements=fnv1.Requirements( + resources={ + "nebius-provider-config": fnv1.ResourceSelector( + api_version="nebius.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), + Case( + name="StagesPoolsProvisioning", + reason="With every cluster resource Ready and fabric-2's GpuCluster unready, ClusterReady is True, gpu-h100 is Provisioning though its node group is Ready, cpu-pool is Ready, and NodePoolsReady counts the Ready pools.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-h100", + role="GPU", + platform="gpu-h100-sxm", + preset="8gpu-128vcpu-1600gb", + diskSizeGb=200, + nodeCount=2, + fabric=v1alpha1.Fabric( + type="InfiniBand", + infiniband=v1alpha1.Infiniband(fabric="fabric-2"), + ), + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + platform="cpu-d3", + preset="8vcpu-32gb", + nodeCount=1, + maxNodeCount=4, + ), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "subnet": _subnet_observed( + cred_kind="ClusterProviderConfig", + cred_name="default", + subnet_id="vpcsubnet-e00test", + ready=True, + ), + "cluster": _observed_managed_resource(ready=True, failure=None), + "filesystem": _filesystem_observed( + cred_kind="ClusterProviderConfig", + cred_name="default", + filesystem_id="computefilesystem-e00test", + ready=True, + ), + "gpu-cluster-fabric-2": _observed_managed_resource(ready=False, failure=None), + "nodegroup-system": _observed_managed_resource(ready=True, failure=None), + "nodegroup-gpu-h100": _observed_managed_resource(ready=True, failure=None), + "nodegroup-cpu-pool": _observed_managed_resource(ready=True, failure=None), + "release-csi-mounted-fs-path": _observed_managed_resource(ready=True, failure=None), + }, + ), + required_resources={ + "nebius-provider-config": fnv1.Resources( + items=[_nebius_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + credentials_secret=True, + node_pools=[{"name": "gpu-h100", "state": "Provisioning"}, {"name": "cpu-pool", "state": "Ready"}], + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "filesystem": _filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cloud-init": _cloud_init_secret(), + "gpu-cluster-fabric-2": _gpu_cluster(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-gpu-h100": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", + cred_name="default", + autoscaling=None, + fixed_node_count=2, + fabric="fabric-2", + ready=fnv1.READY_TRUE, + ), + "nodegroup-cpu-pool": _nodegroup_cpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "release-csi-mounted-fs-path": _csi_release(ready=fnv1.READY_TRUE), + "storage-class-rwx-fs": _storage_class(), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 node pools ready", + ), + ], + requirements=fnv1.Requirements( + resources={ + "nebius-provider-config": fnv1.ResourceSelector( + api_version="nebius.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), + Case( + name="StagesAllReady", + reason="With every resource Ready, ClusterReady and NodePoolsReady are True and both pools are Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-h100", + role="GPU", + platform="gpu-h100-sxm", + preset="8gpu-128vcpu-1600gb", + diskSizeGb=200, + nodeCount=2, + fabric=v1alpha1.Fabric( + type="InfiniBand", + infiniband=v1alpha1.Infiniband(fabric="fabric-2"), + ), + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + platform="cpu-d3", + preset="8vcpu-32gb", + nodeCount=1, + maxNodeCount=4, + ), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "subnet": _subnet_observed( + cred_kind="ClusterProviderConfig", + cred_name="default", + subnet_id="vpcsubnet-e00test", + ready=True, + ), + "cluster": _observed_managed_resource(ready=True, failure=None), + "filesystem": _filesystem_observed( + cred_kind="ClusterProviderConfig", + cred_name="default", + filesystem_id="computefilesystem-e00test", + ready=True, + ), + "gpu-cluster-fabric-2": _observed_managed_resource(ready=True, failure=None), + "nodegroup-system": _observed_managed_resource(ready=True, failure=None), + "nodegroup-gpu-h100": _observed_managed_resource(ready=True, failure=None), + "nodegroup-cpu-pool": _observed_managed_resource(ready=True, failure=None), + "release-csi-mounted-fs-path": _observed_managed_resource(ready=True, failure=None), + }, + ), + required_resources={ + "nebius-provider-config": fnv1.Resources( + items=[_nebius_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + credentials_secret=True, + node_pools=[{"name": "gpu-h100", "state": "Ready"}, {"name": "cpu-pool", "state": "Ready"}], + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "filesystem": _filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cloud-init": _cloud_init_secret(), + "gpu-cluster-fabric-2": _gpu_cluster(ready=fnv1.READY_TRUE), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-gpu-h100": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", + cred_name="default", + autoscaling=None, + fixed_node_count=2, + fabric="fabric-2", + ready=fnv1.READY_TRUE, + ), + "nodegroup-cpu-pool": _nodegroup_cpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "release-csi-mounted-fs-path": _csi_release(ready=fnv1.READY_TRUE), + "storage-class-rwx-fs": _storage_class(), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + requirements=fnv1.Requirements( + resources={ + "nebius-provider-config": fnv1.ResourceSelector( + api_version="nebius.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), + Case( + name="StagesPoolFailed", + reason="With the provider reporting fabric-2's GpuCluster create failed, gpu-h100 is Failed with the provider's error prefixed by gpu-cluster-fabric-2, and NodePoolsReady is Failed while ClusterReady is still Provisioning.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-h100", + role="GPU", + platform="gpu-h100-sxm", + preset="8gpu-128vcpu-1600gb", + diskSizeGb=200, + nodeCount=2, + fabric=v1alpha1.Fabric( + type="InfiniBand", + infiniband=v1alpha1.Infiniband(fabric="fabric-2"), + ), + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + platform="cpu-d3", + preset="8vcpu-32gb", + nodeCount=1, + maxNodeCount=4, + ), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "subnet": _subnet_observed( + cred_kind="ClusterProviderConfig", + cred_name="default", + subnet_id="vpcsubnet-e00test", + ready=True, + ), + "filesystem": _filesystem_observed( + cred_kind="ClusterProviderConfig", + cred_name="default", + filesystem_id="computefilesystem-e00test", + ready=True, + ), + "cluster": _observed_managed_resource(ready=False, failure=None), + "gpu-cluster-fabric-2": _observed_managed_resource( + ready=False, + failure="async create failed: RESOURCE_EXHAUSTED: GPU cluster quota exceeded", + ), + }, + ), + required_resources={ + "nebius-provider-config": fnv1.Resources( + items=[_nebius_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + credentials_secret=True, + node_pools=[ + { + "name": "gpu-h100", + "state": "Failed", + "message": "gpu-cluster-fabric-2: async create failed: RESOURCE_EXHAUSTED: GPU cluster quota exceeded", + }, + {"name": "cpu-pool", "state": "Provisioning"}, + ], + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "filesystem": _filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cloud-init": _cloud_init_secret(), + "gpu-cluster-fabric-2": _gpu_cluster(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodegroup-gpu-h100": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", + cred_name="default", + autoscaling=None, + fixed_node_count=2, + fabric="fabric-2", + ready=fnv1.READY_UNSPECIFIED, + ), + "nodegroup-cpu-pool": _nodegroup_cpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "release-csi-mounted-fs-path": _csi_release(ready=fnv1.READY_UNSPECIFIED), + "storage-class-rwx-fs": _storage_class(), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="7 of 10 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 2 node pools failed: gpu-h100", + ), + ], + requirements=fnv1.Requirements( + resources={ + "nebius-provider-config": fnv1.ResourceSelector( + api_version="nebius.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), + Case( + name="StagesClusterFailed", + reason="With the provider reporting the mk8s cluster and system node group creates failed, ClusterReady is Failed with the cluster's error and a count of the other failure, and NodePoolsReady is Pending on it.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-h100", + role="GPU", + platform="gpu-h100-sxm", + preset="8gpu-128vcpu-1600gb", + diskSizeGb=200, + nodeCount=2, + fabric=v1alpha1.Fabric( + type="InfiniBand", + infiniband=v1alpha1.Infiniband(fabric="fabric-2"), + ), + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + platform="cpu-d3", + preset="8vcpu-32gb", + nodeCount=1, + maxNodeCount=4, + ), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "subnet": _subnet_observed( + cred_kind="ClusterProviderConfig", + cred_name="default", + subnet_id="vpcsubnet-e00test", + ready=True, + ), + "filesystem": _filesystem_observed( + cred_kind="ClusterProviderConfig", + cred_name="default", + filesystem_id="computefilesystem-e00test", + ready=True, + ), + "cluster": _observed_managed_resource( + ready=False, + failure="async create failed: INVALID_ARGUMENT: unsupported Kubernetes version", + ), + "nodegroup-system": _observed_managed_resource( + ready=False, + failure="async create failed: RESOURCE_EXHAUSTED: cpu-d3 quota exceeded", + ), + }, + ), + required_resources={ + "nebius-provider-config": fnv1.Resources( + items=[_nebius_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + credentials_secret=True, + node_pools=[ + {"name": "gpu-h100", "state": "Provisioning"}, + {"name": "cpu-pool", "state": "Provisioning"}, + ], + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "filesystem": _filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cloud-init": _cloud_init_secret(), + "gpu-cluster-fabric-2": _gpu_cluster(ready=fnv1.READY_UNSPECIFIED), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "nodegroup-gpu-h100": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", + cred_name="default", + autoscaling=None, + fixed_node_count=2, + fabric="fabric-2", + ready=fnv1.READY_UNSPECIFIED, + ), + "nodegroup-cpu-pool": _nodegroup_cpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "release-csi-mounted-fs-path": _csi_release(ready=fnv1.READY_UNSPECIFIED), + "storage-class-rwx-fs": _storage_class(), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="cluster: async create failed: INVALID_ARGUMENT: unsupported Kubernetes version (and 1 more)", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + requirements=fnv1.Requirements( + resources={ + "nebius-provider-config": fnv1.ResourceSelector( + api_version="nebius.m.upbound.io/v1beta1", + kind="ClusterProviderConfig", + match_name="default", + ), + }, + ), + ), + ), + Case( + name="ClusterUnreadyPoolsReady", + reason="With the mk8s cluster gone unready while both pools' resources are Ready, ClusterReady is Provisioning and NodePoolsReady reports the pools' own state, True, not Pending.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-h100", + role="GPU", + platform="gpu-h100-sxm", + preset="8gpu-128vcpu-1600gb", + diskSizeGb=200, + nodeCount=2, + fabric=v1alpha1.Fabric( + type="InfiniBand", + infiniband=v1alpha1.Infiniband(fabric="fabric-2"), + ), + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + v1alpha1.NodePool( + name="cpu-pool", + role="CPU", + platform="cpu-d3", + preset="8vcpu-32gb", + nodeCount=1, + maxNodeCount=4, + ), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "subnet": _subnet_observed( + cred_kind="ClusterProviderConfig", + cred_name="default", + subnet_id="vpcsubnet-e00test", + ready=True, + ), + "cluster": _observed_managed_resource(ready=False, failure=None), + "filesystem": _filesystem_observed( + cred_kind="ClusterProviderConfig", + cred_name="default", + filesystem_id="computefilesystem-e00test", + ready=True, + ), + "gpu-cluster-fabric-2": _observed_managed_resource(ready=True, failure=None), + "nodegroup-system": _observed_managed_resource(ready=True, failure=None), + "nodegroup-gpu-h100": _observed_managed_resource(ready=True, failure=None), + "nodegroup-cpu-pool": _observed_managed_resource(ready=True, failure=None), + "release-csi-mounted-fs-path": _observed_managed_resource(ready=True, failure=None), + }, + ), + required_resources={ + "nebius-provider-config": fnv1.Resources( + items=[_nebius_provider_config(kind="ClusterProviderConfig", name="default", namespace=None)], + ), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + credentials_secret=True, + node_pools=[{"name": "gpu-h100", "state": "Ready"}, {"name": "cpu-pool", "state": "Ready"}], + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "subnet": _subnet(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "filesystem": _filesystem( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "cloud-init": _cloud_init_secret(), + "gpu-cluster-fabric-2": _gpu_cluster(ready=fnv1.READY_TRUE), + "nodegroup-system": _nodegroup_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "nodegroup-gpu-h100": _nodegroup_gpu( + cred_kind="ClusterProviderConfig", + cred_name="default", + autoscaling=None, + fixed_node_count=2, + fabric="fabric-2", + ready=fnv1.READY_TRUE, + ), + "nodegroup-cpu-pool": _nodegroup_cpu( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "release-csi-mounted-fs-path": _csi_release(ready=fnv1.READY_TRUE), + "storage-class-rwx-fs": _storage_class(), + "provider-config-kubernetes": _provider_config(api_version="kubernetes.m.crossplane.io/v1alpha1"), + "provider-config-helm": _provider_config(api_version="helm.m.crossplane.io/v1beta1"), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="9 of 10 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], requirements=fnv1.Requirements( resources={ "nebius-provider-config": fnv1.ResourceSelector( diff --git a/functions/compose-upcloud-cluster/function/fn.py b/functions/compose-upcloud-cluster/function/fn.py index 762fc9226..289cfcbbe 100644 --- a/functions/compose-upcloud-cluster/function/fn.py +++ b/functions/compose-upcloud-cluster/function/fn.py @@ -189,6 +189,38 @@ # nodeCount into forProvider, defeating the purpose.) _AUTOSCALED_POOL_MANAGEMENT: list[_ManagementPolicy] = ["Observe", "Create", "Update", "Delete"] +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: the network, +# the UKS cluster, the system node group, the ProviderConfigs, the token +# Secret's Object and the cluster autoscaler. compose-inference-cluster relays +# both conditions onto the InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + + +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + def _name(meta: metav1.ObjectMeta | None) -> str: """The object's name, always set on resources read from the API server.""" @@ -267,8 +299,10 @@ def compose(self) -> None: if autoscaler: self.compose_autoscaler_secret(autoscaler) self.compose_cluster_autoscaler() - self.write_status() self.mark_readiness() + node_pools = self.node_pool_statuses() + self.derive_conditions(node_pools) + self.write_status(node_pools) def _cluster_ready(self) -> bool: return resource.get_condition(self.req.observed.resources.get("cluster"), "Ready").status == "True" @@ -649,8 +683,9 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> nodegroupv1alpha1.KubernetesNod spec=spec, ) - def write_status(self) -> None: + def write_status(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: status = v1alpha1.Status( + nodePools=node_pools, secrets=[ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -676,3 +711,108 @@ def mark_readiness(self) -> None: continue if resource.get_condition(self.req.observed.resources.get(r), "Ready").status == "True": self.rsp.desired.resources[r].ready = fnv1.READY_TRUE + + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool. The autoscaler release + scales every autoscaled pool, so it's part of the cluster instead.""" + return [f"node-pool-{pool.name}"] + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def _stage_pools(self) -> list[v1alpha1.NodePool]: + """The node pools NodePoolsReady and status.nodePools report. A System + pool runs the cluster's controllers and serving stack, so it's part of + ClusterReady.""" + return [pool for pool in self.xr.spec.nodePools if pool.role != "System"] + + def node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each node pool, from its resources.""" + statuses = [] + for pool in self._stage_pools(): + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: + """Derive ClusterReady from the readiness mark_readiness set, and + NodePoolsReady from node_pools. + + NodePoolsReady waits on ClusterReady only while no pool is Ready, so on + a running cluster the pools report their own state while the cluster + or a new pool provisions. A failed pool is reported Failed even while + the cluster is still provisioning. + """ + in_pools = {r for pool in self._stage_pools() for r in self.pool_resources(pool)} + cluster = sorted(r for r in self.rsp.desired.resources if r not in in_pools) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True" and not ready: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) diff --git a/functions/compose-upcloud-cluster/tests/test_fn.py b/functions/compose-upcloud-cluster/tests/test_fn.py index fcdf9004c..35b2caa08 100644 --- a/functions/compose-upcloud-cluster/tests/test_fn.py +++ b/functions/compose-upcloud-cluster/tests/test_fn.py @@ -62,9 +62,10 @@ def _xr( return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr() -> fnv1.Resource: - """The desired XR, publishing the cluster's kubeconfig Secret.""" +def _desired_xr(*, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR, publishing node_pools' states and the cluster's kubeconfig Secret.""" status: dict[str, Any] = { + "nodePools": node_pools, "secrets": [ { "type": "Kubeconfig", @@ -76,7 +77,7 @@ def _desired_xr() -> fnv1.Resource: return fnv1.Resource(resource=resource.dict_to_struct({"status": status})) -def _network(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _network(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed Network, a dedicated SDN network for the cluster.""" return fnv1.Resource( resource=resource.dict_to_struct( @@ -98,6 +99,7 @@ def _network(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) @@ -165,7 +167,7 @@ def _observed_cluster(*, cred_kind: str, cred_name: str, ready: bool) -> fnv1.Re ) -def _node_pool_system(*, cred_kind: str, cred_name: str) -> fnv1.Resource: +def _node_pool_system(*, cred_kind: str, cred_name: str, ready: fnv1.Ready) -> fnv1.Resource: """The composed KubernetesNodeGroup for the system pool the function injects. The external-name annotation names the UKS node group, since the node @@ -190,6 +192,7 @@ def _node_pool_system(*, cred_kind: str, cred_name: str) -> fnv1.Resource: }, } ), + ready=ready, ) @@ -420,6 +423,30 @@ def _autoscaler_release(*, autoscaling_groups: list[dict]) -> fnv1.Resource: ) +def _observed_managed_resource(*, ready: bool, failure: str | None) -> fnv1.Resource: + """An observed managed resource whose Ready condition is True if ready and False if not, and whose + LastAsyncOperation condition is False with failure as its message unless failure is None.""" + conditions = [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Creating", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ] + if failure is not None: + conditions.append( + { + "type": "LastAsyncOperation", + "status": "False", + "reason": "AsyncCreateFailure", + "message": failure, + "lastTransitionTime": "2024-01-01T00:00:00Z", + } + ) + return fnv1.Resource(resource=resource.dict_to_struct({"status": {"conditions": conditions}})) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -449,15 +476,31 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Pending"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 2 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -486,11 +529,15 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), - "node-pool-system": _node_pool_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "node-pool-system": _node_pool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "node-pool-gpu-l40s": _node_pool_gpu( management_policies=["Observe", "Create", "Update", "Delete"], init_node_count=1, @@ -509,6 +556,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The observed ProviderConfig shows the dependents were composed before. @@ -572,13 +633,17 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "node-pool-system": _node_pool_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "node-pool-system": _node_pool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "node-pool-gpu-l40s": _node_pool_gpu( management_policies=["Observe", "Create", "Update", "Delete"], init_node_count=1, @@ -597,6 +662,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -627,11 +706,15 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), - "node-pool-system": _node_pool_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "node-pool-system": _node_pool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "node-pool-gpu-l40s": _node_pool_gpu( management_policies=["Observe", "Create", "Update", "Delete"], init_node_count=2, @@ -650,6 +733,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The fork lists every node group in the cluster and scales any group @@ -689,11 +786,20 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr( + node_pools=[ + {"name": "gpu-l40s", "state": "Provisioning"}, + {"name": "cpu-pool", "state": "Provisioning"}, + ] + ), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), - "node-pool-system": _node_pool_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "node-pool-system": _node_pool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "node-pool-gpu-l40s": _node_pool_gpu( management_policies=["Observe", "Create", "Update", "Delete"], init_node_count=1, @@ -717,6 +823,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 7 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # Without spec.autoscaler nothing needs to reach into the cluster, so no @@ -747,11 +867,15 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), - "node-pool-system": _node_pool_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "node-pool-system": _node_pool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "node-pool-gpu-l40s": _node_pool_gpu( management_policies=None, init_node_count=None, @@ -761,6 +885,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 3 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The node groups are the only sentinel on a cluster without an @@ -788,20 +926,26 @@ def _to_dict(msg: message.Message) -> dict: ), resources={ "cluster": _observed_cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=False), - "node-pool-system": _node_pool_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "node-pool-system": _node_pool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), }, ), ), want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster( cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED ), - "node-pool-system": _node_pool_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "node-pool-system": _node_pool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "node-pool-gpu-l40s": _node_pool_gpu( management_policies=None, init_node_count=None, @@ -811,6 +955,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 3 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -841,11 +999,15 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "cpu-pool", "state": "Provisioning"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), - "node-pool-system": _node_pool_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "node-pool-system": _node_pool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "node-pool-cpu-pool": _node_pool_cpu( node_count=2, ready=fnv1.READY_UNSPECIFIED, @@ -853,6 +1015,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 3 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), # The release excludes Delete, so dropping it from desired state when @@ -894,11 +1070,15 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ - "network": _network(cred_kind="ClusterProviderConfig", cred_name="default"), + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), - "node-pool-system": _node_pool_system(cred_kind="ClusterProviderConfig", cred_name="default"), + "node-pool-system": _node_pool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), "node-pool-gpu-l40s": _node_pool_gpu( management_policies=None, init_node_count=None, @@ -911,11 +1091,25 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="3 of 6 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( name="CustomCredentials", - reason="An UpCloudCluster naming its own ProviderConfig gets it on every UpCloud resource, and its System-role pool gets no GPU label or taint.", + reason="An UpCloudCluster naming its own ProviderConfig gets it on every UpCloud resource, and its System-role pool gets no GPU label or taint, counts under ClusterReady, and is left out of status.nodePools.", req=fnv1.RunFunctionRequest( observed=fnv1.State( composite=_xr( @@ -938,11 +1132,13 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[]), resources={ - "network": _network(cred_kind="ProviderConfig", cred_name="team-a"), + "network": _network(cred_kind="ProviderConfig", cred_name="team-a", ready=fnv1.READY_UNSPECIFIED), "cluster": _cluster(cred_kind="ProviderConfig", cred_name="team-a", ready=fnv1.READY_TRUE), - "node-pool-system": _node_pool_system(cred_kind="ProviderConfig", cred_name="team-a"), + "node-pool-system": _node_pool_system( + cred_kind="ProviderConfig", cred_name="team-a", ready=fnv1.READY_UNSPECIFIED + ), "node-pool-workers": fnv1.Resource( resource=resource.dict_to_struct( { @@ -966,6 +1162,347 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 4 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + Case( + name="PoolsProvisioning", + reason="With every cluster resource observed Ready and the cpu-pool node group unready, ClusterReady is True, NodePoolsReady is Provisioning, and only gpu-l40s is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + autoscaler=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="GPU-8xCPU-64GB-1xL40S", + nodeCount=2, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="PREMIUM-8xCPU-32GB", nodeCount=2), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "cluster": _observed_cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=True), + "node-pool-system": _observed_managed_resource(ready=True, failure=None), + "node-pool-gpu-l40s": _observed_managed_resource(ready=True, failure=None), + "node-pool-cpu-pool": _observed_managed_resource(ready=False, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Ready"}, {"name": "cpu-pool", "state": "Provisioning"}] + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "node-pool-system": _node_pool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "node-pool-gpu-l40s": _node_pool_gpu( + management_policies=None, + init_node_count=None, + node_count=2, + ready=fnv1.READY_TRUE, + ), + "node-pool-cpu-pool": _node_pool_cpu(node_count=2, ready=fnv1.READY_UNSPECIFIED), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 node pools ready", + ), + ], + ), + ), + Case( + name="AllReady", + reason="With every resource observed Ready, ClusterReady and NodePoolsReady are True and every pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + autoscaler=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="GPU-8xCPU-64GB-1xL40S", + nodeCount=2, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="PREMIUM-8xCPU-32GB", nodeCount=2), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "cluster": _observed_cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=True), + "node-pool-system": _observed_managed_resource(ready=True, failure=None), + "node-pool-gpu-l40s": _observed_managed_resource(ready=True, failure=None), + "node-pool-cpu-pool": _observed_managed_resource(ready=True, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Ready"}, {"name": "cpu-pool", "state": "Ready"}] + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "node-pool-system": _node_pool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "node-pool-gpu-l40s": _node_pool_gpu( + management_policies=None, + init_node_count=None, + node_count=2, + ready=fnv1.READY_TRUE, + ), + "node-pool-cpu-pool": _node_pool_cpu(node_count=2, ready=fnv1.READY_TRUE), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + ), + ), + Case( + name="PoolFailed", + reason="With the provider reporting the gpu-l40s node group's LastAsyncOperation False, that pool is Failed with the provider's error prefixed by the resource name, and NodePoolsReady is Failed naming it.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + autoscaler=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="GPU-8xCPU-64GB-1xL40S", + nodeCount=2, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="PREMIUM-8xCPU-32GB", nodeCount=2), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "cluster": _observed_cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=True), + "node-pool-system": _observed_managed_resource(ready=True, failure=None), + "node-pool-gpu-l40s": _observed_managed_resource( + ready=False, failure="creating node group: quota exceeded" + ), + "node-pool-cpu-pool": _observed_managed_resource(ready=True, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[ + { + "name": "gpu-l40s", + "state": "Failed", + "message": "node-pool-gpu-l40s: creating node group: quota exceeded", + }, + {"name": "cpu-pool", "state": "Ready"}, + ] + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "node-pool-system": _node_pool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "node-pool-gpu-l40s": _node_pool_gpu( + management_policies=None, + init_node_count=None, + node_count=2, + ready=fnv1.READY_UNSPECIFIED, + ), + "node-pool-cpu-pool": _node_pool_cpu(node_count=2, ready=fnv1.READY_TRUE), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 2 node pools failed: gpu-l40s", + ), + ], + ), + ), + Case( + name="ClusterFailed", + reason="With the provider reporting the network's and cluster's LastAsyncOperation False, ClusterReady is Failed with the first failed resource's error and a count of the rest, NodePoolsReady is Pending on it, and each pool is Pending because its node group isn't composed.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + autoscaler=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="GPU-8xCPU-64GB-1xL40S", + nodeCount=2, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="PREMIUM-8xCPU-32GB", nodeCount=2), + ], + ), + resources={ + "network": _observed_managed_resource(ready=False, failure="creating network: zone unavailable"), + "cluster": _observed_managed_resource(ready=False, failure="creating cluster: quota exceeded"), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Pending"}, {"name": "cpu-pool", "state": "Pending"}] + ), + resources={ + "network": _network( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="cluster: creating cluster: quota exceeded (and 1 more)", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + Case( + name="ClusterRegressedPoolsReady", + reason="With the cluster observed unready after its pools came up, ClusterReady is Provisioning and NodePoolsReady reports the pools' own state, True, rather than Pending on ClusterReady.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + credentials=None, + autoscaler=None, + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="GPU-8xCPU-64GB-1xL40S", + nodeCount=2, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="PREMIUM-8xCPU-32GB", nodeCount=2), + ], + ), + resources={ + "network": _observed_managed_resource(ready=True, failure=None), + "cluster": _observed_cluster(cred_kind="ClusterProviderConfig", cred_name="default", ready=False), + "node-pool-system": _observed_managed_resource(ready=True, failure=None), + "node-pool-gpu-l40s": _observed_managed_resource(ready=True, failure=None), + "node-pool-cpu-pool": _observed_managed_resource(ready=True, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Ready"}, {"name": "cpu-pool", "state": "Ready"}] + ), + resources={ + "network": _network(cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE), + "cluster": _cluster( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_UNSPECIFIED + ), + "node-pool-system": _node_pool_system( + cred_kind="ClusterProviderConfig", cred_name="default", ready=fnv1.READY_TRUE + ), + "node-pool-gpu-l40s": _node_pool_gpu( + management_policies=None, + init_node_count=None, + node_count=2, + ready=fnv1.READY_TRUE, + ), + "node-pool-cpu-pool": _node_pool_cpu(node_count=2, ready=fnv1.READY_TRUE), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 3 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], ), ), ] diff --git a/functions/compose-vultr-cluster/function/fn.py b/functions/compose-vultr-cluster/function/fn.py index 3d9105bac..b981c44cb 100644 --- a/functions/compose-vultr-cluster/function/fn.py +++ b/functions/compose-vultr-cluster/function/fn.py @@ -110,6 +110,37 @@ " && object.status.numberReady == object.status.desiredNumberScheduled" ) +# Conditions reporting which provisioning stage the cluster is in. The cluster +# is everything this XR composes except the user's node pools: the VKE cluster +# with its inline system node pool, and the provider-kubernetes ProviderConfig. +# compose-inference-cluster relays both conditions onto the InferenceCluster. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_TYPE_NODE_POOLS_READY = "NodePoolsReady" + +CONDITION_REASON_AVAILABLE = "Available" +CONDITION_REASON_PENDING = "Pending" +CONDITION_REASON_PROVISIONING = "Provisioning" +CONDITION_REASON_FAILED = "Failed" + +# status.nodePools[].state values. +_POOL_STATE_PENDING = "Pending" +_POOL_STATE_PROVISIONING = "Provisioning" +_POOL_STATE_READY = "Ready" +_POOL_STATE_FAILED = "Failed" + +# upjet sets this condition False, with the provider's error as its message, +# when its last create, update or delete of the external resource failed. It's +# the only failure signal read: a managed resource is also Synced=False while it +# waits for a reference to resolve, which is routine during provisioning. +_CONDITION_LAST_ASYNC_OPERATION = "LastAsyncOperation" + + +def _first(failures: list[str]) -> str: + """The first failure, noting how many more there are.""" + if len(failures) == 1: + return failures[0] + return f"{failures[0]} (and {len(failures) - 1} more)" + def _name(meta: metav1.ObjectMeta | None) -> str: """The object's name, always set on resources read from the API server.""" @@ -175,8 +206,10 @@ def compose(self) -> None: # hold the cluster un-Ready forever. if any(pool.role == "GPU" for pool in self.xr.spec.nodePools): self.compose_gpu_observer() - self.write_status() self.mark_readiness() + node_pools = self.node_pool_statuses() + self.derive_conditions(node_pools) + self.write_status(node_pools) def _cluster_ready(self) -> bool: return resource.get_condition(self.req.observed.resources.get("cluster"), "Ready").status == "True" @@ -350,8 +383,9 @@ def _node_pool(self, pool: v1alpha1.NodePool) -> vkenodepoolv1beta1.KubernetesNo ), ) - def write_status(self) -> None: + def write_status(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: status = v1alpha1.Status( + nodePools=node_pools, secrets=[ v1alpha1.Secret( type=_SECRET_TYPE_KUBECONFIG, @@ -378,3 +412,113 @@ def mark_readiness(self) -> None: continue if resource.get_condition(self.req.observed.resources.get(r), "Ready").status == "True": self.rsp.desired.resources[r].ready = fnv1.READY_TRUE + + def pool_resources(self, pool: v1alpha1.NodePool) -> list[str]: + """The composed resources that build pool. Every GPU pool includes the + gpu-observer, which stays unready until the NVIDIA operator validator + has rolled out on the GPU nodes, so counting it toward the cluster + would hold ClusterReady on the GPU pools.""" + resources = [f"node-pool-{pool.name}"] + if pool.role == "GPU": + resources.append("gpu-observer") + return resources + + def _failures(self, resources: list[str]) -> list[str]: + """Each composed resource whose provider reports its last operation + failed, as its name and the provider's error.""" + failures = [] + for r in resources: + c = resource.get_condition(self.req.observed.resources.get(r), _CONDITION_LAST_ASYNC_OPERATION) + if c.status == "False": + failures.append(f"{r}: {c.message}") + return failures + + def _unready(self, resources: list[str]) -> list[str]: + """Each of resources that isn't composed and marked ready.""" + desired = self.rsp.desired.resources + return [r for r in resources if r not in desired or desired[r].ready != fnv1.READY_TRUE] + + def _stage_pools(self) -> list[v1alpha1.NodePool]: + """The node pools NodePoolsReady and status.nodePools report. A System + pool runs the cluster's controllers and serving stack, so it's part of + ClusterReady.""" + return [pool for pool in self.xr.spec.nodePools if pool.role != "System"] + + def node_pool_statuses(self) -> list[v1alpha1.NodePoolModel]: + """The provisioning state of each node pool, from its resources.""" + statuses = [] + for pool in self._stage_pools(): + resources = self.pool_resources(pool) + failures = self._failures(resources) + if failures: + statuses.append( + v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_FAILED, message=_first(failures)) + ) + elif not any(r in self.rsp.desired.resources for r in resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PENDING)) + elif self._unready(resources): + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_PROVISIONING)) + else: + statuses.append(v1alpha1.NodePoolModel(name=pool.name, state=_POOL_STATE_READY)) + return statuses + + def derive_conditions(self, node_pools: list[v1alpha1.NodePoolModel]) -> None: + """Derive ClusterReady from the readiness mark_readiness set, and + NodePoolsReady from node_pools. + + NodePoolsReady waits on ClusterReady only while no pool is Ready, so on + a running cluster the pools report their own state while the cluster + or a new pool provisions. A failed pool is reported Failed even while + the cluster is still provisioning. + """ + in_pools = {r for pool in self._stage_pools() for r in self.pool_resources(pool)} + cluster = sorted(r for r in self.rsp.desired.resources if r not in in_pools) + failures = self._failures(cluster) + unready = self._unready(cluster) + if failures: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=_first(failures), + ) + elif unready: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(cluster) - len(unready)} of {len(cluster)} resources ready", + ) + else: + cluster_ready = resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + + failed = [p.name for p in node_pools if p.state == _POOL_STATE_FAILED] + ready = [p.name for p in node_pools if p.state == _POOL_STATE_READY] + if failed: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_FAILED, + message=f"{len(failed)} of {len(node_pools)} node pools failed: {', '.join(failed)}", + ) + elif cluster_ready.status != "True" and not ready: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PENDING, + message=f"Waiting for {CONDITION_TYPE_CLUSTER_READY}", + ) + elif len(ready) < len(node_pools): + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, + status="False", + reason=CONDITION_REASON_PROVISIONING, + message=f"{len(ready)} of {len(node_pools)} node pools ready", + ) + else: + pools_ready = resource.Condition( + typ=CONDITION_TYPE_NODE_POOLS_READY, status="True", reason=CONDITION_REASON_AVAILABLE + ) + response.set_conditions(self.rsp, cluster_ready, pools_ready) diff --git a/functions/compose-vultr-cluster/tests/test_fn.py b/functions/compose-vultr-cluster/tests/test_fn.py index 9cb6e416d..49303f36f 100644 --- a/functions/compose-vultr-cluster/tests/test_fn.py +++ b/functions/compose-vultr-cluster/tests/test_fn.py @@ -56,12 +56,13 @@ def _xr(*, node_pools: list[v1alpha1.NodePool]) -> fnv1.Resource: return fnv1.Resource(resource=resource.dict_to_struct(xr.model_dump(exclude_none=True, mode="json", by_alias=True))) -def _desired_xr() -> fnv1.Resource: - """The desired XR, publishing the cluster's kubeconfig Secret.""" +def _desired_xr(*, node_pools: list[dict]) -> fnv1.Resource: + """The desired XR, publishing node_pools' states and the cluster's kubeconfig Secret.""" return fnv1.Resource( resource=resource.dict_to_struct( { "status": { + "nodePools": node_pools, "secrets": [ { "type": "Kubeconfig", @@ -276,6 +277,30 @@ def _gpu_observer(*, ready: fnv1.Ready) -> fnv1.Resource: ) +def _observed_managed_resource(*, ready: bool, failure: str | None) -> fnv1.Resource: + """An observed managed resource whose Ready condition is True if ready and False if not, and whose + LastAsyncOperation condition is False with failure as its message unless failure is None.""" + conditions = [ + { + "type": "Ready", + "status": "True" if ready else "False", + "reason": "Available" if ready else "Creating", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ] + if failure is not None: + conditions.append( + { + "type": "LastAsyncOperation", + "status": "False", + "reason": "AsyncCreateFailure", + "message": failure, + "lastTransitionTime": "2024-01-01T00:00:00Z", + } + ) + return fnv1.Resource(resource=resource.dict_to_struct({"status": {"conditions": conditions}})) + + def _to_dict(msg: message.Message) -> dict: """msg as a dict with sorted keys, so pytest's diff of two lines them up.""" return json.loads(json_format.MessageToJson(msg, sort_keys=True)) @@ -303,12 +328,26 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Pending"}]), resources={ "cluster": _cluster(ready=fnv1.READY_UNSPECIFIED), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -335,7 +374,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ "cluster": _cluster(ready=fnv1.READY_TRUE), "node-pool-gpu-l40s": _node_pool_gpu( @@ -348,6 +387,19 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 node pools ready", + ), + ], ), ), Case( @@ -377,7 +429,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "cpu-pool", "state": "Provisioning"}]), resources={ "cluster": _cluster(ready=fnv1.READY_TRUE), "node-pool-cpu-pool": _node_pool_cpu( @@ -389,6 +441,19 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 node pools ready", + ), + ], ), ), Case( @@ -447,7 +512,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ "cluster": _cluster(ready=fnv1.READY_UNSPECIFIED), "node-pool-gpu-l40s": _node_pool_gpu( @@ -460,6 +525,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -522,7 +601,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ "cluster": _cluster(ready=fnv1.READY_UNSPECIFIED), "node-pool-gpu-l40s": _node_pool_gpu( @@ -535,6 +614,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -561,7 +654,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}]), resources={ "cluster": _cluster(ready=fnv1.READY_TRUE), "node-pool-gpu-l40s": _node_pool_gpu( @@ -574,6 +667,19 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 node pools ready", + ), + ], ), ), # The pool's nodeCount and minNodeCount are both 2, so this case can't tell @@ -581,8 +687,9 @@ def _to_dict(msg: message.Message) -> dict: Case( name="SystemRolePool", reason=( - "A VultrCluster's System-role pool autoscales from a floor of 2 nodes to maxNodeCount, and gets " - "neither GPU labels nor a taint - and with no GPU pool there is no GPU observer to compose." + "A VultrCluster's System-role pool autoscales from a floor of 2 nodes to maxNodeCount, gets neither " + "GPU labels nor a taint, counts under ClusterReady, and is left out of status.nodePools - and with no " + "GPU pool there is no GPU observer to compose." ), req=fnv1.RunFunctionRequest( observed=fnv1.State( @@ -606,7 +713,7 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[]), resources={ "cluster": _cluster(ready=fnv1.READY_TRUE), "node-pool-workers": fnv1.Resource( @@ -634,6 +741,20 @@ def _to_dict(msg: message.Message) -> dict: }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="2 of 3 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], ), ), Case( @@ -740,7 +861,247 @@ def _to_dict(msg: message.Message) -> dict: want=fnv1.RunFunctionResponse( meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), desired=fnv1.State( - composite=_desired_xr(), + composite=_desired_xr(node_pools=[{"name": "gpu-l40s", "state": "Ready"}]), + resources={ + "cluster": _cluster(ready=fnv1.READY_TRUE), + "node-pool-gpu-l40s": _node_pool_gpu( + node_quantity=1, + autoscaling={"autoScaler": True, "minNodes": 1, "maxNodes": 4}, + ready=fnv1.READY_TRUE, + ), + "provider-config-kubernetes": _provider_config(), + "gpu-observer": _gpu_observer(ready=fnv1.READY_TRUE), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + ), + ), + Case( + name="ClusterProvisioning", + reason="With the cluster observed unready, ClusterReady counts the 0 of 1 resources ready, NodePoolsReady is Pending on it, and each pool is Pending because its KubernetesNodePool isn't composed.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="vcg-l40s-16c-180g-48vram", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="vc2-6c-16gb", maxNodeCount=4), + ], + ), + resources={ + "cluster": _observed_cluster(ready=False), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Pending"}, {"name": "cpu-pool", "state": "Pending"}] + ), + resources={ + "cluster": _cluster(ready=fnv1.READY_UNSPECIFIED), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="0 of 1 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + Case( + name="PoolsProvisioning", + reason="With the cluster and both KubernetesNodePools observed Ready but the gpu-observer unready, gpu-l40s is Provisioning because the gpu-observer counts toward it, cpu-pool is Ready, and NodePoolsReady is Provisioning.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="vcg-l40s-16c-180g-48vram", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="vc2-6c-16gb", maxNodeCount=4), + ], + ), + resources={ + "cluster": _observed_cluster(ready=True), + "node-pool-gpu-l40s": _observed_managed_resource(ready=True, failure=None), + "node-pool-cpu-pool": _observed_managed_resource(ready=True, failure=None), + "gpu-observer": _observed_managed_resource(ready=False, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Provisioning"}, {"name": "cpu-pool", "state": "Ready"}] + ), + resources={ + "cluster": _cluster(ready=fnv1.READY_TRUE), + "node-pool-gpu-l40s": _node_pool_gpu( + node_quantity=1, + autoscaling={"autoScaler": True, "minNodes": 1, "maxNodes": 4}, + ready=fnv1.READY_TRUE, + ), + "node-pool-cpu-pool": _node_pool_cpu( + node_quantity=1, + autoscaling={"autoScaler": True, "minNodes": 1, "maxNodes": 4}, + ready=fnv1.READY_TRUE, + ), + "provider-config-kubernetes": _provider_config(), + "gpu-observer": _gpu_observer(ready=fnv1.READY_UNSPECIFIED), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 node pools ready", + ), + ], + ), + ), + Case( + name="AllReady", + reason="With every resource observed Ready, ClusterReady and NodePoolsReady are True and every pool is Ready.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="vcg-l40s-16c-180g-48vram", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="vc2-6c-16gb", maxNodeCount=4), + ], + ), + resources={ + "cluster": _observed_cluster(ready=True), + "node-pool-gpu-l40s": _observed_managed_resource(ready=True, failure=None), + "node-pool-cpu-pool": _observed_managed_resource(ready=True, failure=None), + "gpu-observer": _observed_managed_resource(ready=True, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Ready"}, {"name": "cpu-pool", "state": "Ready"}] + ), + resources={ + "cluster": _cluster(ready=fnv1.READY_TRUE), + "node-pool-gpu-l40s": _node_pool_gpu( + node_quantity=1, + autoscaling={"autoScaler": True, "minNodes": 1, "maxNodes": 4}, + ready=fnv1.READY_TRUE, + ), + "node-pool-cpu-pool": _node_pool_cpu( + node_quantity=1, + autoscaling={"autoScaler": True, "minNodes": 1, "maxNodes": 4}, + ready=fnv1.READY_TRUE, + ), + "provider-config-kubernetes": _provider_config(), + "gpu-observer": _gpu_observer(ready=fnv1.READY_TRUE), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + ), + ), + Case( + name="PoolFailed", + reason="With the provider reporting the cpu-pool KubernetesNodePool's LastAsyncOperation False, that pool is Failed with the provider's error prefixed by the resource name, and NodePoolsReady is Failed naming it.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="vcg-l40s-16c-180g-48vram", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="vc2-6c-16gb", maxNodeCount=4), + ], + ), + resources={ + "cluster": _observed_cluster(ready=True), + "node-pool-gpu-l40s": _observed_managed_resource(ready=True, failure=None), + "node-pool-cpu-pool": _observed_managed_resource( + ready=False, failure="async create failed: Plan is not available in the selected region" + ), + "gpu-observer": _observed_managed_resource(ready=True, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[ + {"name": "gpu-l40s", "state": "Ready"}, + { + "name": "cpu-pool", + "state": "Failed", + "message": "node-pool-cpu-pool: async create failed: Plan is not available in the selected region", + }, + ] + ), resources={ "cluster": _cluster(ready=fnv1.READY_TRUE), "node-pool-gpu-l40s": _node_pool_gpu( @@ -748,11 +1109,143 @@ def _to_dict(msg: message.Message) -> dict: autoscaling={"autoScaler": True, "minNodes": 1, "maxNodes": 4}, ready=fnv1.READY_TRUE, ), + "node-pool-cpu-pool": _node_pool_cpu( + node_quantity=1, + autoscaling={"autoScaler": True, "minNodes": 1, "maxNodes": 4}, + ready=fnv1.READY_UNSPECIFIED, + ), + "provider-config-kubernetes": _provider_config(), + "gpu-observer": _gpu_observer(ready=fnv1.READY_TRUE), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="1 of 2 node pools failed: cpu-pool", + ), + ], + ), + ), + Case( + name="ClusterFailed", + reason="With the provider reporting the cluster's LastAsyncOperation False, ClusterReady is Failed with its error and NodePoolsReady is Pending on it.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="vcg-l40s-16c-180g-48vram", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="vc2-6c-16gb", maxNodeCount=4), + ], + ), + resources={ + "cluster": _observed_managed_resource( + ready=False, failure="async create failed: Region is not available" + ), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Pending"}, {"name": "cpu-pool", "state": "Pending"}] + ), + resources={ + "cluster": _cluster(ready=fnv1.READY_UNSPECIFIED), + }, + ), + context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Failed", + message="cluster: async create failed: Region is not available", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Pending", + message="Waiting for ClusterReady", + ), + ], + ), + ), + Case( + name="ClusterRegressedPoolsReady", + reason="With the cluster observed unready after its pools came up, ClusterReady is Provisioning and NodePoolsReady reports the pools' own state, True, rather than Pending on ClusterReady.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=_xr( + node_pools=[ + v1alpha1.NodePool( + name="gpu-l40s", + role="GPU", + plan="vcg-l40s-16c-180g-48vram", + maxNodeCount=4, + gpu=v1alpha1.Gpu(acceleratorType="nvidia-l40s"), + ), + v1alpha1.NodePool(name="cpu-pool", role="CPU", plan="vc2-6c-16gb", maxNodeCount=4), + ], + ), + resources={ + "cluster": _observed_cluster(ready=False), + "node-pool-gpu-l40s": _observed_managed_resource(ready=True, failure=None), + "node-pool-cpu-pool": _observed_managed_resource(ready=True, failure=None), + "gpu-observer": _observed_managed_resource(ready=True, failure=None), + }, + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_xr( + node_pools=[{"name": "gpu-l40s", "state": "Ready"}, {"name": "cpu-pool", "state": "Ready"}] + ), + resources={ + "cluster": _cluster(ready=fnv1.READY_UNSPECIFIED), + "node-pool-gpu-l40s": _node_pool_gpu( + node_quantity=1, + autoscaling={"autoScaler": True, "minNodes": 1, "maxNodes": 4}, + ready=fnv1.READY_TRUE, + ), + "node-pool-cpu-pool": _node_pool_cpu( + node_quantity=1, + autoscaling={"autoScaler": True, "minNodes": 1, "maxNodes": 4}, + ready=fnv1.READY_TRUE, + ), "provider-config-kubernetes": _provider_config(), "gpu-observer": _gpu_observer(ready=fnv1.READY_TRUE), }, ), context=structpb.Struct(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + message="1 of 2 resources ready", + ), + fnv1.Condition( + type="NodePoolsReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], ), ), ] diff --git a/schemas/.lock.json b/schemas/.lock.json index 2ef0716fc..03808b4c1 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,6 +1,6 @@ { "packages": { - "fs://apis": "b6e082c5c56966cb06dc6eaf45b14b83938971c8002ca7639b7e223576f77e2d", + "fs://apis": "c9210e3c0469d7ffccb38bc7fe8975d7ea2d58e880e7e5d94b9e615a6013a1fe", "git://https://github.com/crossplane/crossplane/cluster/crds": "14c53aac914d393b5bcc00962fbc90d4be5d35bc", "git://https://github.com/modelplaneai/cluster-agent/cluster/charts/cluster-agent/files/crds": "a833d3f6f7d9a6ac6083e7453375b58b65ee741a", "xpkg://xpkg.upbound.io/crossplane-contrib/provider-upcloud:v1.1.0": "sha256:ef449e492e8277974c502672765ea83e479a167738f4fe3e59577f67b8fda622", diff --git a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py index 502c23621..608427b3e 100644 --- a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py @@ -404,6 +404,21 @@ class Gateway(BaseModel): """ +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class ProviderConfigRef(BaseModel): name: str | None = None """ @@ -487,6 +502,10 @@ class Status(BaseModel): """ Namespace where the internal XRs (cluster, backend) were created. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools, as the cluster Modelplane provisions reports it. Not set when source is Existing, because Modelplane doesn't provision those pools. + """ providerConfigRef: ProviderConfigRef | None = None schedulablePools: list[SchedulablePool] | None = Field(None, max_length=8) """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py index 95ef71792..d2a1610d6 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/akscluster/v1alpha1.py @@ -156,6 +156,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -180,6 +195,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools, except System pools. System pools run the cluster's controllers and serving stack and are covered under the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this AKSCluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py index 2b7d90e04..581d263ec 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/civocluster/v1alpha1.py @@ -118,6 +118,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -138,6 +153,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools, except System pools. System pools run the cluster's controllers and serving stack and are covered under the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this CivoCluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/coreweavecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/coreweavecluster/v1alpha1.py index 726cff8e3..2ccc81b2c 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/coreweavecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/coreweavecluster/v1alpha1.py @@ -127,6 +127,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -151,6 +166,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools, except System pools. System pools run the cluster's controllers and serving stack and are covered under the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this CoreWeaveCluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py index 37ce4eefa..87529c5a7 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/ekscluster/v1alpha1.py @@ -180,6 +180,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -204,6 +219,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools, except System pools. System pools run the cluster's controllers and serving stack and are covered under the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this EKSCluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py index 18144b5a5..22736381f 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/gkecluster/v1alpha1.py @@ -175,6 +175,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -199,6 +214,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools, except System pools. System pools run the cluster's controllers and serving stack and are covered under the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this GKECluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py index 4e44a948e..722f32654 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/nebiuscluster/v1alpha1.py @@ -155,6 +155,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -183,6 +198,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools, except System pools. System pools run the cluster's controllers and serving stack and are covered under the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by or passed through this cluster. Consumers use these to authenticate to the cluster. Secrets are in the same namespace as this NebiusCluster unless an entry says otherwise. diff --git a/schemas/python/models/ai/modelplane/infrastructure/upcloudcluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/upcloudcluster/v1alpha1.py index 84b303e21..219ad07b2 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/upcloudcluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/upcloudcluster/v1alpha1.py @@ -140,6 +140,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=63) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -160,6 +175,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools, except System pools. System pools run the cluster's controllers and serving stack and are covered under the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this UpCloudCluster. diff --git a/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py index 52f8d2dac..53897288e 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/vultrcluster/v1alpha1.py @@ -118,6 +118,21 @@ class Condition(BaseModel): type: str +class NodePoolModel(BaseModel): + message: str | None = None + """ + The provider's error, prefixed with the name of the composed resource it's about. Set only when state is Failed. + """ + name: constr(max_length=40) + """ + Name of the node pool in spec.nodePools. + """ + state: Literal['Pending', 'Provisioning', 'Ready', 'Failed'] + """ + Pending until Modelplane composes the pool's cloud resources, Provisioning until their provider reports them all Ready, then Ready. Failed while the provider reports that its last create, update or delete of one of them failed. + """ + + class Secret(BaseModel): key: constr(max_length=253) """ @@ -138,6 +153,10 @@ class Status(BaseModel): """ Conditions of the resource. """ + nodePools: list[NodePoolModel] | None = Field(None, max_length=8) + """ + The provisioning state of each node pool in spec.nodePools, except System pools. System pools run the cluster's controllers and serving stack and are covered under the ClusterReady condition. + """ secrets: list[Secret] | None = None """ Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this VultrCluster.