[infra] make the general node groups min and max sizes code-owned, leaving only desired size out of band

infra/space/aws/eks/modules/controllers/cluster_autoscaler.tf

@@ -10,10 +10,12 @@
 # the same ASG ARNs for the same reason.
 #
 # Discovery rather than `--nodes=<min>:<max>:<asg>` because a discovered group's
-# bounds are the ASG's own min and max, read live. Terraform leaves the node
-# group's sizes alone after creation, so an operator resizes the pool through
-# the EKS API and cluster-autoscaler follows, and no apply ever sends a stale
-# desired size back to the group.
+# bounds are the ASG's own min and max, read live: the caller's apply sets them,
+# and cluster-autoscaler follows without its arguments changing. The caller
+# leaves the group's desired size to cluster-autoscaler, so only an apply that
+# changes min or max sends a desired size back to the group, the one in its
+# plan, or one that creates or replaces the group, which comes up at its
+# configured size.
 #
 # Several groups are the general pool split one per zone. A single-zone ASG
 # never runs AZRebalance, which EKS drains without honoring PodDisruptionBudgets

infra/space/aws/eks/modules/controllers/variables.tf

@@ -82,7 +82,7 @@ variable "aws_load_balancer_controller_resources" {
 }
 
 variable "cluster_autoscaler" {
-  description = "cluster-autoscaler for the serving general node groups, or null to install none of it. It scales each between that group's own min and max, read live from its ASG, so a resize through the EKS API moves its bounds too. With more than one group it balances scale-ups across them, so give it only groups that differ by zone alone: the same instance type, launch template, labels and taints."
+  description = "cluster-autoscaler for the serving general node groups, or null to install none of it. It scales each between that group's own min and max, read live from its ASG, so it follows the bounds an apply sets without a restart. With more than one group it balances scale-ups across them, so give it only groups that differ by zone alone: the same instance type, launch template, labels and taints."
   type = object({
     role_name          = string
     kubernetes_version = string

infra/space/aws/eks/modules/eks-cluster/node_groups.tf

@@ -194,18 +194,55 @@ resource "aws_eks_node_group" "managed" {
     update = var.node_group_update_timeout
   }
 
-  # These groups' size moves only outside Terraform: an operator resizing them
-  # through the EKS API, or a cluster-autoscaler the caller runs on one of them,
-  # which sets the ASG's desired capacity between the min and max it reads live
-  # from that ASG. Without this, the next apply for any unrelated change would
-  # reset each group to its creation-time sizing and scale it down under
-  # whatever is running on it. The sizes in node_pools still decide what a new
-  # cluster comes up with.
+  # min_size and max_size are code-owned: node_pools decides them, and a change
+  # lands through an apply. desired_size isn't. Something else moves it: an
+  # operator through the EKS API, or a cluster-autoscaler the caller runs on
+  # one of these groups. Without this, every unrelated apply would reset each
+  # group to its configured desired size and could scale it down under
+  # whatever is running on it. node_pools' desired_size still decides what a
+  # new group comes up with.
+  #
+  # The provider sends all three fields in one UpdateNodegroupConfig call,
+  # desired_size as it stood in state when the plan was made, and EKS requires
+  # min_size <= desired_size <= max_size. So raising min_size above a group's
+  # live desired size, or lowering max_size below it (say after
+  # cluster-autoscaler has grown the pool), can't apply: the managed_node_groups
+  # output refuses the plan. And an apply of a saved plan, on a pool
+  # cluster-autoscaler has resized since the plan, sends the old desired size
+  # back and resizes the pool to it. Both go away if the live group already
+  # has the new bounds when the plan is made: it then has no bound change and
+  # sends no size. Make that EKS change immediately before planning the
+  # matching code change, since any other apply in between reverts it (see
+  # below). To raise bounds, raise max, then desired if it's below the new
+  # min, then min. To lower them, lower min, then shrink the pool to the new
+  # max, then lower max. Shrink by draining a chosen node and terminating it
+  # with `aws autoscaling terminate-instance-in-auto-scaling-group
+  # --should-decrement-desired-capacity`, one at a time, each from the zone
+  # with the most nodes: a smaller desired size lets the ASG pick which
+  # instances go, and a multi-zone ASG left uneven runs AZRebalance, neither
+  # of which honors PodDisruptionBudgets.
+  #
+  # Otherwise, on a cluster running the autoscaler, plan immediately before
+  # applying a min/max change, check the plan's desired_size against the live
+  # group, and keep the change out of an apply that also rolls the group: the
+  # provider rolls first, which can take hours, and only then sends the
+  # desired size it planned.
+  #
+  # An out-of-band min/max change through the EKS API is drift: every later
+  # apply on the cluster, whatever it's for, plans to revert it, and the plan
+  # fails while the live desired size sits outside the configured bounds.
+  # Record an emergency resize in node_pools rather than leaving it live-only.
   lifecycle {
     ignore_changes = [
-      scaling_config[0].min_size,
       scaling_config[0].desired_size,
-      scaling_config[0].max_size,
     ]
   }
 }
+
+locals {
+  managed_node_groups_outside_bounds = [
+    for name, group in aws_eks_node_group.managed : name
+    if group.scaling_config[0].desired_size < group.scaling_config[0].min_size ||
+    group.scaling_config[0].desired_size > group.scaling_config[0].max_size
+  ]
+}

infra/space/aws/eks/modules/eks-cluster/outputs.tf

@@ -100,6 +100,14 @@ output "managed_node_groups" {
       scaling_config = group.scaling_config
     }
   }
+
+  # desired_size is ignored, so the plan carries the size EKS reported when it
+  # refreshed, and a min/max change sends that size with the new bounds. EKS
+  # refuses one outside them partway through the apply; this refuses the plan.
+  precondition {
+    condition     = length(local.managed_node_groups_outside_bounds) == 0
+    error_message = "The live desired size of ${join(", ", local.managed_node_groups_outside_bounds)} is outside its configured min_size and max_size, and EKS refuses those bounds. Move the live group to the new bounds through the EKS API, in the order the comment on aws_eks_node_group.managed in the eks-cluster module gives, and re-plan; or change the configured sizes, which for a group being created or replaced are its creation sizes."
+  }
 }
 
 output "managed_node_group_arns" {

infra/space/aws/eks/modules/eks-cluster/variables.tf

@@ -134,11 +134,11 @@ variable "node_pools" {
     by default; `AL2023_ARM_64_STANDARD` for Graviton instance types); set, the
     group runs that AMI as `CUSTOM`, `ami_type` is ignored, and
     `node_pool_user_data` must carry the pool's bootstrap script. `additional_policy_arns` keys must be known at plan time; their
-    values may not be. For a pool with no `ami_id`, `min_size`,
-    `desired_size`, and `max_size` apply at creation only: afterwards the
-    group is resized through the EKS API (or its ASG's desired capacity by a
-    cluster-autoscaler the caller runs on it) and Terraform ignores all three. A custom-AMI pool
-    keeps its size under Terraform. `update_strategy` is the group's
+    values may not be. For a pool with no `ami_id`, `min_size` and
+    `max_size` are applied whenever they change, and `desired_size` at creation
+    only: afterwards it moves through the EKS API or a cluster-autoscaler the
+    caller runs on the group, and Terraform ignores it. A custom-AMI pool
+    keeps all three under Terraform. `update_strategy` is the group's
     rolling-update strategy: null leaves EKS's default, which launches
     replacements before draining; `MINIMAL` drains first, so a roll never runs
     more nodes than the group's size.

infra/space/aws/eks/modules/space/controllers.tf

@@ -18,7 +18,7 @@ locals {
   )
 
   # The groups general_serving_node_group names, each with the recorded floor
-  # its live min has to meet: one group, or every per-AZ group for
+  # its min has to meet: one group, or every per-AZ group for
   # general-per-az. Its validations guarantee the names exist in
   # general_node_groups: general can only be retired while another group
   # serves, and general-2 or the per-AZ groups can only serve once they're
@@ -29,6 +29,11 @@ locals {
       var.general_blue_green_node_group == null && var.general_per_az_node_groups == null ? 1 : var.general_min_size
     )
   }
+  general_node_groups_outside_bounds = [
+    for group in concat(aws_eks_node_group.general_blue_green, values(aws_eks_node_group.general_per_az)) : group.node_group_name
+    if group.scaling_config[0].desired_size < group.scaling_config[0].min_size ||
+    group.scaling_config[0].desired_size > group.scaling_config[0].max_size
+  ]
   serving_general_node_groups = {
     for name in keys(local.serving_general_node_group_floors) : name => local.general_node_groups[name]
   }

infra/space/aws/eks/modules/space/locals.tf

@@ -227,7 +227,12 @@ locals {
           var.general_labels,
         )
         max_size = var.general_max_size
-        min_size = var.general_min_size
+        # The recorded floor applies only while general serves. Parked, its
+        # minimum is zero, so an operator can scale it to zero through EKS
+        # without the next apply raising it again. desired_size is ignored, but
+        # the apply that drops the minimum sends the desired size the plan read,
+        # so plan it immediately before applying while the pool serves.
+        min_size = var.general_serving_node_group == "general" ? var.general_min_size : 0
         ami_id   = null
         service_account = {
           name      = var.general_service_account_name

infra/space/aws/eks/modules/space/node_groups.tf

@@ -252,7 +252,8 @@ resource "aws_eks_node_group" "general_blue_green" {
   scaling_config {
     desired_size = var.general_blue_green_node_group.recorded_active_size
     max_size     = var.general_blue_green_node_group.max_size
-    min_size     = var.general_blue_green_node_group.recorded_active_size
+    # As on general: the recorded floor only while this pool serves.
+    min_size = var.general_serving_node_group == "general-2" ? var.general_blue_green_node_group.recorded_active_size : 0
   }
 
   update_config {
@@ -279,16 +280,16 @@ resource "aws_eks_node_group" "general_blue_green" {
     delete = "2h"
   }
 
-  # After creation, operators scale this pool between zero and its active
-  # target through EKS, and while it serves, cluster-autoscaler moves its
-  # desired size inside those bounds. Unrelated Terraform applies must not
-  # wake a dormant pool or scale down one that is carrying the active general
-  # workload.
+  # Same ownership split as module.eks's managed groups, and the same
+  # constraint on raising min_size above the live desired size: min and max
+  # are code-owned, desired moves through EKS or, while this pool serves,
+  # cluster-autoscaler. So switching general_serving_node_group to a pool
+  # parked at zero, this one, general or a per-AZ group, needs it grown
+  # through EKS first, and the switch, like any bound change, should be
+  # planned immediately before it's applied (see the per-AZ steps below).
   lifecycle {
     ignore_changes = [
-      scaling_config[0].min_size,
       scaling_config[0].desired_size,
-      scaling_config[0].max_size,
     ]
   }
 }
@@ -304,11 +305,18 @@ resource "aws_eks_node_group" "general_blue_green" {
 #
 # Moving a cluster onto them takes three applies, each with nothing else in it:
 # 1. Declare general_per_az_node_groups. The groups come up at their floors
-#    beside the serving group, which keeps serving. A serving general's live
-#    min_size now has to hold general_min_size, as it does beside general-2.
+#    beside the serving group, which keeps serving. Their minimums stay zero
+#    until they serve.
 # 2. Set general_serving_node_group = "general-per-az". The discovery tags move,
 #    so cluster-autoscaler stops scaling the old group and balances the new
-#    ones. Then cordon the old group's nodes and drain them through the
+#    ones, and the per-AZ minimums rise to each zone's min_size while the old
+#    group's falls to zero. The plan refuses a minimum above a group's live
+#    desired size, so grow any per-AZ group scaled below its floor through EKS
+#    and plan again. Each bound change sends the desired size from the plan,
+#    and cluster-autoscaler is still scaling the old group until this apply
+#    moves its tag, so plan immediately before applying: a saved plan applied
+#    after the autoscaler has grown the old group shrinks it back.
+#    Then cordon the old group's nodes and drain them through the
 #    eviction API, which honors PodDisruptionBudgets and each pod's grace
 #    period; the evicted pods land on the per-AZ groups, which
 #    cluster-autoscaler grows as they go Pending (without it, grow them through
@@ -356,7 +364,8 @@ resource "aws_eks_node_group" "general_per_az" {
   scaling_config {
     desired_size = each.value.min_size
     max_size     = each.value.max_size
-    min_size     = each.value.min_size
+    # As on general-2: each zone's floor only while the per-AZ groups serve.
+    min_size = var.general_serving_node_group == "general-per-az" ? each.value.min_size : 0
   }
 
   update_config {
@@ -383,14 +392,10 @@ resource "aws_eks_node_group" "general_per_az" {
     delete = "2h"
   }
 
-  # The same contract as general-2: after creation each group is sized through
-  # EKS, and while the groups serve, cluster-autoscaler moves their desired
-  # sizes, so no apply sends a stale size back.
+  # The same ownership split as general-2.
   lifecycle {
     ignore_changes = [
-      scaling_config[0].min_size,
       scaling_config[0].desired_size,
-      scaling_config[0].max_size,
     ]
   }
 }

infra/space/aws/eks/modules/space/outputs.tf

@@ -102,16 +102,24 @@ output "general_serving_node_group" {
   precondition {
     # Existing single-pool environments retain their positive-capacity gate.
     # Blue/green and per-AZ configurations additionally enforce the
-    # operator-recorded floor on every serving group. The sizes are the live
-    # ones: Terraform ignores them after creation, so the plan carries what EKS
-    # reports.
+    # operator-recorded floor on every serving group. Terraform sets a serving
+    # group's minimum to that floor, so those hold by construction; the
+    # single-pool gate still refuses a general_min_size of zero.
     condition = alltrue([
       for name, group in local.serving_general_node_groups :
       group.scaling_config[0].min_size >= local.serving_general_node_group_floors[name]
     ])
     error_message = "The recorded general_serving_node_group must be live and, for blue/green and per-AZ configurations, meet its recorded capacity floor."
   }
 
+  # As module.eks checks for its managed groups: a serving switch raises the
+  # incoming group's minimum to its floor, and EKS refuses one above the live
+  # desired size partway through the apply, so this refuses the plan instead.
+  precondition {
+    condition     = length(local.general_node_groups_outside_bounds) == 0
+    error_message = "The live desired size of ${join(", ", local.general_node_groups_outside_bounds)} is outside the min_size and max_size this plan sets, and EKS refuses those bounds. Move the live group to the new bounds through the EKS API, in the order the comment on aws_eks_node_group.managed in the eks-cluster module gives, and re-plan; or change the configured sizes, which for a group being created or replaced are its creation sizes."
+  }
+
   precondition {
     condition = alltrue([
       for name, asg in data.aws_autoscaling_group.retiring_general :
@@ -153,7 +161,7 @@ output "general_serving_node_group" {
 }
 
 output "general_serving_min_size" {
-  description = "Largest live floor of the declared general groups. Any of them may carry the active workload, so a root-owned DaemonSet that must roll on the general pool gates on this while the inactive pools stay at zero; referencing it also orders the consumer after a floor update."
+  description = "Largest configured minimum among the declared general groups: the serving group's floor, or the largest zone floor while the per-AZ groups serve, since the parked groups' minimums are zero. A root-owned DaemonSet that must roll on the general pool gates on this; referencing it also orders the consumer after a floor update."
   value       = max([for group in values(local.general_node_groups) : group.scaling_config[0].min_size]...)
 }