[infra] retire the parked general node group, add a default-off cluster-autoscaler for the serving general group, and give the general-pool controllers PodDisruptionBudgets

infra/space/aws/eks/charts/pod-disruption-budget/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: pod-disruption-budget
+description: One PodDisruptionBudget for a workload whose own chart can't render one
+type: application
+version: 0.1.0
+appVersion: "1.0.0"

infra/space/aws/eks/charts/pod-disruption-budget/templates/pdb.yaml

@@ -0,0 +1,33 @@
+{{- /* An empty selector matches every pod in the namespace, so refuse it. */ -}}
+{{- if empty .Values.matchLabels }}
+{{- fail "matchLabels must name at least one label" }}
+{{- end -}}
+{{- /*
+Only a positive whole number, or a percentage up to 100, with no leading zero.
+Zero refuses every eviction and pins the node, and the value renders unquoted,
+so "+0" would reach the API as 0 and "010" as octal 8.
+*/ -}}
+{{- $maxUnavailable := trim (toString (required "maxUnavailable is required" .Values.maxUnavailable)) }}
+{{- if not (regexMatch "^([1-9][0-9]*|[1-9][0-9]?%|100%)$" $maxUnavailable) }}
+{{- fail "maxUnavailable must be a positive whole number or a percentage from 1% to 100%" }}
+{{- end -}}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ required "name is required" .Values.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version }}
+spec:
+  maxUnavailable: {{ $maxUnavailable }}
+  {{- /*
+  A pod that isn't Ready isn't serving, so evicting it costs nothing. Without
+  this, a crashlooping singleton refuses every eviction and blocks any drain of
+  its node.
+  */}}
+  unhealthyPodEvictionPolicy: AlwaysAllow
+  selector:
+    matchLabels:
+      {{- toYaml .Values.matchLabels | nindent 6 }}

infra/space/aws/eks/charts/pod-disruption-budget/values.yaml

@@ -0,0 +1,13 @@
+# Set by the owning Terraform module to the chart's content hash, so a template
+# edit changes the release's values and plans. Nothing renders it.
+chart:
+  checksum: ""
+
+# Name of the PodDisruptionBudget.
+name: ""
+
+# Labels of the pods the budget covers. Use the workload's own selector labels
+# so the budget never matches pods from another release.
+matchLabels: {}
+
+maxUnavailable: 1

infra/space/aws/eks/modules/controllers/cluster_autoscaler.tf

@@ -0,0 +1,269 @@
+# cluster-autoscaler for the serving general node group, and nothing else.
+#
+# It discovers that group by a SPACE-specific tag this module puts on its one
+# ASG, not by the stock `k8s.io/cluster-autoscaler/enabled` +
+# `k8s.io/cluster-autoscaler/<cluster>` pair, because EKS stamps that pair on
+# every managed node group's ASG: workers, builders and cred-authority as well
+# as general. Worker nodes carry Firecracker sandboxes whose drain path
+# cluster-autoscaler knows nothing about, so the stock pair would let it scale
+# those pools down under live sandboxes. The IAM write actions are scoped to
+# the same one ASG ARN for the same reason.
+#
+# Discovery rather than `--nodes=<min>:<max>:<asg>` because a discovered group's
+# bounds are the ASG's own min and max, read live. Terraform leaves the node
+# group's sizes alone after creation, so an operator resizes the pool through
+# the EKS API and cluster-autoscaler follows, and no apply ever sends a stale
+# desired size back to the group.
+
+locals {
+  # Pinned to the control plane's minor, as upstream recommends. Add the next
+  # minor here before bumping `kubernetes_version` on a cluster that runs it;
+  # the precondition on the release fails the plan until then. Upstream marks
+  # --scale-down-enabled (set below) deprecated, so check each new minor still
+  # accepts it. The chart's RBAC is built for appVersion 1.35.0 (chart 9.59.0),
+  # a minor behind the 1.36 image; check the cluster role against each new
+  # image too.
+  cluster_autoscaler_image_tags = {
+    "1.35" = "v1.35.2"
+    "1.36" = "v1.36.1"
+  }
+
+  # The default cutoff, spelled out so it's a reviewable literal rather than an
+  # upstream default that can move under a chart bump. Pods below it are
+  # expendable: cluster-autoscaler adds no node to run one and doesn't count one
+  # as a reason to keep a node. Every PriorityClass a general-pool workload
+  # names has to sit at or above it, the SPACE ones in
+  # infra/space/kind/manifests/priorityclasses.yaml included.
+  cluster_autoscaler_expendable_pods_priority_cutoff = -10
+
+  # The discovery tag, unique per cluster so a tagged ASG can't be picked up by
+  # another cluster's autoscaler in the same account.
+  cluster_autoscaler_discovery_tag_key = "space.pplx.net/cluster-autoscaler/${var.cluster_name}"
+}
+
+# Moving general_serving_node_group retargets this tag, so the old group drops
+# out of discovery and the new one joins it in the same apply.
+resource "aws_autoscaling_group_tag" "cluster_autoscaler" {
+  count = var.cluster_autoscaler == null ? 0 : 1
+
+  autoscaling_group_name = var.cluster_autoscaler.autoscaling_group_name
+
+  lifecycle {
+    precondition {
+      condition     = var.cluster_autoscaler.autoscaling_group_name != null
+      error_message = "cluster-autoscaler needs the serving node group's ASG name, and the group reports none; check the node group's health in EKS."
+    }
+    precondition {
+      condition     = length(local.cluster_autoscaler_discovery_tag_key) <= 128
+      error_message = "The discovery tag key ${local.cluster_autoscaler_discovery_tag_key} is longer than the 128 characters AWS allows; the cluster name is too long for it."
+    }
+  }
+
+  tag {
+    key                 = local.cluster_autoscaler_discovery_tag_key
+    value               = "owned"
+    propagate_at_launch = false
+  }
+}
+
+data "aws_iam_policy_document" "cluster_autoscaler_assume_role" {
+  count = var.cluster_autoscaler == null ? 0 : 1
+
+  statement {
+    actions = ["sts:AssumeRoleWithWebIdentity"]
+
+    principals {
+      type        = "Federated"
+      identifiers = [var.oidc_provider_arn]
+    }
+
+    condition {
+      test     = "StringEquals"
+      variable = "${var.oidc_issuer_host}:sub"
+      values   = ["system:serviceaccount:kube-system:cluster-autoscaler"]
+    }
+
+    condition {
+      test     = "StringEquals"
+      variable = "${var.oidc_issuer_host}:aud"
+      values   = ["sts.amazonaws.com"]
+    }
+  }
+}
+
+resource "aws_iam_role" "cluster_autoscaler" {
+  count = var.cluster_autoscaler == null ? 0 : 1
+
+  name               = var.cluster_autoscaler.role_name
+  assume_role_policy = data.aws_iam_policy_document.cluster_autoscaler_assume_role[0].json
+}
+
+data "aws_iam_policy_document" "cluster_autoscaler" {
+  count = var.cluster_autoscaler == null ? 0 : 1
+
+  # The Describe and Get calls take no resource-level scoping.
+  statement {
+    actions = [
+      "autoscaling:DescribeAutoScalingGroups",
+      "autoscaling:DescribeAutoScalingInstances",
+      "autoscaling:DescribeLaunchConfigurations",
+      "autoscaling:DescribeScalingActivities",
+      "autoscaling:DescribeTags",
+      "ec2:DescribeImages",
+      "ec2:DescribeInstanceTypes",
+      "ec2:DescribeLaunchTemplateVersions",
+      "ec2:GetInstanceTypesFromInstanceRequirements",
+    ]
+    resources = ["*"]
+  }
+
+  statement {
+    actions   = ["eks:DescribeNodegroup"]
+    resources = [var.cluster_autoscaler.node_group_arn]
+  }
+
+  statement {
+    actions = [
+      "autoscaling:SetDesiredCapacity",
+      "autoscaling:TerminateInstanceInAutoScalingGroup",
+    ]
+    resources = [var.cluster_autoscaler.autoscaling_group_arn]
+  }
+}
+
+resource "aws_iam_role_policy" "cluster_autoscaler" {
+  count = var.cluster_autoscaler == null ? 0 : 1
+
+  name   = var.cluster_autoscaler.role_name
+  role   = aws_iam_role.cluster_autoscaler[0].id
+  policy = data.aws_iam_policy_document.cluster_autoscaler[0].json
+}
+
+resource "helm_release" "cluster_autoscaler" {
+  count = var.cluster_autoscaler == null ? 0 : 1
+
+  name       = "cluster-autoscaler"
+  repository = "https://kubernetes.github.io/autoscaler"
+  chart      = "cluster-autoscaler"
+  namespace  = "kube-system"
+  version    = "9.59.0"
+
+  values = [yamlencode({
+    cloudProvider    = "aws"
+    awsRegion        = var.aws_region
+    fullnameOverride = "cluster-autoscaler"
+    # The chart renders `--node-group-auto-discovery=asg:tag=<tags>` once
+    # clusterName is set; `tags` replaces the stock pair rather than adding to
+    # it, so only the ASG carrying this key matches.
+    autoDiscovery = {
+      clusterName = var.cluster_name
+      tags        = [local.cluster_autoscaler_discovery_tag_key]
+    }
+    image = {
+      tag = lookup(local.cluster_autoscaler_image_tags, var.cluster_autoscaler.kubernetes_version, null)
+    }
+    # A pod with an emptyDir (a scratch or upload spool, say) would otherwise
+    # pin its node: skip-nodes-with-local-storage keeps any such node, and a
+    # workload that runs one on most general nodes would pin most of the pool.
+    # An emptyDir dies with its pod on every rollout anyway.
+    # skip-nodes-with-system-pods only delays kube-system pods with no PDB: once
+    # one is older than --blocking-system-pod-distruption-timeout (default 1h)
+    # it's evicted like any other. The Deployments in the kube-system Helm
+    # releases this module installs on general all carry budgets; give any the
+    # calling root adds there one too before turning scale-down on. The flip
+    # side: a single-replica `maxUnavailable: 1` budget allows no disruption
+    # while its pod isn't Ready, and the pdb drainability rule blocks the node
+    # on that with no timeout, whatever the budget's
+    # `unhealthyPodEvictionPolicy`, so a crashlooping singleton pins its node
+    # from scale-down until it recovers. Other drains go through the eviction
+    # API, which honors that policy: every single-replica budget here comes
+    # from the in-repo chart, which sets `AlwaysAllow`, so an unready singleton
+    # never blocks a drain.
+    #
+    # The drain cap and the scale-up delay are added only where a cluster sets
+    # them, so a cluster that doesn't keeps the upstream defaults (600s and 0s)
+    # with no change to its values. A null can't stand in for "unset": the chart
+    # renders a null extraArgs entry as the bare flag.
+    extraArgs = merge(
+      {
+        "scale-down-enabled"              = var.cluster_autoscaler.scale_down_enabled
+        "scale-down-unneeded-time"        = "30m"
+        "scale-down-delay-after-add"      = "30m"
+        "skip-nodes-with-system-pods"     = true
+        "skip-nodes-with-local-storage"   = false
+        "expendable-pods-priority-cutoff" = local.cluster_autoscaler_expendable_pods_priority_cutoff
+      },
+      # Caps each evicted pod's grace period during a scale-down drain, and
+      # bounds how long the drain waits for the pods to go (the cap plus 30s).
+      # A pod with a shorter grace keeps its own. Upstream refuses to start if
+      # --drain-priority-config is also set.
+      var.cluster_autoscaler.max_graceful_termination_sec == null ? {} : {
+        "max-graceful-termination-sec" = var.cluster_autoscaler.max_graceful_termination_sec
+      },
+      # Pods younger than this aren't considered for scale-up, so a rollout
+      # surge pod whose room a terminating pod frees within the delay doesn't
+      # add a node.
+      var.cluster_autoscaler.new_pod_scale_up_delay == null ? {} : {
+        "new-pod-scale-up-delay" = var.cluster_autoscaler.new_pod_scale_up_delay
+      },
+    )
+    priorityClassName = "system-cluster-critical"
+    # One replica, and a budget that lets it go: the pod lives on the pool it
+    # scales, so a budget that refused its eviction would pin whichever general
+    # node it landed on. A rescheduled replica rebuilds its view from the API
+    # server and the ASG, losing in-memory state such as scale-down timers and
+    # scale-up backoff. The budget comes from disruption_budgets.tf, which can
+    # set `AlwaysAllow`; null drops the chart's own.
+    replicaCount        = 1
+    podDisruptionBudget = null
+    # Memory is per cluster because cluster-autoscaler's shared informer caches
+    # every pod in the cluster, sandboxes included, and no flag narrows it
+    # (main.go builds it with only a managedFields-stripping transform). Request
+    # equals limit, so the pod never counts on memory its node hasn't reserved
+    # for it. GOMEMLIMIT at 90% of the limit makes the Go GC collect harder as
+    # the runtime's total memory (heap, stacks, GC metadata) nears it, instead
+    # of letting the initial pod list's garbage push RSS past the limit. The
+    # other 10% is for memory the Go runtime doesn't manage.
+    resources = {
+      requests = {
+        cpu    = "100m"
+        memory = "${var.cluster_autoscaler.memory_mib}Mi"
+      }
+      limits = {
+        memory = "${var.cluster_autoscaler.memory_mib}Mi"
+      }
+    }
+    extraEnv = {
+      GOMEMLIMIT = "${floor(var.cluster_autoscaler.memory_mib * 0.9)}MiB"
+    }
+    nodeSelector = {
+      "space.pplx.net/node-pool" = "general"
+    }
+    rbac = {
+      serviceAccount = {
+        create = true
+        name   = "cluster-autoscaler"
+        annotations = {
+          "eks.amazonaws.com/role-arn" = aws_iam_role.cluster_autoscaler[0].arn
+        }
+      }
+    }
+  })]
+
+  lifecycle {
+    precondition {
+      condition     = contains(keys(local.cluster_autoscaler_image_tags), var.cluster_autoscaler.kubernetes_version)
+      error_message = "cluster-autoscaler has no image pinned for Kubernetes ${var.cluster_autoscaler.kubernetes_version}; add one to local.cluster_autoscaler_image_tags."
+    }
+    # Like the other controllers here, its pod pins to the general node pool.
+    precondition {
+      condition     = var.nodes_ready
+      error_message = "The general node pool must exist before cluster-autoscaler installs."
+    }
+  }
+
+  depends_on = [
+    aws_autoscaling_group_tag.cluster_autoscaler,
+    aws_iam_role_policy.cluster_autoscaler,
+  ]
+}