diff --git a/infra/space/aws/eks/charts/pod-disruption-budget/Chart.yaml b/infra/space/aws/eks/charts/pod-disruption-budget/Chart.yaml new file mode 100644 index 000000000..2f41a6380 --- /dev/null +++ b/infra/space/aws/eks/charts/pod-disruption-budget/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: pod-disruption-budget +description: One PodDisruptionBudget for a workload whose own chart can't render one +type: application +version: 0.1.0 +appVersion: "1.0.0" diff --git a/infra/space/aws/eks/charts/pod-disruption-budget/templates/pdb.yaml b/infra/space/aws/eks/charts/pod-disruption-budget/templates/pdb.yaml new file mode 100644 index 000000000..ccbdc2fb5 --- /dev/null +++ b/infra/space/aws/eks/charts/pod-disruption-budget/templates/pdb.yaml @@ -0,0 +1,33 @@ +{{- /* An empty selector matches every pod in the namespace, so refuse it. */ -}} +{{- if empty .Values.matchLabels }} +{{- fail "matchLabels must name at least one label" }} +{{- end -}} +{{- /* +Only a positive whole number, or a percentage up to 100, with no leading zero. +Zero refuses every eviction and pins the node, and the value renders unquoted, +so "+0" would reach the API as 0 and "010" as octal 8. +*/ -}} +{{- $maxUnavailable := trim (toString (required "maxUnavailable is required" .Values.maxUnavailable)) }} +{{- if not (regexMatch "^([1-9][0-9]*|[1-9][0-9]?%|100%)$" $maxUnavailable) }} +{{- fail "maxUnavailable must be a positive whole number or a percentage from 1% to 100%" }} +{{- end -}} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ required "name is required" .Values.name }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version }} +spec: + maxUnavailable: {{ $maxUnavailable }} + {{- /* + A pod that isn't Ready isn't serving, so evicting it costs nothing. Without + this, a crashlooping singleton refuses every eviction and blocks any drain of + its node. + */}} + unhealthyPodEvictionPolicy: AlwaysAllow + selector: + matchLabels: + {{- toYaml .Values.matchLabels | nindent 6 }} diff --git a/infra/space/aws/eks/charts/pod-disruption-budget/values.yaml b/infra/space/aws/eks/charts/pod-disruption-budget/values.yaml new file mode 100644 index 000000000..a81e20ce5 --- /dev/null +++ b/infra/space/aws/eks/charts/pod-disruption-budget/values.yaml @@ -0,0 +1,13 @@ +# Set by the owning Terraform module to the chart's content hash, so a template +# edit changes the release's values and plans. Nothing renders it. +chart: + checksum: "" + +# Name of the PodDisruptionBudget. +name: "" + +# Labels of the pods the budget covers. Use the workload's own selector labels +# so the budget never matches pods from another release. +matchLabels: {} + +maxUnavailable: 1 diff --git a/infra/space/aws/eks/modules/controllers/cluster_autoscaler.tf b/infra/space/aws/eks/modules/controllers/cluster_autoscaler.tf new file mode 100644 index 000000000..afeb967ac --- /dev/null +++ b/infra/space/aws/eks/modules/controllers/cluster_autoscaler.tf @@ -0,0 +1,269 @@ +# cluster-autoscaler for the serving general node group, and nothing else. +# +# It discovers that group by a SPACE-specific tag this module puts on its one +# ASG, not by the stock `k8s.io/cluster-autoscaler/enabled` + +# `k8s.io/cluster-autoscaler/` pair, because EKS stamps that pair on +# every managed node group's ASG: workers, builders and cred-authority as well +# as general. Worker nodes carry Firecracker sandboxes whose drain path +# cluster-autoscaler knows nothing about, so the stock pair would let it scale +# those pools down under live sandboxes. The IAM write actions are scoped to +# the same one ASG ARN for the same reason. +# +# Discovery rather than `--nodes=::` because a discovered group's +# bounds are the ASG's own min and max, read live. Terraform leaves the node +# group's sizes alone after creation, so an operator resizes the pool through +# the EKS API and cluster-autoscaler follows, and no apply ever sends a stale +# desired size back to the group. + +locals { + # Pinned to the control plane's minor, as upstream recommends. Add the next + # minor here before bumping `kubernetes_version` on a cluster that runs it; + # the precondition on the release fails the plan until then. Upstream marks + # --scale-down-enabled (set below) deprecated, so check each new minor still + # accepts it. The chart's RBAC is built for appVersion 1.35.0 (chart 9.59.0), + # a minor behind the 1.36 image; check the cluster role against each new + # image too. + cluster_autoscaler_image_tags = { + "1.35" = "v1.35.2" + "1.36" = "v1.36.1" + } + + # The default cutoff, spelled out so it's a reviewable literal rather than an + # upstream default that can move under a chart bump. Pods below it are + # expendable: cluster-autoscaler adds no node to run one and doesn't count one + # as a reason to keep a node. Every PriorityClass a general-pool workload + # names has to sit at or above it, the SPACE ones in + # infra/space/kind/manifests/priorityclasses.yaml included. + cluster_autoscaler_expendable_pods_priority_cutoff = -10 + + # The discovery tag, unique per cluster so a tagged ASG can't be picked up by + # another cluster's autoscaler in the same account. + cluster_autoscaler_discovery_tag_key = "space.pplx.net/cluster-autoscaler/${var.cluster_name}" +} + +# Moving general_serving_node_group retargets this tag, so the old group drops +# out of discovery and the new one joins it in the same apply. +resource "aws_autoscaling_group_tag" "cluster_autoscaler" { + count = var.cluster_autoscaler == null ? 0 : 1 + + autoscaling_group_name = var.cluster_autoscaler.autoscaling_group_name + + lifecycle { + precondition { + condition = var.cluster_autoscaler.autoscaling_group_name != null + error_message = "cluster-autoscaler needs the serving node group's ASG name, and the group reports none; check the node group's health in EKS." + } + precondition { + condition = length(local.cluster_autoscaler_discovery_tag_key) <= 128 + error_message = "The discovery tag key ${local.cluster_autoscaler_discovery_tag_key} is longer than the 128 characters AWS allows; the cluster name is too long for it." + } + } + + tag { + key = local.cluster_autoscaler_discovery_tag_key + value = "owned" + propagate_at_launch = false + } +} + +data "aws_iam_policy_document" "cluster_autoscaler_assume_role" { + count = var.cluster_autoscaler == null ? 0 : 1 + + statement { + actions = ["sts:AssumeRoleWithWebIdentity"] + + principals { + type = "Federated" + identifiers = [var.oidc_provider_arn] + } + + condition { + test = "StringEquals" + variable = "${var.oidc_issuer_host}:sub" + values = ["system:serviceaccount:kube-system:cluster-autoscaler"] + } + + condition { + test = "StringEquals" + variable = "${var.oidc_issuer_host}:aud" + values = ["sts.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "cluster_autoscaler" { + count = var.cluster_autoscaler == null ? 0 : 1 + + name = var.cluster_autoscaler.role_name + assume_role_policy = data.aws_iam_policy_document.cluster_autoscaler_assume_role[0].json +} + +data "aws_iam_policy_document" "cluster_autoscaler" { + count = var.cluster_autoscaler == null ? 0 : 1 + + # The Describe and Get calls take no resource-level scoping. + statement { + actions = [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeScalingActivities", + "autoscaling:DescribeTags", + "ec2:DescribeImages", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplateVersions", + "ec2:GetInstanceTypesFromInstanceRequirements", + ] + resources = ["*"] + } + + statement { + actions = ["eks:DescribeNodegroup"] + resources = [var.cluster_autoscaler.node_group_arn] + } + + statement { + actions = [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + ] + resources = [var.cluster_autoscaler.autoscaling_group_arn] + } +} + +resource "aws_iam_role_policy" "cluster_autoscaler" { + count = var.cluster_autoscaler == null ? 0 : 1 + + name = var.cluster_autoscaler.role_name + role = aws_iam_role.cluster_autoscaler[0].id + policy = data.aws_iam_policy_document.cluster_autoscaler[0].json +} + +resource "helm_release" "cluster_autoscaler" { + count = var.cluster_autoscaler == null ? 0 : 1 + + name = "cluster-autoscaler" + repository = "https://kubernetes.github.io/autoscaler" + chart = "cluster-autoscaler" + namespace = "kube-system" + version = "9.59.0" + + values = [yamlencode({ + cloudProvider = "aws" + awsRegion = var.aws_region + fullnameOverride = "cluster-autoscaler" + # The chart renders `--node-group-auto-discovery=asg:tag=` once + # clusterName is set; `tags` replaces the stock pair rather than adding to + # it, so only the ASG carrying this key matches. + autoDiscovery = { + clusterName = var.cluster_name + tags = [local.cluster_autoscaler_discovery_tag_key] + } + image = { + tag = lookup(local.cluster_autoscaler_image_tags, var.cluster_autoscaler.kubernetes_version, null) + } + # A pod with an emptyDir (a scratch or upload spool, say) would otherwise + # pin its node: skip-nodes-with-local-storage keeps any such node, and a + # workload that runs one on most general nodes would pin most of the pool. + # An emptyDir dies with its pod on every rollout anyway. + # skip-nodes-with-system-pods only delays kube-system pods with no PDB: once + # one is older than --blocking-system-pod-distruption-timeout (default 1h) + # it's evicted like any other. The Deployments in the kube-system Helm + # releases this module installs on general all carry budgets; give any the + # calling root adds there one too before turning scale-down on. The flip + # side: a single-replica `maxUnavailable: 1` budget allows no disruption + # while its pod isn't Ready, and the pdb drainability rule blocks the node + # on that with no timeout, whatever the budget's + # `unhealthyPodEvictionPolicy`, so a crashlooping singleton pins its node + # from scale-down until it recovers. Other drains go through the eviction + # API, which honors that policy: every single-replica budget here comes + # from the in-repo chart, which sets `AlwaysAllow`, so an unready singleton + # never blocks a drain. + # + # The drain cap and the scale-up delay are added only where a cluster sets + # them, so a cluster that doesn't keeps the upstream defaults (600s and 0s) + # with no change to its values. A null can't stand in for "unset": the chart + # renders a null extraArgs entry as the bare flag. + extraArgs = merge( + { + "scale-down-enabled" = var.cluster_autoscaler.scale_down_enabled + "scale-down-unneeded-time" = "30m" + "scale-down-delay-after-add" = "30m" + "skip-nodes-with-system-pods" = true + "skip-nodes-with-local-storage" = false + "expendable-pods-priority-cutoff" = local.cluster_autoscaler_expendable_pods_priority_cutoff + }, + # Caps each evicted pod's grace period during a scale-down drain, and + # bounds how long the drain waits for the pods to go (the cap plus 30s). + # A pod with a shorter grace keeps its own. Upstream refuses to start if + # --drain-priority-config is also set. + var.cluster_autoscaler.max_graceful_termination_sec == null ? {} : { + "max-graceful-termination-sec" = var.cluster_autoscaler.max_graceful_termination_sec + }, + # Pods younger than this aren't considered for scale-up, so a rollout + # surge pod whose room a terminating pod frees within the delay doesn't + # add a node. + var.cluster_autoscaler.new_pod_scale_up_delay == null ? {} : { + "new-pod-scale-up-delay" = var.cluster_autoscaler.new_pod_scale_up_delay + }, + ) + priorityClassName = "system-cluster-critical" + # One replica, and a budget that lets it go: the pod lives on the pool it + # scales, so a budget that refused its eviction would pin whichever general + # node it landed on. A rescheduled replica rebuilds its view from the API + # server and the ASG, losing in-memory state such as scale-down timers and + # scale-up backoff. The budget comes from disruption_budgets.tf, which can + # set `AlwaysAllow`; null drops the chart's own. + replicaCount = 1 + podDisruptionBudget = null + # Memory is per cluster because cluster-autoscaler's shared informer caches + # every pod in the cluster, sandboxes included, and no flag narrows it + # (main.go builds it with only a managedFields-stripping transform). Request + # equals limit, so the pod never counts on memory its node hasn't reserved + # for it. GOMEMLIMIT at 90% of the limit makes the Go GC collect harder as + # the runtime's total memory (heap, stacks, GC metadata) nears it, instead + # of letting the initial pod list's garbage push RSS past the limit. The + # other 10% is for memory the Go runtime doesn't manage. + resources = { + requests = { + cpu = "100m" + memory = "${var.cluster_autoscaler.memory_mib}Mi" + } + limits = { + memory = "${var.cluster_autoscaler.memory_mib}Mi" + } + } + extraEnv = { + GOMEMLIMIT = "${floor(var.cluster_autoscaler.memory_mib * 0.9)}MiB" + } + nodeSelector = { + "space.pplx.net/node-pool" = "general" + } + rbac = { + serviceAccount = { + create = true + name = "cluster-autoscaler" + annotations = { + "eks.amazonaws.com/role-arn" = aws_iam_role.cluster_autoscaler[0].arn + } + } + } + })] + + lifecycle { + precondition { + condition = contains(keys(local.cluster_autoscaler_image_tags), var.cluster_autoscaler.kubernetes_version) + error_message = "cluster-autoscaler has no image pinned for Kubernetes ${var.cluster_autoscaler.kubernetes_version}; add one to local.cluster_autoscaler_image_tags." + } + # Like the other controllers here, its pod pins to the general node pool. + precondition { + condition = var.nodes_ready + error_message = "The general node pool must exist before cluster-autoscaler installs." + } + } + + depends_on = [ + aws_autoscaling_group_tag.cluster_autoscaler, + aws_iam_role_policy.cluster_autoscaler, + ] +} diff --git a/infra/space/aws/eks/modules/controllers/disruption_budgets.tf b/infra/space/aws/eks/modules/controllers/disruption_budgets.tf new file mode 100644 index 000000000..3301390cb --- /dev/null +++ b/infra/space/aws/eks/modules/controllers/disruption_budgets.tf @@ -0,0 +1,121 @@ +locals { + # The helm provider diffs a release's `values`, not the contents of a local + # chart path, so without this a template edit produces no plan diff and never + # reaches the cluster. The chart ignores the key; its only job is to change. + # Same pattern as modules/app_deploy. + disruption_budget_chart_checksum = sha256(join("", [ + for path in sort(fileset(var.disruption_budget_chart_path, "**")) : + filesha256("${var.disruption_budget_chart_path}/${path}") + ])) + + # The webhook's budget comes from its chart: with two replicas, an unready one + # still leaves a healthy count the eviction API can spend. With both unready + # the default policy refuses evictions until one recovers, because the 0.16.1 + # chart can't render `unhealthyPodEvictionPolicy`. The chart's default + # `minAvailable: 1` is cleared, since the chart renders each of the two fields + # that's set and the API rejects a budget with both. + external_secrets_webhook_disruption_budget = { + enabled = true + minAvailable = null + maxUnavailable = 1 + } + + # Single-replica controllers whose own charts can't render + # `unhealthyPodEvictionPolicy`. Under the default policy the eviction API + # lets an unready pod go only while the budget's desired-healthy count is + # above zero, and one replica at `maxUnavailable: 1` desires none, so a + # crashlooping singleton would block every drain of its node, a node-group + # roll included. + # Their budgets come from the in-repo chart instead, which sets `AlwaysAllow`, + # and their own charts render none. Each selector names the owning release, + # which also orders the budget after it: the release's upgrade removes any + # budget its chart used to render before this one installs. + singleton_disruption_budgets = merge( + { + external-secrets = { + "app.kubernetes.io/name" = "external-secrets" + "app.kubernetes.io/instance" = helm_release.external_secrets_operator.name + } + external-secrets-cert-controller = { + "app.kubernetes.io/name" = "external-secrets-cert-controller" + "app.kubernetes.io/instance" = helm_release.external_secrets_operator.name + } + }, + var.cluster_autoscaler == null ? {} : { + # The chart names its pods after the cloud provider, not the release. + cluster-autoscaler = { + "app.kubernetes.io/name" = "aws-cluster-autoscaler" + "app.kubernetes.io/instance" = helm_release.cluster_autoscaler[0].name + } + }, + ) + + # Preferred, not required: two webhook replicas on one node don't stop + # cluster-autoscaler from removing it. The planner marks such a node risky and + # tries it last, and the refused second eviction is retried for up to + # --max-pod-eviction-time while the first replacement becomes Ready. A + # required rule would instead leave a rollout's surge pod Pending on a + # two-node pool, where both nodes already hold a replica. + external_secrets_webhook_affinity = { + podAntiAffinity = { + preferredDuringSchedulingIgnoredDuringExecution = [{ + weight = 100 + podAffinityTerm = { + topologyKey = "kubernetes.io/hostname" + labelSelector = { + matchLabels = { + "app.kubernetes.io/name" = "external-secrets-webhook" + "app.kubernetes.io/instance" = "external-secrets" + } + } + } + }] + } + } +} + +# external-dns runs one replica and its chart renders no PodDisruptionBudget, so +# the budget comes from a chart of its own. `maxUnavailable: 1` on one replica +# allows its eviction whenever it's Ready, and the chart's `AlwaysAllow` +# unhealthy-pod policy allows it when it isn't. An eviction only delays record +# changes until the replacement starts: external-dns rebuilds its view from the +# Ingresses and the zone's TXT registry on every sync. +resource "helm_release" "external_dns_disruption_budget" { + count = var.external_dns_enabled ? 1 : 0 + + name = "external-dns-pdb" + chart = var.disruption_budget_chart_path + namespace = helm_release.external_dns[0].namespace + + values = [yamlencode({ + chart = { + checksum = local.disruption_budget_chart_checksum + } + name = "external-dns" + # The chart's selector labels: its own name and the release's. + matchLabels = { + "app.kubernetes.io/name" = helm_release.external_dns[0].chart + "app.kubernetes.io/instance" = helm_release.external_dns[0].name + } + maxUnavailable = 1 + })] +} + +# Named `-singleton` so none collides with a budget the workload's own +# chart rendered before. +resource "helm_release" "singleton_disruption_budget" { + for_each = local.singleton_disruption_budgets + + name = "${each.key}-pdb" + chart = var.disruption_budget_chart_path + namespace = "kube-system" + + values = [yamlencode({ + chart = { + checksum = local.disruption_budget_chart_checksum + } + name = "${each.key}-singleton" + matchLabels = each.value + maxUnavailable = 1 + })] +} diff --git a/infra/space/aws/eks/modules/controllers/main.tf b/infra/space/aws/eks/modules/controllers/main.tf index a018091c5..fdea73bed 100644 --- a/infra/space/aws/eks/modules/controllers/main.tf +++ b/infra/space/aws/eks/modules/controllers/main.tf @@ -1,6 +1,19 @@ terraform { # Variable validations reference other variables (1.9+). required_version = ">= 1.9.0" + + required_providers { + # Lower bounds, as in the space module that calls this one. Exact + # selection belongs to the calling root and its lock file. + aws = { + source = "hashicorp/aws" + version = ">= 6.40.0" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.17.0" + } + } } # The zone's domain is read from the zone itself rather than taken as a second @@ -119,10 +132,37 @@ resource "helm_release" "external_secrets_operator" { namespace = "kube-system" version = "0.16.1" + # Each component's nodeSelector is its own key; the top-level one places only + # the controller. + # + # The chart renders only the webhook's budget; the controller and + # cert-controller get theirs from disruption_budgets.tf, because the 0.16.1 + # chart can't render `unhealthyPodEvictionPolicy`. The controller and + # cert-controller stay single replicas: evicting one only pauses secret sync + # or certificate rotation until the replacement starts. The webhook gets a + # second replica because its validating webhooks fail closed: the + # ExternalSecret one sets `failurePolicy: Fail` and the SecretStore ones take + # the API default, which is also Fail. While no replica serves, the API server + # rejects every create, update and delete of those resources, so an apply that + # creates or updates one in a chart that carries them (space-secrets-bootstrap, + # or one the calling root adds) fails. values = [yamlencode({ nodeSelector = { "space.pplx.net/node-pool" = "general" } + webhook = { + replicaCount = 2 + nodeSelector = { + "space.pplx.net/node-pool" = "general" + } + podDisruptionBudget = local.external_secrets_webhook_disruption_budget + affinity = local.external_secrets_webhook_affinity + } + certController = { + nodeSelector = { + "space.pplx.net/node-pool" = "general" + } + } })] # Every controller pins its pods to the general node pool; waiting on it # keeps a fresh cluster's releases from timing out before nodes exist. @@ -150,6 +190,18 @@ resource "helm_release" "aws_load_balancer_controller" { nodeSelector = { "space.pplx.net/node-pool" = "general" } + # The chart's two replicas keep one serving its webhooks through an + # eviction; all six (the pod, Service and TargetGroupBinding mutators and + # the Ingress, IngressClassParams and TargetGroupBinding validators) are + # `failurePolicy: Fail`. The chart's default anti-affinity only + # prefers separate nodes. When both replicas share one, cluster-autoscaler + # still removes it, evicting them one at a time. The chart renders the + # budget map as given, so it can carry `AlwaysAllow`: with both replicas + # unready, none is serving and a drain shouldn't wait for one. + podDisruptionBudget = { + maxUnavailable = 1 + unhealthyPodEvictionPolicy = "AlwaysAllow" + } serviceAccount = { create = true name = "aws-load-balancer-controller" diff --git a/infra/space/aws/eks/modules/controllers/variables.tf b/infra/space/aws/eks/modules/controllers/variables.tf index 83edcdbb1..f5391c0d4 100644 --- a/infra/space/aws/eks/modules/controllers/variables.tf +++ b/infra/space/aws/eks/modules/controllers/variables.tf @@ -49,6 +49,11 @@ variable "external_dns_role_name" { type = string } +variable "disruption_budget_chart_path" { + description = "Path to the in-repo pod-disruption-budget chart, for controllers whose own chart can't render a budget." + type = string +} + variable "resource_tags" { type = map(string) description = "Default AWS resource tags applied by in-cluster controllers." @@ -76,8 +81,46 @@ variable "aws_load_balancer_controller_resources" { } } +variable "cluster_autoscaler" { + description = "cluster-autoscaler for the one serving general node group, or null to install none of it. It scales between that group's own min and max, read live from its ASG, so a resize through the EKS API moves its bounds too." + type = object({ + role_name = string + kubernetes_version = string + autoscaling_group_arn = string + autoscaling_group_name = string + node_group_arn = string + # optional so a null takes false: the chart renders a null value as a bare + # `--scale-down-enabled`, which the flag parser reads as true. + scale_down_enabled = optional(bool, false) + memory_mib = number + # Each null leaves cluster-autoscaler's own default (600s, 0s) in force. + max_graceful_termination_sec = optional(number) + new_pod_scale_up_delay = optional(string) + }) + default = null + + validation { + condition = try(floor(var.cluster_autoscaler.memory_mib) == var.cluster_autoscaler.memory_mib && var.cluster_autoscaler.memory_mib >= 256, var.cluster_autoscaler == null) + error_message = "cluster_autoscaler.memory_mib must be set, a whole number of MiB, at least 256." + } + + validation { + # Each side carries its own try: `||` short-circuits only from Terraform + # 1.12, and this module still allows 1.9. The upper bound keeps the value + # below 1e6, from which Helm prints it in exponent form and + # cluster-autoscaler's integer flag refuses it. + condition = try(var.cluster_autoscaler.max_graceful_termination_sec == null, true) || try(floor(var.cluster_autoscaler.max_graceful_termination_sec) == var.cluster_autoscaler.max_graceful_termination_sec && var.cluster_autoscaler.max_graceful_termination_sec >= 1 && var.cluster_autoscaler.max_graceful_termination_sec <= 86400, false) + error_message = "cluster_autoscaler.max_graceful_termination_sec must be null or a whole number of seconds from 1 to 86400." + } + + validation { + condition = try(var.cluster_autoscaler.new_pod_scale_up_delay == null, true) || can(regex("^[0-9]+[sm]$", var.cluster_autoscaler.new_pod_scale_up_delay)) + error_message = "cluster_autoscaler.new_pod_scale_up_delay must be null or a whole number of seconds or minutes, such as \"90s\"." + } +} + variable "nodes_ready" { - description = "Readiness token for the general node pool that every controller release schedules onto. Pass an expression that stays unknown until the pool exists (the space module passes its general node group's ARN check) so the releases wait for it; the default suits callers that order the pool first themselves." + description = "Readiness token for the general node pool that every controller release schedules onto. Pass an expression that stays unknown until the pool exists (the space module passes its serving general node group's ARN check) so the releases wait for it; the default suits callers that order the pool first themselves." type = bool default = true nullable = false diff --git a/infra/space/aws/eks/modules/eks-cluster/node_groups.tf b/infra/space/aws/eks/modules/eks-cluster/node_groups.tf index 4846ac8ad..dea610a3f 100644 --- a/infra/space/aws/eks/modules/eks-cluster/node_groups.tf +++ b/infra/space/aws/eks/modules/eks-cluster/node_groups.tf @@ -76,7 +76,7 @@ resource "aws_launch_template" "node_group" { resource "aws_eks_node_group" "custom" { for_each = { for key, value in var.node_pools : key => value - if value.ami_id != null + if value.ami_id != null && !value.launch_template_and_role_only } cluster_name = aws_eks_cluster.this.name @@ -135,7 +135,7 @@ resource "aws_eks_node_group" "custom" { resource "aws_eks_node_group" "managed" { for_each = { for key, value in var.node_pools : key => value - if value.ami_id == null + if value.ami_id == null && !value.launch_template_and_role_only } cluster_name = aws_eks_cluster.this.name @@ -194,11 +194,13 @@ resource "aws_eks_node_group" "managed" { update = var.node_group_update_timeout } - # These groups have no autoscaler, so their size moves only when an operator - # resizes them through the EKS API. Without this, the next apply for any - # unrelated change would reset each group to its creation-time sizing and - # scale it down under whatever is running on it. The sizes in node_pools - # still decide what a new cluster comes up with. + # These groups' size moves only outside Terraform: an operator resizing them + # through the EKS API, or a cluster-autoscaler the caller runs on one of them, + # which sets the ASG's desired capacity between the min and max it reads live + # from that ASG. Without this, the next apply for any unrelated change would + # reset each group to its creation-time sizing and scale it down under + # whatever is running on it. The sizes in node_pools still decide what a new + # cluster comes up with. lifecycle { ignore_changes = [ scaling_config[0].min_size, diff --git a/infra/space/aws/eks/modules/eks-cluster/outputs.tf b/infra/space/aws/eks/modules/eks-cluster/outputs.tf index c06b47fbb..8ab301819 100644 --- a/infra/space/aws/eks/modules/eks-cluster/outputs.tf +++ b/infra/space/aws/eks/modules/eks-cluster/outputs.tf @@ -91,6 +91,17 @@ output "vpc_cni_role_arn" { value = aws_iam_role.vpc_cni.arn } +output "managed_node_groups" { + description = "EKS-managed-AMI node groups by pool: arn, the backing ASG names, and scaling_config. It depends only on those groups, so a consumer doesn't wait on custom-AMI pools." + value = { + for name, group in aws_eks_node_group.managed : name => { + arn = group.arn + asg_names = flatten([for r in group.resources : [for asg in r.autoscaling_groups : asg.name]]) + scaling_config = group.scaling_config + } + } +} + output "managed_node_group_arns" { description = "ARNs of the EKS-managed-AMI node groups by pool. Depends only on those groups, so a readiness edge through it doesn't wait on custom-AMI pools." value = { for name, group in aws_eks_node_group.managed : name => group.arn } diff --git a/infra/space/aws/eks/modules/eks-cluster/variables.tf b/infra/space/aws/eks/modules/eks-cluster/variables.tf index 2ba2dfc2b..a6a80c592 100644 --- a/infra/space/aws/eks/modules/eks-cluster/variables.tf +++ b/infra/space/aws/eks/modules/eks-cluster/variables.tf @@ -136,11 +136,19 @@ variable "node_pools" { `node_pool_user_data` must carry the pool's bootstrap script. `additional_policy_arns` keys must be known at plan time; their values may not be. For a pool with no `ami_id`, `min_size`, `desired_size`, and `max_size` apply at creation only: afterwards the - group is resized through the EKS API and Terraform ignores all three. A - custom-AMI pool keeps its size under Terraform. `update_strategy` is the - group's rolling-update strategy: null leaves EKS's default, which launches + group is resized through the EKS API (or its ASG's desired capacity by a + cluster-autoscaler the caller runs on it) and Terraform ignores all three. A custom-AMI pool + keeps its size under Terraform. `update_strategy` is the group's + rolling-update strategy: null leaves EKS's default, which launches replacements before draining; `MINIMAL` drains first, so a roll never runs more nodes than the group's size. + + `launch_template_and_role_only` true keeps only the pool's launch template, + node IAM role, and the role's policy attachments, and creates no node + group, so a node group declared outside this module can run on them (the + `launch_templates` and `node_role_arns` outputs). The pool's node-group + fields (sizes, instance and capacity types, labels, taints, subnets, + `update_strategy`) are then unused. EOT type = map(object({ instance_types = list(string) @@ -160,6 +168,7 @@ variable "node_pools" { additional_policy_arns = optional(map(string), {}) volume_tags = optional(map(string), {}) update_strategy = optional(string) + launch_template_and_role_only = optional(bool, false) })) default = {} @@ -175,11 +184,20 @@ variable "node_pools" { validation { condition = alltrue([ for name in keys(var.cluster_addons) : contains(["vpc-cni", "kube-proxy"], name) - ]) || anytrue([for pool in values(var.node_pools) : pool.desired_size > 0]) - error_message = "Addons other than vpc-cni and kube-proxy need a node pool in this module with desired_size above zero to schedule on." + ]) || var.external_node_group_hosts_addons || anytrue([ + for pool in values(var.node_pools) : !pool.launch_template_and_role_only && pool.desired_size > 0 + ]) + error_message = "Addons other than vpc-cni and kube-proxy need a node pool with a node group and desired_size above zero to schedule on: one in this module, or one declared outside it with external_node_group_hosts_addons set." } } +variable "external_node_group_hosts_addons" { + description = "Whether a node group the caller declares outside this module, on a pool's launch template and role, has nodes the deployment-backed addons (coredns, metrics-server, ...) can schedule on. Set it when that group is the only such capacity, so the addon check in node_pools counts it. This module can't wait for that group: the group takes its cluster name from this module's outputs, so the addons only become healthy once it has nodes." + type = bool + default = false + nullable = false +} + variable "node_group_update_timeout" { description = "How long Terraform waits for a node group update, such as a roll to a new launch template, before giving up. A caller whose apply runs under a shorter job limit should lower it, or the job is killed mid-apply with the state lock held." type = string diff --git a/infra/space/aws/eks/modules/space/cluster_controller_pod_identity.tf b/infra/space/aws/eks/modules/space/cluster_controller_pod_identity.tf index 2ce79f67c..606289705 100644 --- a/infra/space/aws/eks/modules/space/cluster_controller_pod_identity.tf +++ b/infra/space/aws/eks/modules/space/cluster_controller_pod_identity.tf @@ -45,11 +45,11 @@ locals { aws_eks_node_group.rollout_managed, ) - # Names of every ASG backing a SPACE worker node group. A single EKS - # node group can publish multiple ASGs when the instance-type list is - # heterogeneous (one ASG per AZ in some cases), so flatten across both - # `resources[*].autoscaling_groups[*]` AND across pools. `flatten` is - # recursive, so the triple nesting collapses to a flat list. + # Names of every ASG backing a SPACE worker node group. EKS backs each + # managed node group with one ASG spanning all its subnets, but the provider + # exposes it as a list under `resources[*].autoscaling_groups[*]`, so flatten + # across that nesting AND across pools. `flatten` is recursive, so the triple + # nesting collapses to a flat list. # # Three sources, concatenated then flattened: # @@ -88,12 +88,10 @@ locals { var.additional_worker_asg_names, )) - # IAM resource ARNs for those ASGs. The middle `*` stands in for the - # per-ASG UUID, which AWS assigns at creation time and is not part of the - # human-readable name. + # IAM resource ARNs for those ASGs. worker_capacity_asg_arns = [ for name in values(local.worker_capacity_asg_names) : - "arn:${data.aws_partition.current.partition}:autoscaling:${var.aws_region}:${data.aws_caller_identity.current.account_id}:autoScalingGroup:*:autoScalingGroupName/${name}" + "${local.autoscaling_group_arn_prefix}${name}" ] worker_capacity_nodegroup_arns = [ @@ -103,13 +101,13 @@ locals { cluster_controller_worker_asg_arns = concat( [ for name in local.cluster_controller_worker_asg_names : - "arn:${data.aws_partition.current.partition}:autoscaling:${var.aws_region}:${data.aws_caller_identity.current.account_id}:autoScalingGroup:*:autoScalingGroupName/${name}" + "${local.autoscaling_group_arn_prefix}${name}" ], # Root-declared worker ASGs whose names are unknown until apply: fine here # because this is an IAM policy `resources` value, not a for_each key. [ for name in var.additional_worker_asg_names_policy_only : - "arn:${data.aws_partition.current.partition}:autoscaling:${var.aws_region}:${data.aws_caller_identity.current.account_id}:autoScalingGroup:*:autoScalingGroupName/${name}" + "${local.autoscaling_group_arn_prefix}${name}" ], local.worker_capacity_asg_arns, ) diff --git a/infra/space/aws/eks/modules/space/controllers.tf b/infra/space/aws/eks/modules/space/controllers.tf index b55bef459..16196c22c 100644 --- a/infra/space/aws/eks/modules/space/controllers.tf +++ b/infra/space/aws/eks/modules/space/controllers.tf @@ -1,3 +1,28 @@ +locals { + # Every general MNG this cluster declares, keyed by name: general unless it's + # retired, and general-2 when it's declared. general comes from the + # managed-group output rather than node_groups, which merges every pool and + # would hold its consumers behind a custom-AMI pool update. Consumers still + # wait on both general pools whichever one serves: Terraform orders by + # reference, not by the branch a value takes, and they must be able to follow + # general-2 once general is retired. + general_node_groups = merge( + { for name, group in module.eks.managed_node_groups : name => group if name == "general" }, + { + for group in aws_eks_node_group.general_blue_green : "general-2" => { + arn = group.arn + asg_names = flatten([for r in group.resources : [for asg in r.autoscaling_groups : asg.name]]) + scaling_config = group.scaling_config + } + }, + ) + + # The one general_serving_node_group names. Its validations guarantee the + # key exists: general can only be retired while general-2 serves, and + # general-2 can only serve once it's declared. + serving_general_node_group = local.general_node_groups[var.general_serving_node_group] +} + module "controllers" { source = "../controllers" @@ -11,9 +36,28 @@ module "controllers" { cluster_vpc_id = data.aws_vpc.shared.id alb_controller_role_name = "${local.name}-aws-load-balancer-controller" external_dns_role_name = "${local.name}-external-dns" + disruption_budget_chart_path = "${var.charts_dir}/pod-disruption-budget" resource_tags = var.common_tags aws_load_balancer_controller_resources = var.aws_load_balancer_controller_resources - # Through the managed-group output rather than node_groups, which merges - # every pool and would hold the releases behind a custom-AMI pool update. - nodes_ready = module.eks.managed_node_group_arns["general"] != "" + # The serving general group carries the general pool's labels, so it's the + # one the releases schedule onto, and once general is retired it's the only + # one. + nodes_ready = local.serving_general_node_group.arn != "" + + # A managed node group is backed by one ASG. one() fails the plan if it + # reports several, and gives null if it reports none, which the controllers + # module's precondition on the discovery tag turns into a named error. The + # ARN is "" then, so it doesn't fail first on interpolating the null. + cluster_autoscaler = var.cluster_autoscaler_enabled ? { + role_name = "${local.name}-cluster-autoscaler" + kubernetes_version = var.kubernetes_version + autoscaling_group_arn = one(local.serving_general_node_group.asg_names) == null ? "" : "${local.autoscaling_group_arn_prefix}${one(local.serving_general_node_group.asg_names)}" + autoscaling_group_name = one(local.serving_general_node_group.asg_names) + node_group_arn = local.serving_general_node_group.arn + scale_down_enabled = var.cluster_autoscaler_scale_down_enabled + memory_mib = var.cluster_autoscaler_memory_mib + + max_graceful_termination_sec = var.cluster_autoscaler_max_graceful_termination_sec + new_pod_scale_up_delay = var.cluster_autoscaler_new_pod_scale_up_delay + } : null } diff --git a/infra/space/aws/eks/modules/space/eks.tf b/infra/space/aws/eks/modules/space/eks.tf index 5ea06f528..708e30c68 100644 --- a/infra/space/aws/eks/modules/space/eks.tf +++ b/infra/space/aws/eks/modules/space/eks.tf @@ -25,6 +25,10 @@ module "eks" { node_pools = local.core_node_pools node_pool_user_data = local.core_node_pool_user_data + + # Once general-2 serves, it's general capacity the addons can run on, even + # with general retired. + external_node_group_hosts_addons = var.general_serving_node_group == "general-2" } # Joins the general and cred-authority pools' launch templates through diff --git a/infra/space/aws/eks/modules/space/locals.tf b/infra/space/aws/eks/modules/space/locals.tf index c3a37ec64..f7ff91690 100644 --- a/infra/space/aws/eks/modules/space/locals.tf +++ b/infra/space/aws/eks/modules/space/locals.tf @@ -8,6 +8,11 @@ locals { cluster_id = coalesce(var.cluster_id, var.environment) name = "${var.name_prefix}-${local.cluster_id}" + # An ASG's IAM resource ARN is this prefix plus its name. The `*` stands in + # for the per-ASG UUID, which AWS assigns at creation time and isn't part of + # the human-readable name. + autoscaling_group_arn_prefix = "arn:${data.aws_partition.current.partition}:autoscaling:${var.aws_region}:${data.aws_caller_identity.current.account_id}:autoScalingGroup:*:autoScalingGroupName/" + azs = var.availability_zones @@ -311,6 +316,8 @@ locals { # An ownership-tag policy can require Team on EBS root volumes; common_tags # already supplies the calling root's other ownership tags. volume_tags = { Team = "team-space" } + # general-2 runs on a retired general's launch template and node role. + launch_template_and_role_only = name == "general" && var.general_node_group_retired } } diff --git a/infra/space/aws/eks/modules/space/node_groups.tf b/infra/space/aws/eks/modules/space/node_groups.tf index 2bbf7599c..b6d60ebab 100644 --- a/infra/space/aws/eks/modules/space/node_groups.tf +++ b/infra/space/aws/eks/modules/space/node_groups.tf @@ -280,8 +280,10 @@ resource "aws_eks_node_group" "general_blue_green" { } # After creation, operators scale this pool between zero and its active - # target through EKS. Unrelated Terraform applies must not wake a dormant - # pool or scale down one that is carrying the active general workload. + # target through EKS, and while it serves, cluster-autoscaler moves its + # desired size inside those bounds. Unrelated Terraform applies must not + # wake a dormant pool or scale down one that is carrying the active general + # workload. lifecycle { ignore_changes = [ scaling_config[0].min_size, @@ -291,6 +293,63 @@ resource "aws_eks_node_group" "general_blue_green" { } } +# general_node_group_retired deletes the general MNG together with any node it +# still runs, so the general_serving_node_group output refuses the plan until +# general is parked: its ASG wants no instances and none are left. Each lookup +# below runs only if the one before it found its target, and lists rather than +# reads, so none fails on a missing cluster or node group. Once retirement has +# landed, general isn't among the cluster's node groups and the check passes. +# The cluster name is read from local.name rather than module.eks so that the +# lookups run at plan time even when the cluster itself has pending changes. +# +# Retirement also needs a cluster that exists. A new cluster comes up with +# general: its deployment-backed addons wait only on node groups inside +# module.eks, while general-2 waits only on the pre-node addons (vpc-cni, +# kube-proxy), so a cluster created retired would install its other addons +# with no node group they wait for. +data "aws_eks_clusters" "retiring_general" { + count = var.general_node_group_retired ? 1 : 0 +} + +data "aws_eks_node_groups" "retiring_general" { + count = length([for clusters in data.aws_eks_clusters.retiring_general : true if contains(clusters.names, local.name)]) + + cluster_name = local.name +} + +data "aws_eks_node_group" "retiring_general" { + count = length([for groups in data.aws_eks_node_groups.retiring_general : true if contains(groups.names, "general")]) + + cluster_name = local.name + node_group_name = "general" +} + +locals { + retiring_general_asg_names = toset(flatten([ + for group in data.aws_eks_node_group.retiring_general : [ + for r in group.resources : [for asg in r.autoscaling_groups : asg.name] + ] + ])) +} + +data "aws_autoscaling_group" "retiring_general" { + for_each = local.retiring_general_asg_names + + name = each.value +} + +# A desired capacity of zero only says the ASG has stopped asking for nodes; +# the ones it's removing can still be draining. Every instance still alive in +# the ASG counts until it's gone. +data "aws_instances" "retiring_general" { + for_each = local.retiring_general_asg_names + + instance_tags = { + "aws:autoscaling:groupName" = each.value + } + instance_state_names = ["pending", "running", "shutting-down", "stopping", "stopped"] +} + data "aws_iam_policy_document" "irsa_assume_role" { # Excludes `workers-cgnat` unconditionally — its IRSA role is now # owned by the standalone `aws_iam_role.workers_service_account` diff --git a/infra/space/aws/eks/modules/space/outputs.tf b/infra/space/aws/eks/modules/space/outputs.tf index 2397edad2..e0dec1df8 100644 --- a/infra/space/aws/eks/modules/space/outputs.tf +++ b/infra/space/aws/eks/modules/space/outputs.tf @@ -101,24 +101,34 @@ output "general_serving_node_group" { precondition { # Existing single-pool environments retain their positive-capacity gate. - # Blue/green configurations additionally enforce the operator-recorded floor. - condition = var.general_blue_green_node_group == null ? ( - module.eks.node_groups["general"].scaling_config[0].min_size > 0 - ) : var.general_serving_node_group == "general" ? ( - module.eks.node_groups["general"].scaling_config[0].min_size >= var.general_min_size - ) : ( - aws_eks_node_group.general_blue_green[0].scaling_config[0].min_size >= var.general_blue_green_node_group.recorded_active_size + # Blue/green configurations additionally enforce the operator-recorded + # floor on the serving pool. The sizes are the live ones: Terraform + # ignores them after creation, so the plan carries what EKS reports. + condition = local.serving_general_node_group.scaling_config[0].min_size >= ( + var.general_blue_green_node_group == null ? 1 : + var.general_serving_node_group == "general" ? var.general_min_size : + var.general_blue_green_node_group.recorded_active_size ) error_message = "The recorded general_serving_node_group must be live and, for blue/green configurations, meet its recorded capacity floor." } + + precondition { + condition = alltrue([ + for name, asg in data.aws_autoscaling_group.retiring_general : + asg.desired_capacity == 0 && length(data.aws_instances.retiring_general[name].ids) == 0 + ]) + error_message = "general_node_group_retired deletes the general node group, which still wants or runs instances. Cut over to general-2 in one apply, then scale general to zero through EKS and wait for its nodes to leave, then retire it." + } + + precondition { + condition = alltrue([for clusters in data.aws_eks_clusters.retiring_general : contains(clusters.names, local.name)]) + error_message = "general_node_group_retired retires a general that exists, and cluster ${local.name} doesn't exist yet. Create it with general, cut over to general-2, park general at zero, then retire it." + } } output "general_serving_min_size" { - description = "Larger configured floor of the two permanent general pools. Either may carry the active workload, so a root-owned DaemonSet that must roll on the general pool gates on this while the inactive pool stays at zero; referencing it also orders the consumer after a floor update." - value = max( - module.eks.node_groups["general"].scaling_config[0].min_size, - try(aws_eks_node_group.general_blue_green[0].scaling_config[0].min_size, 0), - ) + description = "Larger live floor of the declared general pools. Either may carry the active workload, so a root-owned DaemonSet that must roll on the general pool gates on this while the inactive pool stays at zero; referencing it also orders the consumer after a floor update." + value = max([for group in values(local.general_node_groups) : group.scaling_config[0].min_size]...) } output "vpc_cni_service_account_role_arn" { diff --git a/infra/space/aws/eks/modules/space/variables.tf b/infra/space/aws/eks/modules/space/variables.tf index c993b255b..9a73223c0 100644 --- a/infra/space/aws/eks/modules/space/variables.tf +++ b/infra/space/aws/eks/modules/space/variables.tf @@ -284,6 +284,43 @@ variable "external_dns_records_enabled" { } } +variable "cluster_autoscaler_enabled" { + description = "Whether to run cluster-autoscaler on the serving general node group (general_serving_node_group), bounded by that group's own min and max size as EKS reports them, so resizing the group through the EKS API moves the bounds. The serving pool's recorded floor (general_min_size, or general_blue_green_node_group.recorded_active_size) still has to hold for that min, so lower the record before lowering the min. It never touches any other pool." + type = bool + default = false + nullable = false +} + +variable "cluster_autoscaler_memory_mib" { + description = "cluster-autoscaler's memory request and limit, in MiB. Read only when cluster_autoscaler_enabled, which requires it." + type = number + default = null + + validation { + condition = !var.cluster_autoscaler_enabled || try(floor(var.cluster_autoscaler_memory_mib) == var.cluster_autoscaler_memory_mib && var.cluster_autoscaler_memory_mib >= 256, false) + error_message = "cluster_autoscaler_memory_mib must be set when cluster_autoscaler_enabled, to a whole number of MiB, at least 256." + } +} + +variable "cluster_autoscaler_scale_down_enabled" { + description = "Whether cluster-autoscaler may remove serving general nodes it finds unneeded. Off, it still adds nodes for Pending pods and still removes instances that never registered or failed to create. Read only when cluster_autoscaler_enabled." + type = bool + default = false + nullable = false +} + +variable "cluster_autoscaler_max_graceful_termination_sec" { + description = "The longest grace period, in seconds, cluster-autoscaler gives a pod it evicts to scale down a serving general node. A pod with a shorter terminationGracePeriodSeconds keeps its own. Null leaves the upstream default, 600. Read only when cluster_autoscaler_enabled." + type = number + default = null +} + +variable "cluster_autoscaler_new_pod_scale_up_delay" { + description = "How old a Pending pod must be before cluster-autoscaler adds a node for it, as a duration such as \"90s\". Null leaves the upstream default, 0s. Read only when cluster_autoscaler_enabled." + type = string + default = null +} + variable "cluster_service_cidr" { description = "Kubernetes service CIDR used when bootstrapping custom AMIs" type = string @@ -782,7 +819,7 @@ variable "general_taints" { } variable "general_blue_green_node_group" { - description = "Optional permanent general-2 MNG used for additive blue/green swaps. recorded_active_size creates the pool initially and records its operator-managed active target for capacity checks." + description = "Optional permanent general-2 MNG used for additive blue/green swaps. recorded_active_size creates the pool initially and records its operator-managed active target for capacity checks. After creation the pool is sized only through the EKS API, by operators or, while it serves, cluster-autoscaler." type = object({ instance_types = list(string) recorded_active_size = number @@ -803,7 +840,7 @@ variable "general_blue_green_node_group" { } variable "general_serving_node_group" { - description = "General MNG that is currently intended to serve SPACE workloads. Keep this aligned with the operator-managed blue/green cutover so capacity checks use the live node shape." + description = "General MNG that is currently intended to serve SPACE workloads. Keep this aligned with the blue/green cutover so capacity checks use the live node shape. cluster-autoscaler, where enabled, scales this group and only this group, so changing it retargets the autoscaler in the same apply." type = string default = "general" @@ -816,6 +853,18 @@ variable "general_serving_node_group" { } } +variable "general_node_group_retired" { + description = "Whether the original `general` MNG has been removed, leaving general-2 as the only general pool. Its launch template, node IAM role, and service-account role stay, because general-2 runs on the first two. Set `true` only once general-2 serves and general is drained and parked at zero nodes. The plan fails while general's ASG still wants or runs any instance, and on a cluster that doesn't exist yet: a new cluster comes up with general, because the cluster's addons wait only on node groups inside module.eks. Those checks read AWS at plan time, so while retiring, don't put this module behind a module-level depends_on, which defers the reads and fails the plan." + type = bool + default = false + nullable = false + + validation { + condition = !var.general_node_group_retired || var.general_serving_node_group == "general-2" + error_message = "general_node_group_retired needs general_serving_node_group = \"general-2\"." + } +} + variable "general_service_account_name" { description = "Service account name for general workloads" type = string