feat: verify the aggregated API server certificate through the APIService caBundle

.cspell.json

@@ -56,7 +56,8 @@
     "portforward",
     "livez",
     "requestheader",
-    "subjectaccessreviews"
+    "subjectaccessreviews",
+    "cabundle"
   ],
   "ignorePaths": [
     ".git/**",

.github/workflows/ci.yaml

@@ -320,6 +320,36 @@ jobs:
           kubectl wait --for=condition=Available deploy/coder-k8s -n coder-system --timeout=120s
           kubectl wait --for=condition=Available apiservice/v1alpha1.aggregation.coder.com --timeout=180s
 
+      # The aggregated API server sets the APIService caBundle from its CA Secret and turns
+      # insecureSkipTLSVerify off shortly after it starts. kube-apiserver's Available check does
+      # not verify the serving certificate, so the proof is a request proxied with verification on.
+      - name: Verify the APIService trusts the serving CA
+        if: env.E2E_FULL == 'true'
+        run: |
+          set -euo pipefail
+          apisvc=apiservice/v1alpha1.aggregation.coder.com
+          want="" have="" insecure=""
+          for _ in $(seq 1 60); do
+            want=$(kubectl -n coder-system get secret coder-k8s-apiserver-tls -o jsonpath='{.data.ca\.crt}' 2>/dev/null || true)
+            have=$(kubectl get "$apisvc" -o jsonpath='{.spec.caBundle}')
+            insecure=$(kubectl get "$apisvc" -o jsonpath='{.spec.insecureSkipTLSVerify}')
+            if [[ -n $want && $have == "$want" && -z $insecure ]]; then
+              break
+            fi
+            sleep 2
+          done
+          if [[ -z $want || $have != "$want" || -n $insecure ]]; then
+            echo "assertion failed: APIService caBundle does not match the serving CA Secret (insecureSkipTLSVerify='$insecure')" >&2
+            kubectl get "$apisvc" -o yaml >&2
+            exit 1
+          fi
+          kubectl wait --for=condition=Available "$apisvc" --timeout=180s
+          for _ in $(seq 1 30); do
+            kubectl get --raw /apis/aggregation.coder.com/v1alpha1 >/dev/null && break
+            sleep 2
+          done
+          kubectl get --raw /apis/aggregation.coder.com/v1alpha1 >/dev/null
+
       - name: Install CloudNativePG operator
         if: env.E2E_FULL == 'true'
         run: |
@@ -430,6 +460,55 @@ jobs:
           E2E_WORKDIR: ${{ runner.temp }}/workspace-lifecycle
         run: bash ./hack/e2e-workspace-lifecycle.sh
 
+      # Last functional step: with the sync's permission revoked (so it cannot repair the value),
+      # a wrong caBundle makes requests proxied by kube-apiserver fail, and kube-apiserver logs the
+      # x509 error (its Available check does not verify certificates). Restoring the permission
+      # lets the sync put the right CA back.
+      - name: Verify the APIService rejects a wrong CA
+        if: env.E2E_FULL == 'true'
+        run: |
+          set -euo pipefail
+          apisvc=apiservice/v1alpha1.aggregation.coder.com
+          can_patch() {
+            kubectl auth can-i patch apiservices.apiregistration.k8s.io/v1alpha1.aggregation.coder.com \
+              --as=system:serviceaccount:coder-system:coder-k8s 2>/dev/null || true
+          }
+          kubectl delete clusterrolebinding coder-k8s-apiservice-cabundle
+          for _ in $(seq 1 30); do
+            [[ $(can_patch) == no ]] && break
+            sleep 1
+          done
+          test "$(can_patch)" = no
+          wrong=$(openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -keyout /dev/null \
+            -days 1 -subj /CN=e2e-wrong-ca 2>/dev/null | base64 -w0)
+          since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+          kubectl patch "$apisvc" --type=merge -p "{\"spec\":{\"caBundle\":\"$wrong\"}}"
+          for _ in $(seq 1 30); do
+            kubectl get --raw /apis/aggregation.coder.com/v1alpha1 >/dev/null 2>&1 || break
+            sleep 1
+          done
+          if kubectl get --raw /apis/aggregation.coder.com/v1alpha1 >/dev/null; then
+            echo "assertion failed: aggregated API still served with a wrong caBundle" >&2
+            exit 1
+          fi
+          x509=""
+          for _ in $(seq 1 30); do
+            x509=$(kubectl -n kube-system logs -l component=kube-apiserver --since-time="$since" --tail=-1 |
+              grep 'v1alpha1.aggregation.coder.com' | grep -m1 'x509: certificate signed by unknown authority' || true)
+            [[ -n $x509 ]] && break
+            sleep 2
+          done
+          echo "${x509:?assertion failed: kube-apiserver logged no x509 error for the wrong caBundle}"
+          kubectl apply -f config/rbac/apiservice-cabundle-role.yaml
+          want=$(kubectl -n coder-system get secret coder-k8s-apiserver-tls -o jsonpath='{.data.ca\.crt}')
+          for _ in $(seq 1 90); do
+            [[ $(kubectl get "$apisvc" -o jsonpath='{.spec.caBundle}') == "$want" ]] &&
+              kubectl get --raw /apis/aggregation.coder.com/v1alpha1 >/dev/null 2>&1 && break
+            sleep 2
+          done
+          test "$(kubectl get "$apisvc" -o jsonpath='{.spec.caBundle}')" = "$want"
+          kubectl get --raw /apis/aggregation.coder.com/v1alpha1 >/dev/null
+
       # ---- Failure diagnostics ----
       - name: Dump cluster state on failure
         if: failure() && env.E2E_FULL == 'true'

config/rbac/apiservice-cabundle-role.yaml

@@ -0,0 +1,33 @@
+# Lets the aggregated API server keep its own APIService trusting its serving CA
+# (spec.caBundle, spec.insecureSkipTLSVerify). Limited to that one APIService; list and watch are
+# authorized only for requests that select it by metadata.name. Not part of dist/install.yaml,
+# which installs the controller only.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: coder-k8s-apiservice-cabundle
+rules:
+  - apiGroups:
+      - apiregistration.k8s.io
+    resources:
+      - apiservices
+    resourceNames:
+      - v1alpha1.aggregation.coder.com
+    verbs:
+      - get
+      - list
+      - watch
+      - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: coder-k8s-apiservice-cabundle
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: coder-k8s-apiservice-cabundle
+subjects:
+  - kind: ServiceAccount
+    name: coder-k8s
+    namespace: coder-system

deploy/apiserver-apiservice.yaml

@@ -10,4 +10,3 @@ spec:
     namespace: coder-system
   groupPriorityMinimum: 1000
   versionPriority: 100
-  insecureSkipTLSVerify: true

docs/how-to/deploy-aggregated-apiserver.md

@@ -118,12 +118,50 @@ In a cluster, the aggregated API server serves a certificate signed by its own C
 
 - The server creates the Secret on first start and reuses it afterwards. With several replicas, they all use the same Secret.
 - The serving certificate is valid for 1 year. The server checks it at startup and every 12 hours, and renews it with the same CA when less than a third of its lifetime is left. The new certificate is served without a restart.
-- The CA is valid for 10 years. To replace it, delete the Secret and restart the Deployment (`kubectl -n coder-system rollout restart deployment/coder-k8s`). Clients that trusted the old CA must then trust the new one.
+- The CA is valid for 10 years. To replace it, delete the Secret and restart **every** replica (`kubectl -n coder-system rollout restart deployment/coder-k8s`). The first new pod generates a CA and updates the APIService `caBundle`; until the old pods are gone, requests routed to them fail certificate verification, so expect `503 ServiceUnavailable` for several seconds (about 10 seconds with two replicas in testing). Other clients that trusted the old CA must then trust the new one.
 - If the Secret exists but is unusable (a missing key, unparsable PEM, a key that does not match its certificate, a serving certificate not signed by the CA, or an expired CA), the server does not start and the log names the field. Fix the Secret or delete it.
 - Outside a cluster (for example `go run`), the server serves a self-signed certificate for `localhost` instead.
 
 !!! warning "The Secret holds the CA private key"
     Anyone who can read Secrets in the server's namespace can issue certificates that the aggregated API server's CA vouches for. Restrict Secret read access in `coder-system` accordingly.
 
-!!! warning "TLS verification is still off"
-    `deploy/apiserver-apiservice.yaml` still sets `insecureSkipTLSVerify: true`, so kube-apiserver does not check this certificate yet. Registering the CA in the APIService `caBundle` is tracked in [#137](https://github.com/coder/coder-k8s/issues/137).
+## How kube-apiserver trusts the server
+
+kube-apiserver verifies the aggregated API server's certificate against the APIService `spec.caBundle`. The aggregated API server keeps that field set to the CA in `coder-k8s-apiserver-tls`, and keeps `insecureSkipTLSVerify` off:
+
+- It patches only the APIService `v1alpha1.aggregation.coder.com`, and only when the `caBundle` differs from the Secret's CA or `insecureSkipTLSVerify` is set. It reacts to changes of that APIService and of its own CA; it does not poll.
+- It writes only a CA that passed the same checks as at startup. If the Secret is missing or invalid, it leaves the APIService unchanged and logs why.
+- It needs `config/rbac/apiservice-cabundle-role.yaml`: `get`, `list`, `watch`, and `patch` on that one APIService (`resourceNames`). `list` and `watch` are allowed only for requests that select it by name. The controller-only install bundle (`dist/install.yaml`) does not include this file, because it registers no APIService.
+- Without that permission the aggregated API server keeps serving, logs the missing permission (at most every 5 minutes), and retries with backoff up to 60 seconds.
+- On a fresh install, requests through kube-apiserver fail with `503 ServiceUnavailable` for a few seconds, until the first patch. `Available=True` alone does not show that verification works, because kube-apiserver's availability check does not verify the certificate. Check a real request instead: `kubectl get --raw /apis/aggregation.coder.com/v1alpha1`.
+- Running `kubectl apply -f deploy/apiserver-apiservice.yaml` again keeps the injected `caBundle`. Replacing or re-creating the APIService clears it; the aggregated API server sets it again within seconds.
+
+### Upgrade from a version that used `insecureSkipTLSVerify`
+
+Apply the changes in this order:
+
+1. `kubectl apply -f config/rbac/`
+2. Deploy the new image. It sets `caBundle` and turns `insecureSkipTLSVerify` off in one step.
+3. `kubectl apply -f deploy/apiserver-apiservice.yaml` (the file no longer sets `insecureSkipTLSVerify`).
+
+If the new image starts before step 1, it logs a missing-permission error until the RBAC exists, and the APIService keeps working without verification in the meantime. If you apply step 3 before the new image runs, requests through kube-apiserver fail with `503` until the new image starts. Do not re-apply an old copy of `deploy/apiserver-apiservice.yaml`: once a `caBundle` is set, the API rejects `insecureSkipTLSVerify: true`.
+
+To roll back to a version without a managed serving certificate, restore the old registration before you change the image. The opt-out annotation stops the running server from setting the `caBundle` again:
+
+```bash
+kubectl annotate apiservice v1alpha1.aggregation.coder.com coder.com/manage-ca-bundle=false
+kubectl patch apiservice v1alpha1.aggregation.coder.com --type=merge \
+  -p '{"spec":{"caBundle":null,"insecureSkipTLSVerify":true}}'
+```
+
+Then deploy the old image and re-apply its `deploy/apiserver-apiservice.yaml`.
+
+### Opt out
+
+If another tool owns the APIService `caBundle` (for example cert-manager's CA injector, or a GitOps tool that sets it from Git), annotate the APIService so the aggregated API server leaves it alone:
+
+```bash
+kubectl annotate apiservice v1alpha1.aggregation.coder.com coder.com/manage-ca-bundle=false
+```
+
+That tool must then trust a CA that signed the certificate the server actually serves, which is the one in `coder-k8s-apiserver-tls`. Remove the annotation (or set any other value) to hand the field back.

hack/update-manifests.sh

@@ -29,7 +29,7 @@ list_manifests() {
 }
 
 # RBAC manifests that only the aggregated API server needs; the controller-only install bundle leaves them out.
-BUNDLE_EXCLUDED_RBAC=(auth-delegator-binding.yaml authentication-reader-binding.yaml)
+BUNDLE_EXCLUDED_RBAC=(apiservice-cabundle-role.yaml auth-delegator-binding.yaml authentication-reader-binding.yaml)
 
 # write_default_kustomization lists every generated CRD and RBAC file individually, so the install bundle
 # picks up new files automatically without placing kustomization files in the directories above. The bundle