🤖 feat: serve the aggregated API server certificate from a managed CA

docs/how-to/deploy-aggregated-apiserver.md

@@ -112,5 +112,18 @@ If something fails, check `kubectl logs -n coder-system deploy/coder-k8s` and [T
 
 These resources are backed by Coder, not etcd, so some Kubernetes behavior differs. Read [Aggregated API behavior](../reference/aggregated-api-behavior.md) before you write manifests. The most important rule: object names must use Coder's canonical names.
 
-!!! warning "TLS"
-    `deploy/apiserver-apiservice.yaml` sets `insecureSkipTLSVerify: true` for development. Use CA-backed TLS in any real environment.
+## Serving certificate
+
+In a cluster, the aggregated API server serves a certificate signed by its own CA. Both live in the Secret `coder-k8s-apiserver-tls` in the server's namespace (type `coder.com/aggregated-apiserver-serving-ca`, label `app.kubernetes.io/component: aggregated-apiserver-serving-ca`). The certificate is valid for `coder-k8s-apiserver`, `coder-k8s-apiserver.<namespace>`, `coder-k8s-apiserver.<namespace>.svc`, and `coder-k8s-apiserver.<namespace>.svc.cluster.local`.
+
+- The server creates the Secret on first start and reuses it afterwards. With several replicas, they all use the same Secret.
+- The serving certificate is valid for 1 year. The server checks it at startup and every 12 hours, and renews it with the same CA when less than a third of its lifetime is left. The new certificate is served without a restart.
+- The CA is valid for 10 years. To replace it, delete the Secret and restart the Deployment (`kubectl -n coder-system rollout restart deployment/coder-k8s`). Clients that trusted the old CA must then trust the new one.
+- If the Secret exists but is unusable (a missing key, unparsable PEM, a key that does not match its certificate, a serving certificate not signed by the CA, or an expired CA), the server does not start and the log names the field. Fix the Secret or delete it.
+- Outside a cluster (for example `go run`), the server serves a self-signed certificate for `localhost` instead.
+
+!!! warning "The Secret holds the CA private key"
+    Anyone who can read Secrets in the server's namespace can issue certificates that the aggregated API server's CA vouches for. Restrict Secret read access in `coder-system` accordingly.
+
+!!! warning "TLS verification is still off"
+    `deploy/apiserver-apiservice.yaml` still sets `insecureSkipTLSVerify: true`, so kube-apiserver does not check this certificate yet. Registering the CA in the APIService `caBundle` is tracked in [#137](https://github.com/coder/coder-k8s/issues/137).

docs/how-to/troubleshooting.md

@@ -72,6 +72,17 @@ The aggregated API server checks every caller with the Kubernetes API and refuse
 - `no Kubernetes configuration for delegated authentication and authorization` (outside a cluster): set `KUBECONFIG` to one kubeconfig file, or create `~/.kube/config`.
 - `load kubeconfig ...` or `invalid kubeconfig ...`: the file named by `KUBECONFIG` is missing or incomplete. The server does not fall back to another configuration.
 
+## The pod exits with `configure aggregated API server serving certificate`
+
+The Secret `coder-k8s-apiserver-tls` exists but cannot be used; the message names the field (for example `data["ca.key"] is missing or empty`). Fix the Secret, or delete it so the server generates a new CA on its next start:
+
+```bash
+kubectl -n coder-system delete secret coder-k8s-apiserver-tls
+kubectl -n coder-system rollout restart deployment/coder-k8s
+```
+
+A read or create error instead of a field name means the ServiceAccount cannot get or create Secrets in its namespace.
+
 ## Aggregated requests fail with `401 Unauthorized` or `403 Forbidden`
 
 - **`401`:** the request has no valid credential. Requests sent straight to port `6443` need a Kubernetes bearer token; anonymous requests only reach `/healthz`, `/livez`, and `/readyz`. Use `kubectl`, which goes through kube-apiserver.

internal/aggregated/servingcert/manager.go

@@ -0,0 +1,193 @@
+package servingcert
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"sync"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/kubernetes"
+	ctrl "sigs.k8s.io/controller-runtime"
+)
+
+var log = ctrl.Log.WithName("servingcert")
+
+// DefaultCheckInterval is how often Run re-reads the Secret and renews the serving certificate.
+const DefaultCheckInterval = 12 * time.Hour
+
+const maxEnsureAttempts = 5
+
+// Manager keeps the serving certificate in sync with the Secret.
+//
+// Ensure is the adopt path: any valid Secret is used as is (so a later "bring your own Secret"
+// mode only has to skip generation and renewal). Listeners registered through
+// CertKeyContentProvider().AddListener are notified whenever the served certificate or the CA
+// changes, so an APIService caBundle controller can react without polling.
+type Manager struct {
+	client    kubernetes.Interface
+	namespace string
+	now       func() time.Time
+
+	mu        sync.RWMutex
+	current   *Bundle
+	listeners []dynamiccertificates.Listener
+}
+
+var _ dynamiccertificates.CertKeyContentProvider = (*Manager)(nil)
+
+// NewManager returns a Manager for the Secret in namespace.
+func NewManager(client kubernetes.Interface, namespace string) (*Manager, error) {
+	if client == nil {
+		return nil, fmt.Errorf("assertion failed: Kubernetes client must not be nil")
+	}
+	if namespace == "" {
+		return nil, fmt.Errorf("assertion failed: namespace must not be empty")
+	}
+	return &Manager{client: client, namespace: namespace, now: time.Now}, nil
+}
+
+// Ensure loads, creates, or renews the Secret and makes it the served certificate. Invalid
+// trust material returns a *CorruptSecretError and is never overwritten.
+func (m *Manager) Ensure(ctx context.Context) (*Bundle, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("assertion failed: context must not be nil")
+	}
+	secrets := m.client.CoreV1().Secrets(m.namespace)
+	for attempt := 1; attempt <= maxEnsureAttempts; attempt++ {
+		now := m.now()
+		secret, err := secrets.Get(ctx, SecretName, metav1.GetOptions{})
+		switch {
+		case apierrors.IsNotFound(err):
+			bundle, genErr := Generate(m.namespace, now)
+			if genErr != nil {
+				return nil, genErr
+			}
+			_, err = secrets.Create(ctx, &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{Name: SecretName, Namespace: m.namespace, Labels: copyLabels()},
+				Type:       SecretType,
+				Data:       bundle.Data(),
+			}, metav1.CreateOptions{})
+			if apierrors.IsAlreadyExists(err) {
+				continue // Another replica created it first; adopt theirs.
+			}
+			if err != nil {
+				return nil, fmt.Errorf("create secret %s/%s: %w", m.namespace, SecretName, err)
+			}
+			log.Info("Created aggregated API server CA and serving certificate", "namespace", m.namespace, "secret", SecretName)
+			return bundle, m.serve(bundle)
+		case err != nil:
+			return nil, fmt.Errorf("get secret %s/%s: %w", m.namespace, SecretName, err)
+		}
+
+		bundle, err := Parse(secret, m.namespace, now)
+		if err != nil {
+			return nil, err
+		}
+		if !bundle.NeedsRenewal(m.namespace, now) {
+			return bundle, m.serve(bundle)
+		}
+		if err := bundle.issueServingCert(m.namespace, now); err != nil {
+			return nil, err
+		}
+		updated := secret.DeepCopy()
+		updated.Data = bundle.Data()
+		_, err = secrets.Update(ctx, updated, metav1.UpdateOptions{})
+		if apierrors.IsConflict(err) {
+			continue // Another replica renewed it; re-read and adopt.
+		}
+		if err != nil {
+			return nil, fmt.Errorf("update secret %s/%s: %w", m.namespace, SecretName, err)
+		}
+		log.Info("Renewed aggregated API server serving certificate", "namespace", m.namespace, "secret", SecretName, "notAfter", bundle.Cert.NotAfter)
+		return bundle, m.serve(bundle)
+	}
+	return nil, fmt.Errorf("secret %s/%s kept changing; gave up after %d attempts", m.namespace, SecretName, maxEnsureAttempts)
+}
+
+// Run calls Ensure every interval until ctx is done. Failures are logged and the current
+// certificate keeps being served.
+func (m *Manager) Run(ctx context.Context, interval time.Duration) {
+	if interval <= 0 {
+		panic("assertion failed: interval must be positive")
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if _, err := m.Ensure(ctx); err != nil {
+				log.Error(err, "Could not refresh the aggregated API server serving certificate; still serving the current one")
+			}
+		}
+	}
+}
+
+// CABundle returns the PEM CA that signs the served certificate, or nil before the first Ensure.
+func (m *Manager) CABundle() []byte {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	if m.current == nil {
+		return nil
+	}
+	return m.current.CACertPEM
+}
+
+// serve makes bundle the served certificate and notifies listeners if anything changed.
+func (m *Manager) serve(bundle *Bundle) error {
+	// Same check the vendored static provider performs.
+	if _, err := tls.X509KeyPair(bundle.CertPEM, bundle.KeyPEM); err != nil {
+		return fmt.Errorf("assertion failed: validated serving certificate is not a usable key pair: %w", err)
+	}
+	m.mu.Lock()
+	changed := m.current == nil || !bundleEqual(m.current, bundle)
+	m.current = bundle
+	listeners := append([]dynamiccertificates.Listener(nil), m.listeners...)
+	m.mu.Unlock()
+	if changed {
+		for _, l := range listeners {
+			l.Enqueue()
+		}
+	}
+	return nil
+}
+
+// Name implements dynamiccertificates.CertKeyContentProvider.
+func (m *Manager) Name() string {
+	return "coder-k8s-managed-serving-cert::" + m.namespace + "/" + SecretName
+}
+
+// CurrentCertKeyContent implements dynamiccertificates.CertKeyContentProvider.
+func (m *Manager) CurrentCertKeyContent() ([]byte, []byte) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	if m.current == nil {
+		return nil, nil
+	}
+	return m.current.CertPEM, m.current.KeyPEM
+}
+
+// AddListener implements dynamiccertificates.Notifier.
+func (m *Manager) AddListener(listener dynamiccertificates.Listener) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.listeners = append(m.listeners, listener)
+}
+
+func bundleEqual(a, b *Bundle) bool {
+	return string(a.CACertPEM) == string(b.CACertPEM) && string(a.CertPEM) == string(b.CertPEM) && string(a.KeyPEM) == string(b.KeyPEM)
+}
+
+func copyLabels() map[string]string {
+	out := make(map[string]string, len(SecretLabels))
+	for k, v := range SecretLabels {
+		out[k] = v
+	}
+	return out
+}