diff --git a/go.mod b/go.mod index 1d666e4..b5134e4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/butlerdotdev/butler-controller go 1.24.6 require ( - github.com/butlerdotdev/butler-api v0.21.0 + github.com/butlerdotdev/butler-api v0.22.0 github.com/onsi/ginkgo/v2 v2.22.0 github.com/onsi/gomega v1.36.1 github.com/prometheus/client_golang v1.22.0 diff --git a/go.sum b/go.sum index 2854157..8ac96a6 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/butlerdotdev/butler-api v0.21.0 h1:zVI4doowHOHWUMVV6s+N0LcrbGasvNA5kP26Sf+kWnc= -github.com/butlerdotdev/butler-api v0.21.0/go.mod h1:q/RjPFM/r2ELq4DqR78OiAxerKBvH5D4nTGsVLKbjqY= +github.com/butlerdotdev/butler-api v0.22.0 h1:NbojKU2xDeNal1Jp3Wc2BJiysNGTnDgF8cywf2Tklak= +github.com/butlerdotdev/butler-api v0.22.0/go.mod h1:q/RjPFM/r2ELq4DqR78OiAxerKBvH5D4nTGsVLKbjqY= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= diff --git a/internal/capi/builder.go b/internal/capi/builder.go index 6ca489c..7359b02 100644 --- a/internal/capi/builder.go +++ b/internal/capi/builder.go @@ -122,6 +122,8 @@ func (b *Builder) Build() (*ResourceSet, error) { return b.buildHarvesterResources() case butlerv1alpha1.ProviderTypeNutanix: return b.buildNutanixResources() + case butlerv1alpha1.ProviderTypeLocal: + return b.buildLocalResources() default: return nil, fmt.Errorf("unsupported provider type: %s", b.providerConfig.Spec.Provider) } @@ -163,6 +165,141 @@ func (b *Builder) buildHarvesterResources() (*ResourceSet, error) { }, nil } +// buildLocalResources constructs CAPI resources for the local provider. +// Worker nodes run as containers via the Cluster API Docker provider (CAPD) on a +// kind-based management cluster. The control plane is a Steward TenantControlPlane +// (pods), identical to every other provider. The local provider is always kubeadm +// (never Talos), so the bootstrap template is always created and the MachineDeployment +// is created in one pass (the non-Talos 1-phase path in reconcile_infrastructure). +func (b *Builder) buildLocalResources() (*ResourceSet, error) { + clusterName := b.tc.Name + + // Infrastructure cluster (DockerCluster). + infraCluster := b.buildDockerCluster(clusterName) + + // Control plane (StewardControlPlane - shared across providers). + controlPlane := b.buildStewardControlPlane(clusterName) + + // Top-level Cluster. + cluster := b.buildCluster(clusterName, infraCluster, controlPlane) + + // Machine template (DockerMachineTemplate). + machineTemplate := b.buildDockerMachineTemplate(clusterName) + + // Bootstrap config template (CAPD-specific minimal kubeadm join). + bootstrapTemplate := b.buildCAPDKubeadmConfigTemplate(clusterName) + + // Machine deployment (shared). + machineDeployment := b.buildMachineDeployment(clusterName, machineTemplate, bootstrapTemplate) + + return &ResourceSet{ + Cluster: cluster, + InfrastructureCluster: infraCluster, + ControlPlane: controlPlane, + MachineDeployment: machineDeployment, + MachineTemplate: machineTemplate, + BootstrapConfigTemplate: bootstrapTemplate, + }, nil +} + +// buildDockerCluster constructs the DockerCluster infrastructure resource (CAPD). +// CAPD uses the v1beta1 infrastructure contract, like CAPX. +func (b *Builder) buildDockerCluster(name string) *unstructured.Unstructured { + dc := &unstructured.Unstructured{} + dc.SetAPIVersion(fmt.Sprintf("%s/v1beta1", InfrastructureAPIGroup)) + dc.SetKind("DockerCluster") + dc.SetName(name) + dc.SetNamespace(b.namespace) + b.applyCommonMetadata(dc) + + // Control plane endpoint is owned by Steward, not CAPD: the StewardControlPlane + // reports the endpoint once its Service has an address, mirroring the CAPX pattern + // (empty host, patched later). CAPD would otherwise provision an HAProxy load + // balancer container for the control plane, which is unnecessary here because the + // control plane is the external Steward TenantControlPlane. + // NOTE: whether CAPD honors a pre-set endpoint or insists on its own LB must be + // validated live when CAPD is installed on the management cluster (PR 2.3). + spec := map[string]interface{}{ + "controlPlaneEndpoint": map[string]interface{}{ + "host": "", + "port": int64(6443), + }, + } + + dc.Object["spec"] = spec + return dc +} + +// buildDockerMachineTemplate constructs the DockerMachineTemplate for worker nodes (CAPD). +// CAPD runs each node as a container from a kindest/node image whose tag must match the +// tenant Kubernetes version. +func (b *Builder) buildDockerMachineTemplate(name string) *unstructured.Unstructured { + dmt := &unstructured.Unstructured{} + dmt.SetAPIVersion(fmt.Sprintf("%s/v1beta1", InfrastructureAPIGroup)) + dmt.SetKind("DockerMachineTemplate") + dmt.SetName(fmt.Sprintf("%s-worker", name)) + dmt.SetNamespace(b.namespace) + b.applyCommonMetadata(dmt) + + spec := map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "customImage": b.localNodeImage(), + }, + }, + } + + dmt.Object["spec"] = spec + return dmt +} + +// buildCAPDKubeadmConfigTemplate constructs a minimal kubeadm join template for CAPD nodes. +// Unlike buildKubeadmConfigTemplate (Rocky Linux), CAPD nodes boot from a pre-baked +// kindest/node image where containerd, kubelet, and kubeadm already exist, so there are no +// package-install preKubeadmCommands. The eviction-hard override is required because kind +// nodes share the host filesystem and the default eviction thresholds would otherwise fire +// immediately and keep the node NotReady. +func (b *Builder) buildCAPDKubeadmConfigTemplate(name string) *unstructured.Unstructured { + kct := &unstructured.Unstructured{} + kct.SetAPIVersion(fmt.Sprintf("%s/%s", BootstrapAPIGroup, BootstrapAPIVersion)) + kct.SetKind("KubeadmConfigTemplate") + kct.SetName(fmt.Sprintf("%s-worker", name)) + kct.SetNamespace(b.namespace) + b.applyCommonMetadata(kct) + + spec := map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "joinConfiguration": map[string]interface{}{ + "nodeRegistration": map[string]interface{}{ + "criSocket": "unix:///var/run/containerd/containerd.sock", + "kubeletExtraArgs": map[string]interface{}{ + "eviction-hard": "nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%", + }, + }, + }, + }, + }, + } + + kct.Object["spec"] = spec + return kct +} + +// localNodeImage returns the kindest/node image for tenant worker containers. It honors an +// explicit override on the local ProviderConfig and otherwise derives the tag from the +// tenant Kubernetes version. +func (b *Builder) localNodeImage() string { + if b.providerConfig.Spec.Local != nil && b.providerConfig.Spec.Local.KindNodeImage != "" { + return b.providerConfig.Spec.Local.KindNodeImage + } + version := b.tc.Spec.KubernetesVersion + if !strings.HasPrefix(version, "v") { + version = "v" + version + } + return "kindest/node:" + version +} + // buildCluster constructs the top-level CAPI Cluster resource. func (b *Builder) buildCluster(name string, infraRef, cpRef *unstructured.Unstructured) *unstructured.Unstructured { cluster := &unstructured.Unstructured{} diff --git a/internal/capi/builder_test.go b/internal/capi/builder_test.go index 94d9231..f0d12d2 100644 --- a/internal/capi/builder_test.go +++ b/internal/capi/builder_test.go @@ -76,6 +76,77 @@ func TestBuildHarvesterResources(t *testing.T) { } } +func TestBuildLocalResources(t *testing.T) { + tc := newTestTenantCluster("toy", "team-demo") + pc := newTestProviderConfig("local") + + rs, err := NewBuilder(tc, pc, "toy-abc12345").Build() + if err != nil { + t.Fatalf("failed to build local resources: %v", err) + } + + // Every resource except the credential secret is created (CAPD needs no creds). + if rs.Cluster == nil || rs.InfrastructureCluster == nil || rs.ControlPlane == nil || + rs.MachineDeployment == nil || rs.MachineTemplate == nil || rs.BootstrapConfigTemplate == nil { + t.Fatal("expected all CAPD resources to be created") + } + if rs.CredentialSecret != nil { + t.Error("local provider must not create a credential secret") + } + + // Infrastructure cluster is a DockerCluster on the v1beta1 infra contract. + if got := rs.InfrastructureCluster.GetKind(); got != "DockerCluster" { + t.Errorf("InfrastructureCluster kind = %q, want DockerCluster", got) + } + if got := rs.InfrastructureCluster.GetAPIVersion(); got != "infrastructure.cluster.x-k8s.io/v1beta1" { + t.Errorf("DockerCluster apiVersion = %q, want infrastructure.cluster.x-k8s.io/v1beta1", got) + } + + // Machine template is a DockerMachineTemplate; customImage derives from the k8s version. + if got := rs.MachineTemplate.GetKind(); got != "DockerMachineTemplate" { + t.Errorf("MachineTemplate kind = %q, want DockerMachineTemplate", got) + } + img, _, _ := unstructured.NestedString(rs.MachineTemplate.Object, "spec", "template", "spec", "customImage") + if img != "kindest/node:v1.30.2" { + t.Errorf("customImage = %q, want kindest/node:v1.30.2", img) + } + + // Bootstrap is a kubeadm join template (CAPD-specific): containerd criSocket + eviction-hard, + // and NONE of the Rocky Linux package-install preKubeadmCommands. + criSocket, _, _ := unstructured.NestedString(rs.BootstrapConfigTemplate.Object, + "spec", "template", "spec", "joinConfiguration", "nodeRegistration", "criSocket") + if criSocket != "unix:///var/run/containerd/containerd.sock" { + t.Errorf("criSocket = %q, want containerd socket", criSocket) + } + if _, found, _ := unstructured.NestedSlice(rs.BootstrapConfigTemplate.Object, "spec", "template", "spec", "preKubeadmCommands"); found { + t.Error("CAPD bootstrap must not carry Rocky Linux preKubeadmCommands") + } + + // MachineDeployment uses configRef (kubeadm), not a Talos dataSecretName. + bootstrap, _, _ := unstructured.NestedMap(rs.MachineDeployment.Object, "spec", "template", "spec", "bootstrap") + if _, hasConfigRef := bootstrap["configRef"]; !hasConfigRef { + t.Error("MachineDeployment bootstrap must use configRef for the local (kubeadm) path") + } + if _, hasDataSecret := bootstrap["dataSecretName"]; hasDataSecret { + t.Error("local MachineDeployment must not use dataSecretName (that is the Talos path)") + } +} + +func TestLocalNodeImageOverride(t *testing.T) { + tc := newTestTenantCluster("toy", "team-demo") + pc := newTestProviderConfig("local") + pc.Spec.Local = &butlerv1alpha1.LocalProviderConfig{KindNodeImage: "kindest/node:v1.29.4"} + + rs, err := NewBuilder(tc, pc, "toy-abc12345").Build() + if err != nil { + t.Fatalf("failed to build: %v", err) + } + img, _, _ := unstructured.NestedString(rs.MachineTemplate.Object, "spec", "template", "spec", "customImage") + if img != "kindest/node:v1.29.4" { + t.Errorf("customImage = %q, want override kindest/node:v1.29.4", img) + } +} + func TestBuildCluster(t *testing.T) { tc := newTestTenantCluster("prod-cluster", "team-alpha") pc := newTestProviderConfig("harvester") diff --git a/internal/controller/tenantcluster/reconcile_addons.go b/internal/controller/tenantcluster/reconcile_addons.go index a7404cf..ffcbb36 100644 --- a/internal/controller/tenantcluster/reconcile_addons.go +++ b/internal/controller/tenantcluster/reconcile_addons.go @@ -48,6 +48,12 @@ func (r *Reconciler) reconcileAddons(ctx context.Context, tc *butlerv1alpha1.Ten var addonStatuses []butlerv1alpha1.AddonStatus var failedAddons []string + // The local provider runs a single-node toy tenant on CAPD. Cilium (CNI) and + // MetalLB (the control plane LoadBalancer IP) are load-bearing and stay; Longhorn + // and Traefik are unnecessary and only add load to a constrained node. + providerType, _ := resolveProviderType(ctx, r.Client, tc) + isLocal := providerType == string(butlerv1alpha1.ProviderTypeLocal) + // 1. CNI - REQUIRED, nodes won't be Ready without it ciliumVersion := addons.DefaultCiliumVersion if tc.Spec.Addons.CNI != nil && tc.Spec.Addons.CNI.Version != "" { @@ -124,13 +130,14 @@ func (r *Reconciler) reconcileAddons(ctx context.Context, tc *butlerv1alpha1.Ten }) } - // 3. Longhorn - storage + // 3. Longhorn - storage (skipped for local; kind's default StorageClass suffices) longhornVersion := addons.DefaultLonghornVersion if tc.Spec.Addons.Storage != nil && tc.Spec.Addons.Storage.Version != "" { longhornVersion = tc.Spec.Addons.Storage.Version } - logger.Info("installing Longhorn", "version", longhornVersion) - if err := r.Installer.InstallLonghorn(ctx, kubeconfigData, longhornVersion); err != nil { + if isLocal { + logger.Info("skipping Longhorn installation (local provider)") + } else if err := r.Installer.InstallLonghorn(ctx, kubeconfigData, longhornVersion); err != nil { logger.Error(err, "failed to install Longhorn") failedAddons = append(failedAddons, "longhorn") r.Recorder.Eventf(tc, corev1.EventTypeWarning, "AddonInstallFailed", @@ -165,8 +172,8 @@ func (r *Reconciler) reconcileAddons(ctx context.Context, tc *butlerv1alpha1.Ten }) } - // 5. Traefik - Ingress (optional) - if tc.Spec.Addons.Ingress.IsIngressEnabled() { + // 5. Traefik - Ingress (optional; skipped for local) + if tc.Spec.Addons.Ingress.IsIngressEnabled() && !isLocal { traefikVersion := addons.DefaultTraefikVersion if tc.Spec.Addons.Ingress != nil && tc.Spec.Addons.Ingress.Version != "" { traefikVersion = tc.Spec.Addons.Ingress.Version @@ -276,12 +283,19 @@ func (r *Reconciler) reconcileAddonHealth(ctx context.Context, tc *butlerv1alpha metallbVersion = tc.Spec.Addons.LoadBalancer.Version } + // Local toy tenants skip Longhorn and Traefik (see reconcileAddons); only + // cert-manager and MetalLB are expected alongside the required Cilium. + providerType, _ := resolveProviderType(ctx, r.Client, tc) + isLocal := providerType == string(butlerv1alpha1.ProviderTypeLocal) + expected := []expectedAddon{ {"cert-manager", certManagerVersion}, - {"longhorn", longhornVersion}, {"metallb", metallbVersion}, } - if tc.Spec.Addons.Ingress.IsIngressEnabled() { + if !isLocal { + expected = append(expected, expectedAddon{"longhorn", longhornVersion}) + } + if tc.Spec.Addons.Ingress.IsIngressEnabled() && !isLocal { traefikVersion := addons.DefaultTraefikVersion if tc.Spec.Addons.Ingress != nil && tc.Spec.Addons.Ingress.Version != "" { traefikVersion = tc.Spec.Addons.Ingress.Version diff --git a/internal/controller/tenantcluster/reconcile_infrastructure.go b/internal/controller/tenantcluster/reconcile_infrastructure.go index f389eb3..5051fb1 100644 --- a/internal/controller/tenantcluster/reconcile_infrastructure.go +++ b/internal/controller/tenantcluster/reconcile_infrastructure.go @@ -130,8 +130,13 @@ func (r *Reconciler) reconcileInfrastructure(ctx context.Context, tc *butlerv1al // // For non-Talos clusters, create all resources in one pass — kubeadm bootstrap // uses a configRef to a KubeadmConfigTemplate, which CAPI handles natively. + // The local provider is always kubeadm/CAPD, never Talos, so it uses the one-pass + // path even if a TenantCluster mistakenly sets os.type=talos. Only VM providers + // (CAPX) need the Talos two-phase MachineDeployment deferral. + talosTwoPhase := isTalosCluster(tc) && providerConfig.Spec.Provider != butlerv1alpha1.ProviderTypeLocal + var initialResources []*unstructured.Unstructured - if isTalosCluster(tc) { + if talosTwoPhase { initialResources = resourceSet.ResourcesWithoutMachineDeployment() } else { initialResources = resourceSet.AllResources() @@ -183,7 +188,7 @@ func (r *Reconciler) reconcileInfrastructure(ctx context.Context, tc *butlerv1al // Config must be applied BEFORE addon installation — workers need to leave // maintenance mode and join the cluster before Cilium and other addons can // schedule pods on them. - if isTalosCluster(tc) { + if talosTwoPhase { if err := r.reconcileTalosBootstrap(ctx, tc, butlerConfig, providerConfig); err != nil { logger.Error(err, "failed to reconcile Talos bootstrap") return ctrl.Result{RequeueAfter: 15 * time.Second}, nil