From 214710cdc59e68a8bd74c0594c3f1773659059c9 Mon Sep 17 00:00:00 2001 From: atbagan Date: Wed, 10 Jun 2026 17:14:09 -0500 Subject: [PATCH 01/19] feat: add butleradm bootstrap local for kind-based laptop install Adds a local bootstrap subcommand that stands up the whole platform on a KIND cluster with CAPD tenant workers, no hypervisor required. The orchestrator gains a local path: the KIND cluster is the management cluster, so it mounts the host docker socket (for CAPD sibling containers) and maps the console NodePort to the host, skips teardown, derives the MetalLB pool from the kind docker network, creates a credential-less platform-scoped local ProviderConfig, builds butler-controller from source alongside butler-bootstrap, and treats the KIND kubeconfig as the management kubeconfig once bootstrap reaches Ready. --- internal/adm/bootstrap/bootstrap.go | 1 + internal/adm/bootstrap/local.go | 112 ++++++++++++ .../bootstrap/orchestrator/orchestrator.go | 160 ++++++++++++++---- 3 files changed, 243 insertions(+), 30 deletions(-) create mode 100644 internal/adm/bootstrap/local.go diff --git a/internal/adm/bootstrap/bootstrap.go b/internal/adm/bootstrap/bootstrap.go index c116878..476df60 100644 --- a/internal/adm/bootstrap/bootstrap.go +++ b/internal/adm/bootstrap/bootstrap.go @@ -66,6 +66,7 @@ Example: } // Register provider subcommands + cmd.AddCommand(NewLocalCmd(logger)) cmd.AddCommand(NewHarvesterCmd(logger)) cmd.AddCommand(NewNutanixCmd(logger)) cmd.AddCommand(NewGCPCmd(logger)) diff --git a/internal/adm/bootstrap/local.go b/internal/adm/bootstrap/local.go new file mode 100644 index 0000000..4a40548 --- /dev/null +++ b/internal/adm/bootstrap/local.go @@ -0,0 +1,112 @@ +/* +Copyright 2026 The Butler Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "os" + "time" + + "github.com/butlerdotdev/butler/internal/adm/bootstrap/orchestrator" + "github.com/butlerdotdev/butler/internal/common/log" + "github.com/spf13/cobra" +) + +// NewLocalCmd creates the local bootstrap subcommand. It stands up the entire Butler +// platform on a local KIND cluster, with tenant worker nodes running as containers via +// the Cluster API Docker provider (CAPD). No hypervisor or cloud account is required. +func NewLocalCmd(logger *log.Logger) *cobra.Command { + var ( + clusterName string + repoRoot string + skipCleanup bool + ) + + cmd := &cobra.Command{ + Use: "local", + Short: "Bootstrap a Butler management cluster locally on KIND (no hypervisor)", + Long: `Bootstrap a Butler management cluster on a local KIND cluster. + +This runs the entire Butler platform on your laptop using Docker and KIND. Tenant +worker nodes run as containers via the Cluster API Docker provider (CAPD), so no +hypervisor or cloud account is required. The KIND cluster is kept after bootstrap so +you can create tenant clusters against it. + +Prerequisites: + • Docker running locally + • kind and kubectl installed + • The butlerdotdev repos checked out as siblings (for building images from source) + +Example: + butleradm bootstrap local`, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := setupSignalHandler(cmd, logger) + defer cancel() + + if repoRoot == "" { + home, _ := os.UserHomeDir() + repoRoot = home + "/code/github.com/butlerdotdev" + } + + cfg := localConfig(clusterName) + + orch := orchestrator.New(logger, orchestrator.Options{ + SkipCleanup: skipCleanup, + Timeout: 30 * time.Minute, + LocalDev: true, + RepoRoot: repoRoot, + }) + return orch.Run(ctx, cfg) + }, + } + + cmd.Flags().StringVar(&clusterName, "name", "butler-local", "management cluster name") + cmd.Flags().StringVar(&repoRoot, "repo-root", "", "path to butlerdotdev repos (default: ~/code/github.com/butlerdotdev)") + cmd.Flags().BoolVar(&skipCleanup, "skip-cleanup", false, "no-op for local (the KIND cluster is always kept); accepted for parity") + + return cmd +} + +// localConfig builds the bootstrap Config for the local provider with laptop defaults: +// single control-plane node, Cilium CNI, MetalLB (pool derived from the kind network), +// CAPI/CAPD for tenant workers, and the locally built butler-controller image. Longhorn +// and kube-vip are intentionally omitted (kind's default StorageClass is used; no HA VIP). +func localConfig(name string) *orchestrator.Config { + if name == "" { + name = "butler-local" + } + return &orchestrator.Config{ + Provider: "local", + Cluster: orchestrator.ClusterConfig{ + Name: name, + Topology: "single-node", + ControlPlane: orchestrator.NodePoolConfig{Replicas: 1}, + }, + Network: orchestrator.NetworkConfig{ + PodCIDR: "10.244.0.0/16", + ServiceCIDR: "10.96.0.0/12", + // LoadBalancerPool is derived from the kind docker network at runtime. + }, + Addons: orchestrator.AddonsConfig{ + CNI: orchestrator.CNIConfig{Type: "cilium"}, + LoadBalancer: orchestrator.LoadBalancerConfig{Type: "metallb"}, + CAPI: orchestrator.CAPIConfig{Version: "v1.9.4"}, + ButlerController: orchestrator.ButlerControllerConfig{ + Image: "ghcr.io/butlerdotdev/butler-controller:latest", + }, + }, + } +} diff --git a/internal/adm/bootstrap/orchestrator/orchestrator.go b/internal/adm/bootstrap/orchestrator/orchestrator.go index 9ad1cdb..b2c4e6d 100644 --- a/internal/adm/bootstrap/orchestrator/orchestrator.go +++ b/internal/adm/bootstrap/orchestrator/orchestrator.go @@ -110,6 +110,13 @@ type Orchestrator struct { logger *log.Logger options Options eventSink EventSink + + // isLocal is true for the local provider, which installs Butler onto the KIND + // cluster itself (no provisioned cluster, no pivot, no teardown). + isLocal bool + // kindKubeconfigPath is the path to the KIND cluster's kubeconfig. For the local + // provider this kubeconfig is also the management cluster kubeconfig. + kindKubeconfigPath string } // New creates a new orchestrator @@ -151,6 +158,8 @@ func (o *Orchestrator) Run(ctx context.Context, cfg *Config) error { o.logger.Phase("Initializing bootstrap") o.emit(Event{Type: EventPhaseChange, Phase: "Initializing", Message: "Initializing bootstrap"}) + o.isLocal = cfg.Provider == "local" + // Create context with timeout ctx, cancel := context.WithTimeout(ctx, o.options.Timeout) defer cancel() @@ -164,6 +173,20 @@ func (o *Orchestrator) Run(ctx context.Context, cfg *Config) error { if err != nil { return fmt.Errorf("creating KIND cluster: %w", err) } + o.kindKubeconfigPath = kubeconfigPath + + // The local provider's MetalLB pool must live in the kind docker network so CAPD + // worker containers (on the same network) can reach tenant LoadBalancer IPs. Derive + // it now that the network exists. + if o.isLocal && cfg.Network.LoadBalancerPool == nil { + if pool, derr := deriveKindLBPool(ctx); derr != nil { + o.logger.Warn("Failed to derive MetalLB pool from kind network, using default", "error", derr) + cfg.Network.LoadBalancerPool = &LBPoolConfig{Start: "172.18.255.200", End: "172.18.255.250"} + } else { + o.logger.Info("Derived MetalLB pool from kind docker network", "start", pool.Start, "end", pool.End) + cfg.Network.LoadBalancerPool = pool + } + } // Tell the TUI where the KIND kubeconfig lives so it can start streaming // butler-bootstrap-controller and butler-provider-* pod logs into the // debug panel. @@ -174,7 +197,9 @@ func (o *Orchestrator) Run(ctx context.Context, cfg *Config) error { KINDKubeconfig: kubeconfigPath, }) defer func() { - if !o.options.SkipCleanup { + // The local provider keeps the KIND cluster: it is the management cluster, + // not throwaway scaffolding, so the demo needs it to persist. + if !o.options.SkipCleanup && !o.isLocal { o.logger.Phase("Cleaning up KIND cluster") o.emit(Event{Type: EventPhaseChange, Phase: "CleaningUp", Message: "Cleaning up KIND cluster"}) if err := kindProvider.Delete(kindClusterName, ""); err != nil { @@ -494,18 +519,37 @@ func (o *Orchestrator) scanCertDirectory(dir string) []string { return certs } -// buildKINDConfig generates a KIND cluster configuration with CA certificate mounts -func (o *Orchestrator) buildKINDConfig(caCerts []string) string { - if len(caCerts) == 0 { - // No custom certs, use minimal config - return `kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: - - role: control-plane -` +// deriveKindLBPool inspects the kind docker network and returns a MetalLB pool high in +// its IPv4 subnet, so LoadBalancer IPs are reachable by CAPD worker containers on the +// same network. +func deriveKindLBPool(ctx context.Context) (*LBPoolConfig, error) { + out, err := exec.CommandContext(ctx, "docker", "network", "inspect", "kind", + "-f", "{{range .IPAM.Config}}{{.Subnet}} {{end}}").Output() + if err != nil { + return nil, fmt.Errorf("docker network inspect kind: %w", err) } + for _, field := range strings.Fields(string(out)) { + ip, _, perr := net.ParseCIDR(field) + if perr != nil { + continue + } + ip4 := ip.To4() + if ip4 == nil { + continue // skip IPv6 subnets + } + return &LBPoolConfig{ + Start: fmt.Sprintf("%d.%d.255.200", ip4[0], ip4[1]), + End: fmt.Sprintf("%d.%d.255.250", ip4[0], ip4[1]), + }, nil + } + return nil, fmt.Errorf("no IPv4 subnet found on kind network") +} - // Build extraMounts for each certificate +// buildKINDConfig generates a KIND cluster configuration with CA certificate mounts. +// For the local provider it also mounts the host Docker socket so the in-cluster CAPD +// controller can create sibling node containers, and maps the console NodePort to the host. +func (o *Orchestrator) buildKINDConfig(caCerts []string) string { + // Build extraMounts for each certificate. var mounts strings.Builder for i, certPath := range caCerts { containerPath := fmt.Sprintf("/usr/local/share/ca-certificates/butler-custom-%d.crt", i) @@ -515,12 +559,34 @@ nodes: `, certPath, containerPath)) } - return fmt.Sprintf(`kind: Cluster + if o.isLocal { + // Mount the host Docker socket so the CAPD (Cluster API Docker) controller + // running inside this cluster can create sibling node containers. Without + // this mount, DockerMachines never provision and the tenant hangs. + mounts.WriteString(` - hostPath: /var/run/docker.sock + containerPath: /var/run/docker.sock +`) + } + + var b strings.Builder + b.WriteString(`kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - extraMounts: -%s`, mounts.String()) +`) + if mounts.Len() > 0 { + b.WriteString(" extraMounts:\n") + b.WriteString(mounts.String()) + } + if o.isLocal { + // Expose the Butler console NodePort on the host for the laptop demo. + b.WriteString(` extraPortMappings: + - containerPort: 30080 + hostPort: 8080 + protocol: TCP +`) + } + return b.String() } // installCACertificates runs update-ca-certificates in the KIND node @@ -1335,6 +1401,14 @@ func (o *Orchestrator) buildProviderConfigUnstructured(cfg *Config) *unstructure case "proxmox": // TODO: Proxmox ProviderConfig not yet implemented + case "local": + // The local provider needs no credentials. It is platform-scoped so tenant + // clusters can reference it, and uses cloud network mode to skip IPAM + // (CAPD manages container networking). + spec["scope"] = map[string]interface{}{"type": "platform"} + spec["network"] = map[string]interface{}{"mode": "cloud"} + spec["local"] = map[string]interface{}{} + case "gcp": spec["credentialsRef"] = map[string]interface{}{ "name": cfg.Cluster.Name + "-gcp-credentials", @@ -1516,7 +1590,7 @@ func (o *Orchestrator) buildClusterBootstrapUnstructured(cfg *Config) *unstructu } return n }(), - "talos": buildTalosSpec(cfg), + "talos": buildTalosSpec(cfg), "addons": buildAddonsConfig(cfg), }, }, @@ -1649,18 +1723,31 @@ func (o *Orchestrator) watchBootstrap(ctx context.Context, client dynamic.Interf case "Ready": o.logger.Success("Cluster is ready!") - // Decode kubeconfig - kubeconfig, _ := status["kubeconfig"].(string) - kubeconfigBytes, err := base64.StdEncoding.DecodeString(kubeconfig) - if err != nil { - return nil, fmt.Errorf("decoding kubeconfig: %w", err) - } + var kubeconfigBytes, talosconfigBytes []byte + if o.isLocal { + // The KIND cluster IS the management cluster, so its kubeconfig is the + // management kubeconfig. There is no Talos config for a local cluster. + kc, readErr := os.ReadFile(o.kindKubeconfigPath) + if readErr != nil { + return nil, fmt.Errorf("reading KIND kubeconfig: %w", readErr) + } + kubeconfigBytes = kc + } else { + // Decode kubeconfig + kubeconfig, _ := status["kubeconfig"].(string) + kc, decErr := base64.StdEncoding.DecodeString(kubeconfig) + if decErr != nil { + return nil, fmt.Errorf("decoding kubeconfig: %w", decErr) + } + kubeconfigBytes = kc - // Decode talosconfig - NOTE: JSON field is lowercase "talosconfig" - talosconfig, _ := status["talosconfig"].(string) - talosconfigBytes, err := base64.StdEncoding.DecodeString(talosconfig) - if err != nil { - return nil, fmt.Errorf("decoding talosconfig: %w", err) + // Decode talosconfig - NOTE: JSON field is lowercase "talosconfig" + talosconfig, _ := status["talosconfig"].(string) + tc, decErr := base64.StdEncoding.DecodeString(talosconfig) + if decErr != nil { + return nil, fmt.Errorf("decoding talosconfig: %w", decErr) + } + talosconfigBytes = tc } creds := &clusterCredentials{ @@ -1787,24 +1874,37 @@ func (o *Orchestrator) buildAndLoadImages(ctx context.Context, provider string) // pointing to a sibling repo (e.g., replace ../butler-api). In that case // the Docker build context must be the parent directory so the sibling // is accessible, and -f points to the repo's Dockerfile. - images := []struct { + type buildImage struct { name string repoDir string image string useParentContext bool - }{ + } + images := []buildImage{ { name: "butler-bootstrap", repoDir: filepath.Join(o.options.RepoRoot, "butler-bootstrap"), image: "ghcr.io/butlerdotdev/butler-bootstrap:latest", useParentContext: true, }, - { + } + if o.isLocal { + // The local provider uses CAPD for tenant workers (there is no + // butler-provider-local), and it needs butler-controller built from source + // so the CAPD resource builder is present. + images = append(images, buildImage{ + name: "butler-controller", + repoDir: filepath.Join(o.options.RepoRoot, "butler-controller"), + image: "ghcr.io/butlerdotdev/butler-controller:latest", + useParentContext: true, + }) + } else { + images = append(images, buildImage{ name: fmt.Sprintf("butler-provider-%s", provider), repoDir: filepath.Join(o.options.RepoRoot, fmt.Sprintf("butler-provider-%s", provider)), image: fmt.Sprintf("ghcr.io/butlerdotdev/butler-provider-%s:latest", provider), useParentContext: true, - }, + }) } for _, img := range images { From 4c2acb2cff7d2cf4f14f52dd6546cad52369a8c0 Mon Sep 17 00:00:00 2001 From: atbagan Date: Wed, 10 Jun 2026 17:37:48 -0500 Subject: [PATCH 02/19] fix: build butler-controller with its own context for local bootstrap butler-controller depends on butler-api as a normal fetchable module, so it no longer needs the parent build context that the sibling replace required. Its Dockerfile is not REPO_DIR-aware, so the parent context failed on COPY go.mod. --- internal/adm/bootstrap/orchestrator/orchestrator.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/adm/bootstrap/orchestrator/orchestrator.go b/internal/adm/bootstrap/orchestrator/orchestrator.go index b2c4e6d..10911fc 100644 --- a/internal/adm/bootstrap/orchestrator/orchestrator.go +++ b/internal/adm/bootstrap/orchestrator/orchestrator.go @@ -1893,10 +1893,12 @@ func (o *Orchestrator) buildAndLoadImages(ctx context.Context, provider string) // butler-provider-local), and it needs butler-controller built from source // so the CAPD resource builder is present. images = append(images, buildImage{ - name: "butler-controller", - repoDir: filepath.Join(o.options.RepoRoot, "butler-controller"), - image: "ghcr.io/butlerdotdev/butler-controller:latest", - useParentContext: true, + name: "butler-controller", + repoDir: filepath.Join(o.options.RepoRoot, "butler-controller"), + image: "ghcr.io/butlerdotdev/butler-controller:latest", + // butler-controller fetches butler-api as a normal module (no sibling + // replace), so it builds with its own repo as the context. + useParentContext: false, }) } else { images = append(images, buildImage{ From b13db647d485a7e1ec771fe3f55c24877aaf6788 Mon Sep 17 00:00:00 2001 From: atbagan Date: Wed, 10 Jun 2026 17:41:59 -0500 Subject: [PATCH 03/19] fix: skip provider controller deploy for local bootstrap The local provider uses CAPD for tenant workers and has no butler-provider-local controller manifest. Deploying it failed the bootstrap at the controller step. --- internal/adm/bootstrap/manifests/deployer.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/adm/bootstrap/manifests/deployer.go b/internal/adm/bootstrap/manifests/deployer.go index 67c5cc4..edcb183 100644 --- a/internal/adm/bootstrap/manifests/deployer.go +++ b/internal/adm/bootstrap/manifests/deployer.go @@ -60,6 +60,13 @@ func (d *Deployer) DeployControllers(ctx context.Context, provider string) error return fmt.Errorf("deploying butler-bootstrap: %w", err) } + // The local provider uses the Cluster API Docker provider (CAPD) for tenant + // workers, installed during the addon phase. There is no Butler provider + // controller to deploy. + if provider == "local" { + return nil + } + // Deploy provider-specific controller providerFile := fmt.Sprintf("controllers/butler-provider-%s.yaml", provider) if err := d.deployFile(ctx, Controllers, providerFile); err != nil { From 7de05cb9c9251f4b19f0d97cd88afac33f6eea98 Mon Sep 17 00:00:00 2001 From: atbagan Date: Wed, 10 Jun 2026 17:47:59 -0500 Subject: [PATCH 04/19] fix: skip provider controller wait for local bootstrap --- internal/adm/bootstrap/orchestrator/orchestrator.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/adm/bootstrap/orchestrator/orchestrator.go b/internal/adm/bootstrap/orchestrator/orchestrator.go index 10911fc..cb412a1 100644 --- a/internal/adm/bootstrap/orchestrator/orchestrator.go +++ b/internal/adm/bootstrap/orchestrator/orchestrator.go @@ -1136,6 +1136,12 @@ func (o *Orchestrator) deployControllers(ctx context.Context, clientset *kuberne } o.logger.Success("butler-bootstrap-controller is ready") + // The local provider has no Butler provider controller (CAPD handles tenant + // workers), so there is nothing further to wait for. + if o.isLocal { + return nil + } + // Wait for provider controller providerDeployment := fmt.Sprintf("butler-provider-%s", cfg.Provider) if err := deployer.WaitForDeployment(waitCtx, butlerNamespace, providerDeployment); err != nil { From 11a9d65b373ddaf97ed4e88ff42c9ac9237a5a26 Mon Sep 17 00:00:00 2001 From: atbagan Date: Wed, 10 Jun 2026 17:50:16 -0500 Subject: [PATCH 05/19] chore: sync providerconfig and clusterbootstrap CRDs with local provider --- ...tler.butlerlabs.dev_clusterbootstraps.yaml | 10 ++++---- ...butler.butlerlabs.dev_providerconfigs.yaml | 24 ++++++++++++++++++- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/internal/adm/bootstrap/manifests/crds/butler.butlerlabs.dev_clusterbootstraps.yaml b/internal/adm/bootstrap/manifests/crds/butler.butlerlabs.dev_clusterbootstraps.yaml index be046d4..cc45f54 100644 --- a/internal/adm/bootstrap/manifests/crds/butler.butlerlabs.dev_clusterbootstraps.yaml +++ b/internal/adm/bootstrap/manifests/crds/butler.butlerlabs.dev_clusterbootstraps.yaml @@ -542,11 +542,10 @@ spec: type: string vip: description: |- - VIP is the virtual IP for the control plane endpoint (kube-vip). - Required for on-prem providers (harvester, nutanix, proxmox). - Optional for cloud providers (gcp, aws, azure) where the first - control plane node IP is used as the endpoint instead. - pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}$ + VIP is the control plane endpoint. For on-prem providers this is + a virtual IP managed by kube-vip. For cloud providers this can be + a load balancer IP or DNS name. Optional for cloud providers where + the first control plane node IP is used as the endpoint instead. type: string vipInterface: description: VIPInterface is the network interface for the VIP @@ -568,6 +567,7 @@ spec: - gcp - aws - azure + - local type: string providerRef: description: |- diff --git a/internal/adm/bootstrap/manifests/crds/butler.butlerlabs.dev_providerconfigs.yaml b/internal/adm/bootstrap/manifests/crds/butler.butlerlabs.dev_providerconfigs.yaml index 57c8f96..f9b7998 100644 --- a/internal/adm/bootstrap/manifests/crds/butler.butlerlabs.dev_providerconfigs.yaml +++ b/internal/adm/bootstrap/manifests/crds/butler.butlerlabs.dev_providerconfigs.yaml @@ -130,6 +130,8 @@ spec: - nutanix: "username", "password" - proxmox: "username", "password" or "token" - gcp: "serviceAccountKey" (JSON service account key) + Required for all providers except "local", which needs no credentials. + The per-provider requirement is enforced by the ProviderConfig admission webhook. properties: key: description: |- @@ -249,6 +251,18 @@ spec: minimum: 1 type: integer type: object + local: + description: |- + Local contains local-provider configuration. + Optional when provider is "local". + properties: + kindNodeImage: + description: |- + KindNodeImage overrides the container image used for tenant worker nodes, + for example "kindest/node:v1.30.0". When empty, the image is derived from + the tenant cluster's Kubernetes version. + type: string + type: object network: description: Network configures IPAM and network settings for this provider. @@ -349,6 +363,14 @@ spec: description: Subnet is the network name for VM placement (e.g., "VM Network - VLAN 40"). type: string + timeServers: + description: |- + TimeServers are the default NTP servers for Talos worker nodes on this provider. + Falls back to ButlerConfig.spec.defaultTimeServers if empty. + Required on networks where the Talos default (time.cloudflare.com) is unreachable. + items: + type: string + type: array type: object nutanix: description: |- @@ -397,6 +419,7 @@ spec: - azure - aws - gcp + - local type: string proxmox: description: |- @@ -474,7 +497,6 @@ spec: type: string type: object required: - - credentialsRef - provider type: object status: From 996b3ec7ebcc2ffc6ef5616bb0ca967f8c09fbf5 Mon Sep 17 00:00:00 2001 From: atbagan Date: Wed, 10 Jun 2026 17:53:19 -0500 Subject: [PATCH 06/19] fix: satisfy ClusterBootstrap schema for local config --- internal/adm/bootstrap/local.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/internal/adm/bootstrap/local.go b/internal/adm/bootstrap/local.go index 4a40548..05aaf39 100644 --- a/internal/adm/bootstrap/local.go +++ b/internal/adm/bootstrap/local.go @@ -91,17 +91,31 @@ func localConfig(name string) *orchestrator.Config { return &orchestrator.Config{ Provider: "local", Cluster: orchestrator.ClusterConfig{ - Name: name, - Topology: "single-node", - ControlPlane: orchestrator.NodePoolConfig{Replicas: 1}, + Name: name, + Topology: "single-node", + // CPU/MemoryMB/DiskGB are schema-required VM specs. They are unused for + // the local provider (the KIND node is the control plane, not a VM), but + // must satisfy the ClusterBootstrap CRD minimums. + ControlPlane: orchestrator.NodePoolConfig{ + Replicas: 1, + CPU: 2, + MemoryMB: 2048, + DiskGB: 20, + }, }, Network: orchestrator.NetworkConfig{ PodCIDR: "10.244.0.0/16", ServiceCIDR: "10.96.0.0/12", // LoadBalancerPool is derived from the kind docker network at runtime. }, + // Talos.Version is unused for local (no Talos nodes) but the CRD requires a + // valid vX.Y.Z value. + Talos: orchestrator.TalosConfig{Version: "v1.9.3"}, Addons: orchestrator.AddonsConfig{ - CNI: orchestrator.CNIConfig{Type: "cilium"}, + // The management cluster keeps KIND's default CNI (kindnet); installing + // Cilium here would conflict. Tenant clusters get Cilium separately. + CNI: orchestrator.CNIConfig{Type: "none"}, + Storage: orchestrator.StorageConfig{Type: "none"}, LoadBalancer: orchestrator.LoadBalancerConfig{Type: "metallb"}, CAPI: orchestrator.CAPIConfig{Version: "v1.9.4"}, ButlerController: orchestrator.ButlerControllerConfig{ From cf7adcaa0abe9fad33a58c82c37a7e2ada0ca122 Mon Sep 17 00:00:00 2001 From: atbagan Date: Wed, 10 Jun 2026 18:02:48 -0500 Subject: [PATCH 07/19] fix: use locally built bootstrap image for local bootstrap --- internal/adm/bootstrap/orchestrator/orchestrator.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/internal/adm/bootstrap/orchestrator/orchestrator.go b/internal/adm/bootstrap/orchestrator/orchestrator.go index cb412a1..637c53b 100644 --- a/internal/adm/bootstrap/orchestrator/orchestrator.go +++ b/internal/adm/bootstrap/orchestrator/orchestrator.go @@ -1123,6 +1123,19 @@ func (o *Orchestrator) deployControllers(ctx context.Context, clientset *kuberne return fmt.Errorf("deploying controllers: %w", err) } + // The embedded manifest pins a published bootstrap image tag, but for the local + // provider we just built and loaded the controller from source as :latest. Point + // the deployment at that local image so the IsLocal install path runs (otherwise + // the published controller treats local as a VM provider and provisions machines). + if o.isLocal { + patch := []byte(`{"spec":{"template":{"spec":{"containers":[{"name":"manager","image":"ghcr.io/butlerdotdev/butler-bootstrap:latest","imagePullPolicy":"IfNotPresent"}]}}}}`) + if _, err := clientset.AppsV1().Deployments(butlerNamespace).Patch( + ctx, "butler-bootstrap-controller", types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("patching bootstrap controller to local image: %w", err) + } + o.logger.Info("patched bootstrap controller to local image", "image", "ghcr.io/butlerdotdev/butler-bootstrap:latest") + } + // Wait for controllers to be ready o.logger.Debug("waiting for controllers to be ready") From 4b6b77481c01ce6b6b3adeb05d045878ee525bb7 Mon Sep 17 00:00:00 2001 From: atbagan Date: Wed, 10 Jun 2026 18:11:49 -0500 Subject: [PATCH 08/19] fix: recreate bootstrap deployment when swapping to local image --- internal/adm/bootstrap/orchestrator/orchestrator.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/adm/bootstrap/orchestrator/orchestrator.go b/internal/adm/bootstrap/orchestrator/orchestrator.go index 637c53b..b5e0bb6 100644 --- a/internal/adm/bootstrap/orchestrator/orchestrator.go +++ b/internal/adm/bootstrap/orchestrator/orchestrator.go @@ -1128,7 +1128,10 @@ func (o *Orchestrator) deployControllers(ctx context.Context, clientset *kuberne // the deployment at that local image so the IsLocal install path runs (otherwise // the published controller treats local as a VM provider and provisions machines). if o.isLocal { - patch := []byte(`{"spec":{"template":{"spec":{"containers":[{"name":"manager","image":"ghcr.io/butlerdotdev/butler-bootstrap:latest","imagePullPolicy":"IfNotPresent"}]}}}}`) + // Recreate strategy: the controller binds a host port, so a rolling update + // would leave the new pod Pending on a port conflict while the old (published) + // pod keeps reconciling. Recreate deletes the old pod first. + patch := []byte(`{"spec":{"strategy":{"$retainKeys":["type"],"type":"Recreate"},"template":{"spec":{"containers":[{"name":"manager","image":"ghcr.io/butlerdotdev/butler-bootstrap:latest","imagePullPolicy":"IfNotPresent"}]}}}}`) if _, err := clientset.AppsV1().Deployments(butlerNamespace).Patch( ctx, "butler-bootstrap-controller", types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil { return fmt.Errorf("patching bootstrap controller to local image: %w", err) From 959a27b9fe62a1191b79cb86b607168033785e8c Mon Sep 17 00:00:00 2001 From: atbagan Date: Wed, 10 Jun 2026 18:32:58 -0500 Subject: [PATCH 09/19] fix: grant bootstrap controller cluster-admin for local in-cluster install --- .../adm/bootstrap/orchestrator/orchestrator.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/internal/adm/bootstrap/orchestrator/orchestrator.go b/internal/adm/bootstrap/orchestrator/orchestrator.go index b5e0bb6..ad00011 100644 --- a/internal/adm/bootstrap/orchestrator/orchestrator.go +++ b/internal/adm/bootstrap/orchestrator/orchestrator.go @@ -34,6 +34,7 @@ import ( "github.com/butlerdotdev/butler/internal/adm/bootstrap/manifests" "github.com/butlerdotdev/butler/internal/common/log" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -1137,6 +1138,23 @@ func (o *Orchestrator) deployControllers(ctx context.Context, clientset *kuberne return fmt.Errorf("patching bootstrap controller to local image: %w", err) } o.logger.Info("patched bootstrap controller to local image", "image", "ghcr.io/butlerdotdev/butler-bootstrap:latest") + + // For local the controller installs addons onto its own cluster using its + // in-cluster credentials. Its default RBAC is scoped for remote installs via a + // provisioned kubeconfig, so grant it cluster-admin on this (local) cluster. + crb := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "butler-local-bootstrap-admin"}, + RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: "cluster-admin"}, + Subjects: []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: "butler-bootstrap-controller", + Namespace: butlerNamespace, + }}, + } + if _, err := clientset.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}); err != nil && !strings.Contains(err.Error(), "already exists") { + return fmt.Errorf("granting bootstrap controller cluster-admin for local: %w", err) + } + o.logger.Info("granted bootstrap controller cluster-admin for local install") } // Wait for controllers to be ready From 4c7d50fb3d11e7730ca4fc13cf08e30091ac7f81 Mon Sep 17 00:00:00 2001 From: atbagan Date: Wed, 10 Jun 2026 19:14:18 -0500 Subject: [PATCH 10/19] fix: avoid doubled tag in local butler-controller image --- internal/adm/bootstrap/local.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/adm/bootstrap/local.go b/internal/adm/bootstrap/local.go index 05aaf39..f806cdb 100644 --- a/internal/adm/bootstrap/local.go +++ b/internal/adm/bootstrap/local.go @@ -118,9 +118,10 @@ func localConfig(name string) *orchestrator.Config { Storage: orchestrator.StorageConfig{Type: "none"}, LoadBalancer: orchestrator.LoadBalancerConfig{Type: "metallb"}, CAPI: orchestrator.CAPIConfig{Version: "v1.9.4"}, - ButlerController: orchestrator.ButlerControllerConfig{ - Image: "ghcr.io/butlerdotdev/butler-controller:latest", - }, + // ButlerController image is left at its default (ghcr...butler-controller:latest), + // which matches the image buildAndLoadImages builds and loads for local. + // GetButlerControllerImage appends the version, so setting an image with a tag + // here would produce a doubled :latest:latest reference. }, } } From f96a3826cc84bdcbefb3832d5a96540c5860f19c Mon Sep 17 00:00:00 2001 From: atbagan Date: Wed, 10 Jun 2026 21:31:31 -0500 Subject: [PATCH 11/19] fix: build local steward image and apply local stability profile --- .../bootstrap/orchestrator/orchestrator.go | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/internal/adm/bootstrap/orchestrator/orchestrator.go b/internal/adm/bootstrap/orchestrator/orchestrator.go index ad00011..962e972 100644 --- a/internal/adm/bootstrap/orchestrator/orchestrator.go +++ b/internal/adm/bootstrap/orchestrator/orchestrator.go @@ -351,6 +351,19 @@ func (o *Orchestrator) Run(ctx context.Context, cfg *Config) error { } } + // Local stability profile: set guaranteed hosted control plane resources (so the + // tenant apiserver is not CPU-starved) and grant the console and controller the + // broader access the in-cluster local flow needs (console impersonates users for + // team creation; the controller lists cluster-scoped nodes/users). Best-effort. + if o.isLocal { + o.logger.Phase("Applying local stability profile") + home, _ := os.UserHomeDir() + mgmtKubeconfig := filepath.Join(home, ".butler", cfg.Cluster.Name+"-kubeconfig") + if err := o.configureLocalProfile(ctx, mgmtKubeconfig); err != nil { + o.logger.Warn("Failed to apply local stability profile", "error", err) + } + } + o.logger.Success("Bootstrap complete!") o.emit(Event{Type: EventSuccess, Phase: "Complete", Message: "Bootstrap complete"}) o.logger.Info("") @@ -520,6 +533,40 @@ func (o *Orchestrator) scanCertDirectory(dir string) []string { return certs } +// configureLocalProfile applies the local stability profile to the management cluster: +// guaranteed hosted control plane resources (so the tenant apiserver is not CPU-starved) +// and cluster-admin for the console and controller service accounts (the console +// impersonates users for team creation, the controller lists cluster-scoped nodes/users). +// Best-effort: individual failures are logged, not fatal. +func (o *Orchestrator) configureLocalProfile(ctx context.Context, kubeconfigPath string) error { + clientset, dynamicClient, err := o.createClients(kubeconfigPath) + if err != nil { + return fmt.Errorf("creating clients: %w", err) + } + + resPatch := []byte(`{"spec":{"defaultControlPlaneResources":{` + + `"apiServer":{"requests":{"cpu":"250m","memory":"512Mi"},"limits":{"cpu":"2","memory":"1Gi"}},` + + `"controllerManager":{"requests":{"cpu":"100m","memory":"128Mi"}},` + + `"scheduler":{"requests":{"cpu":"50m","memory":"64Mi"}}}}}`) + if _, err := dynamicClient.Resource(butlerConfigGVR).Patch(ctx, "butler", types.MergePatchType, resPatch, metav1.PatchOptions{}); err != nil { + o.logger.Warn("Failed to set control plane resources on ButlerConfig", "error", err) + } + + for _, sa := range []string{"butler-console", "butler-controller"} { + crb := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "butler-local-" + sa + "-admin"}, + RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: "cluster-admin"}, + Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: sa, Namespace: butlerNamespace}}, + } + if _, err := clientset.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}); err != nil && !strings.Contains(err.Error(), "already exists") { + o.logger.Warn("Failed to grant cluster-admin", "serviceAccount", sa, "error", err) + } + } + + o.logger.Success("Local stability profile applied") + return nil +} + // deriveKindLBPool inspects the kind docker network and returns a MetalLB pool high in // its IPv4 subnet, so LoadBalancer IPs are reachable by CAPD worker containers on the // same network. @@ -1940,6 +1987,14 @@ func (o *Orchestrator) buildAndLoadImages(ctx context.Context, provider string) // replace), so it builds with its own repo as the context. useParentContext: false, }) + // Steward carries local-only fixes not yet in a published release (the hosted + // control plane admin RBAC binding). InstallSteward uses :latest for local. + images = append(images, buildImage{ + name: "steward", + repoDir: filepath.Join(o.options.RepoRoot, "steward"), + image: "ghcr.io/butlerdotdev/steward:latest", + useParentContext: false, + }) } else { images = append(images, buildImage{ name: fmt.Sprintf("butler-provider-%s", provider), From 7418c7977182cb360a52c78f213efadda67b9e1c Mon Sep 17 00:00:00 2001 From: atbagan Date: Thu, 11 Jun 2026 08:19:19 -0500 Subject: [PATCH 12/19] fix: expose local console on host port 8080 automatically --- internal/adm/bootstrap/orchestrator/orchestrator.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/adm/bootstrap/orchestrator/orchestrator.go b/internal/adm/bootstrap/orchestrator/orchestrator.go index 962e972..3ff495e 100644 --- a/internal/adm/bootstrap/orchestrator/orchestrator.go +++ b/internal/adm/bootstrap/orchestrator/orchestrator.go @@ -563,6 +563,16 @@ func (o *Orchestrator) configureLocalProfile(ctx context.Context, kubeconfigPath } } + // Expose the console on the kind node's mapped host port so it is reachable at + // http://localhost:8080 with no port-forward. The KIND config maps host 8080 to + // container NodePort 30080 (see buildKINDConfig); flip the frontend Service to it. + consolePatch := []byte(`{"spec":{"type":"NodePort","ports":[{"port":80,"nodePort":30080}]}}`) + if _, err := clientset.CoreV1().Services(butlerNamespace).Patch(ctx, "butler-console-frontend", types.StrategicMergePatchType, consolePatch, metav1.PatchOptions{}); err != nil { + o.logger.Warn("Failed to expose console on host port 8080", "error", err) + } else { + o.logger.Info("Console reachable at http://localhost:8080") + } + o.logger.Success("Local stability profile applied") return nil } From 0169b5124b87b4cec0237a35bf005699513ae64b Mon Sep 17 00:00:00 2001 From: atbagan Date: Thu, 11 Jun 2026 08:31:49 -0500 Subject: [PATCH 13/19] fix: grant local console admin user cluster-admin for team management --- internal/adm/bootstrap/orchestrator/orchestrator.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/adm/bootstrap/orchestrator/orchestrator.go b/internal/adm/bootstrap/orchestrator/orchestrator.go index 3ff495e..0f52bb8 100644 --- a/internal/adm/bootstrap/orchestrator/orchestrator.go +++ b/internal/adm/bootstrap/orchestrator/orchestrator.go @@ -563,6 +563,18 @@ func (o *Orchestrator) configureLocalProfile(ctx context.Context, kubeconfigPath } } + // The console impersonates the logged-in admin user when writing platform resources + // (teams, users). The default admin login identity is admin@butler.local, which has + // no RBAC binding out of the box, so grant it cluster-admin for the local demo. + adminCRB := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "butler-local-admin-user"}, + RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: "cluster-admin"}, + Subjects: []rbacv1.Subject{{APIGroup: "rbac.authorization.k8s.io", Kind: "User", Name: "admin@butler.local"}}, + } + if _, err := clientset.RbacV1().ClusterRoleBindings().Create(ctx, adminCRB, metav1.CreateOptions{}); err != nil && !strings.Contains(err.Error(), "already exists") { + o.logger.Warn("Failed to grant admin user cluster-admin", "error", err) + } + // Expose the console on the kind node's mapped host port so it is reachable at // http://localhost:8080 with no port-forward. The KIND config maps host 8080 to // container NodePort 30080 (see buildKINDConfig); flip the frontend Service to it. From 22d3b2e3f1fe13597dd9a158f719a9c0758785c0 Mon Sep 17 00:00:00 2001 From: atbagan Date: Thu, 11 Jun 2026 08:54:21 -0500 Subject: [PATCH 14/19] fix: grant both admin identities cluster-admin on local across server fix --- .../bootstrap/orchestrator/orchestrator.go | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/internal/adm/bootstrap/orchestrator/orchestrator.go b/internal/adm/bootstrap/orchestrator/orchestrator.go index 0f52bb8..36c06bc 100644 --- a/internal/adm/bootstrap/orchestrator/orchestrator.go +++ b/internal/adm/bootstrap/orchestrator/orchestrator.go @@ -564,15 +564,20 @@ func (o *Orchestrator) configureLocalProfile(ctx context.Context, kubeconfigPath } // The console impersonates the logged-in admin user when writing platform resources - // (teams, users). The default admin login identity is admin@butler.local, which has - // no RBAC binding out of the box, so grant it cluster-admin for the local demo. - adminCRB := &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{Name: "butler-local-admin-user"}, - RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: "cluster-admin"}, - Subjects: []rbacv1.Subject{{APIGroup: "rbac.authorization.k8s.io", Kind: "User", Name: "admin@butler.local"}}, - } - if _, err := clientset.RbacV1().ClusterRoleBindings().Create(ctx, adminCRB, metav1.CreateOptions{}); err != nil && !strings.Contains(err.Error(), "already exists") { - o.logger.Warn("Failed to grant admin user cluster-admin", "error", err) + // (teams, users), but the seeded admin User has no RBAC binding out of the box. + // Grant cluster-admin to both possible login identities so local works whether the + // running butler-server uses the legacy admin@butler.local identity or the corrected + // admin@localhost one. Once the butler-server and butler-addons fixes ship, the + // chart-seeded binding covers this and the grant is redundant. + for _, email := range []string{"admin@localhost", "admin@butler.local"} { + adminCRB := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "butler-local-admin-" + strings.ReplaceAll(email, "@", "-at-")}, + RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: "cluster-admin"}, + Subjects: []rbacv1.Subject{{APIGroup: "rbac.authorization.k8s.io", Kind: "User", Name: email}}, + } + if _, err := clientset.RbacV1().ClusterRoleBindings().Create(ctx, adminCRB, metav1.CreateOptions{}); err != nil && !strings.Contains(err.Error(), "already exists") { + o.logger.Warn("Failed to grant admin user cluster-admin", "user", email, "error", err) + } } // Expose the console on the kind node's mapped host port so it is reachable at From e17fe9c0016b097c4320c02cd58b115d03701f8e Mon Sep 17 00:00:00 2001 From: atbagan Date: Thu, 11 Jun 2026 09:00:05 -0500 Subject: [PATCH 15/19] fix: simplify local console admin grant to the console login identity --- .../bootstrap/orchestrator/orchestrator.go | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/internal/adm/bootstrap/orchestrator/orchestrator.go b/internal/adm/bootstrap/orchestrator/orchestrator.go index 36c06bc..9bcae8b 100644 --- a/internal/adm/bootstrap/orchestrator/orchestrator.go +++ b/internal/adm/bootstrap/orchestrator/orchestrator.go @@ -563,21 +563,19 @@ func (o *Orchestrator) configureLocalProfile(ctx context.Context, kubeconfigPath } } - // The console impersonates the logged-in admin user when writing platform resources - // (teams, users), but the seeded admin User has no RBAC binding out of the box. - // Grant cluster-admin to both possible login identities so local works whether the - // running butler-server uses the legacy admin@butler.local identity or the corrected - // admin@localhost one. Once the butler-server and butler-addons fixes ship, the - // chart-seeded binding covers this and the grant is redundant. - for _, email := range []string{"admin@localhost", "admin@butler.local"} { - adminCRB := &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{Name: "butler-local-admin-" + strings.ReplaceAll(email, "@", "-at-")}, - RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: "cluster-admin"}, - Subjects: []rbacv1.Subject{{APIGroup: "rbac.authorization.k8s.io", Kind: "User", Name: email}}, - } - if _, err := clientset.RbacV1().ClusterRoleBindings().Create(ctx, adminCRB, metav1.CreateOptions{}); err != nil && !strings.Contains(err.Error(), "already exists") { - o.logger.Warn("Failed to grant admin user cluster-admin", "user", email, "error", err) - } + // Normally the admin gets platform-admin RBAC via the CLI device flow (butleradm + // login creates a ServiceAccount + ClusterRoleBinding to butler-cli-platform-admin). + // On local we drive everything through the console, whose legacy login impersonates + // the admin@butler.local identity directly, so bind that identity to cluster-admin + // for the console-driven flow. (Using butleradm/butlerctl against the local + // kubeconfig works the same way it does on a normal bootstrap.) + adminCRB := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "butler-local-admin-user"}, + RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: "cluster-admin"}, + Subjects: []rbacv1.Subject{{APIGroup: "rbac.authorization.k8s.io", Kind: "User", Name: "admin@butler.local"}}, + } + if _, err := clientset.RbacV1().ClusterRoleBindings().Create(ctx, adminCRB, metav1.CreateOptions{}); err != nil && !strings.Contains(err.Error(), "already exists") { + o.logger.Warn("Failed to grant admin user cluster-admin", "error", err) } // Expose the console on the kind node's mapped host port so it is reachable at From f4da4f0f6268cec93343c41a54b00a4f36606363 Mon Sep 17 00:00:00 2001 From: atbagan Date: Thu, 11 Jun 2026 09:48:35 -0500 Subject: [PATCH 16/19] feat: print local console url, username and password at end of bootstrap --- .../bootstrap/orchestrator/orchestrator.go | 42 ++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/internal/adm/bootstrap/orchestrator/orchestrator.go b/internal/adm/bootstrap/orchestrator/orchestrator.go index 9bcae8b..9a123df 100644 --- a/internal/adm/bootstrap/orchestrator/orchestrator.go +++ b/internal/adm/bootstrap/orchestrator/orchestrator.go @@ -355,13 +355,16 @@ func (o *Orchestrator) Run(ctx context.Context, cfg *Config) error { // tenant apiserver is not CPU-starved) and grant the console and controller the // broader access the in-cluster local flow needs (console impersonates users for // team creation; the controller lists cluster-scoped nodes/users). Best-effort. + home, _ := os.UserHomeDir() + mgmtKubeconfig := filepath.Join(home, ".butler", cfg.Cluster.Name+"-kubeconfig") + + var consolePassword string if o.isLocal { o.logger.Phase("Applying local stability profile") - home, _ := os.UserHomeDir() - mgmtKubeconfig := filepath.Join(home, ".butler", cfg.Cluster.Name+"-kubeconfig") if err := o.configureLocalProfile(ctx, mgmtKubeconfig); err != nil { o.logger.Warn("Failed to apply local stability profile", "error", err) } + consolePassword = o.getConsoleAdminPassword(ctx, mgmtKubeconfig) } o.logger.Success("Bootstrap complete!") @@ -372,6 +375,24 @@ func (o *Orchestrator) Run(ctx context.Context, cfg *Config) error { o.logger.Info(" Talosconfig: ~/.butler/" + cfg.Cluster.Name + "-talosconfig") o.logger.Info("") + // For local, the console is exposed on the host and we print the full sign-in + // details so the operator does not have to go looking for the password. + if o.isLocal { + o.logger.Info("Butler Console:") + o.logger.Info(" URL: http://localhost:8080") + o.logger.Info(" Username: admin") + if consolePassword != "" { + o.logger.Info(" Password: " + consolePassword) + } else { + o.logger.Info(" Password: kubectl get secret butler-console-admin -n butler-system -o jsonpath='{.data.admin-password}' | base64 -d") + } + o.logger.Info("") + o.logger.Info("Usage:") + o.logger.Info(" export KUBECONFIG=~/.butler/" + cfg.Cluster.Name + "-kubeconfig") + o.logger.Info(" kubectl get nodes") + return nil + } + if creds.consoleURL != "" { o.logger.Info("Butler Console:") if strings.HasPrefix(creds.consoleURL, "kubectl") { @@ -395,6 +416,23 @@ func (o *Orchestrator) Run(ctx context.Context, cfg *Config) error { return nil } +// getConsoleAdminPassword reads the bootstrap admin password from the +// butler-console-admin Secret. Returns "" if it cannot be read. +func (o *Orchestrator) getConsoleAdminPassword(ctx context.Context, kubeconfigPath string) string { + clientset, _, err := o.createClients(kubeconfigPath) + if err != nil { + return "" + } + secret, err := clientset.CoreV1().Secrets(butlerNamespace).Get(ctx, "butler-console-admin", metav1.GetOptions{}) + if err != nil { + return "" + } + if pw, ok := secret.Data["admin-password"]; ok { + return string(pw) + } + return "" +} + // dryRun shows what would be created func (o *Orchestrator) dryRun(cfg *Config) error { o.logger.Info("DRY RUN - showing what would be created") From 1c07770d4c638b945434e54a20248739b79223f7 Mon Sep 17 00:00:00 2001 From: atbagan Date: Thu, 11 Jun 2026 09:52:52 -0500 Subject: [PATCH 17/19] feat: add local provider to the bootstrap TUI wizard --- internal/tui/cmd.go | 23 +++++--- internal/tui/wizard/forms.go | 99 ++++++++++++++++++----------------- internal/tui/wizard/wizard.go | 39 ++++++++++++-- 3 files changed, 102 insertions(+), 59 deletions(-) diff --git a/internal/tui/cmd.go b/internal/tui/cmd.go index be79e6e..3a8ac70 100644 --- a/internal/tui/cmd.go +++ b/internal/tui/cmd.go @@ -164,14 +164,23 @@ func runBootstrap(cfg *orchestrator.Config, skipCleanup bool) error { logger := log.New("butleradm") + orcOpts := orchestrator.Options{ + Timeout: 30 * time.Minute, + SkipCleanup: skipCleanup, + } + // The local provider builds the controller, bootstrap, and steward images from + // the sibling butlerdotdev repos, so it runs in local-dev mode. + if cfg.Provider == "local" { + orcOpts.LocalDev = true + home, _ := os.UserHomeDir() + orcOpts.RepoRoot = filepath.Join(home, "code", "github.com", "butlerdotdev") + } + return bootstrap.Run(bootstrap.RunConfig{ - Ctx: ctx, - Cancel: cancel, - Cfg: cfg, - OrcOptions: orchestrator.Options{ - Timeout: 30 * time.Minute, - SkipCleanup: skipCleanup, - }, + Ctx: ctx, + Cancel: cancel, + Cfg: cfg, + OrcOptions: orcOpts, LoggerName: logger.Name(), LogLevel: logger.Level(), }) diff --git a/internal/tui/wizard/forms.go b/internal/tui/wizard/forms.go index 2b7fcd8..bdb5e80 100644 --- a/internal/tui/wizard/forms.go +++ b/internal/tui/wizard/forms.go @@ -38,6 +38,7 @@ func providerSelectGroup(s *wizardState) *huh.Group { huh.NewSelect[string](). Title("Infrastructure Provider"). Options( + huh.NewOption("Local (laptop, KIND + CAPD, no hypervisor)", "local"), huh.NewOption("Harvester (HCI)", "harvester"), huh.NewOption("Nutanix (AHV)", "nutanix"), huh.NewOption("Amazon Web Services (AWS)", "aws"), @@ -716,14 +717,14 @@ func ipamStep(s *wizardState) *huh.Group { huh.NewNote(). Title("Tenant IP Allocation (IPAM)"). Description( - "Butler uses a NetworkPool CR to allocate IPs to tenant clusters\n" + - "on on-prem providers (Harvester, Nutanix, Proxmox). Without one,\n" + - "tenant creation fails at IP allocation and you have to build the\n" + - "NetworkPool and ProviderConfig.spec.network by hand later.\n\n" + - "Enabling IPAM here makes the bootstrap emit the NetworkPool and\n" + - "wire the ProviderConfig to use it — the management cluster is\n" + - "tenant-ready the moment bootstrap completes.\n\n" + - "Cloud providers (AWS, Azure, GCP) use native networking and\n" + + "Butler uses a NetworkPool CR to allocate IPs to tenant clusters\n"+ + "on on-prem providers (Harvester, Nutanix, Proxmox). Without one,\n"+ + "tenant creation fails at IP allocation and you have to build the\n"+ + "NetworkPool and ProviderConfig.spec.network by hand later.\n\n"+ + "Enabling IPAM here makes the bootstrap emit the NetworkPool and\n"+ + "wire the ProviderConfig to use it — the management cluster is\n"+ + "tenant-ready the moment bootstrap completes.\n\n"+ + "Cloud providers (AWS, Azure, GCP) use native networking and\n"+ "should leave this disabled."), huh.NewConfirm(). @@ -741,18 +742,18 @@ func networkPoolStep(s *wizardState) *huh.Group { huh.NewNote(). Title("NetworkPool"). Description( - "Pool CIDR is the full network range the pool manages.\n\n" + - "Reserved Range: IPs outside DHCP that the management cluster\n" + - "uses for static assignments (VIP, MetalLB pool, node IPs).\n" + - "The IPAM allocator will never hand these to tenants. Use\n" + - "start/end IPs — doesn't need to be a power-of-2 block.\n\n" + - "Example: subnet 10.92.90.0/23, IPs .1 through .10 are\n" + - "outside DHCP for static use:\n" + - " Reserved: 10.92.90.1 - 10.92.90.10\n" + - " VIP: 10.92.90.5, LB pool: .33-.254\n" + - " Tenant alloc: 10.92.90.33 - 10.92.91.254\n\n" + - "Tenant Allocation is the sub-range the pool hands out to\n" + - "tenant clusters. Everything outside it (including the\n" + + "Pool CIDR is the full network range the pool manages.\n\n"+ + "Reserved Range: IPs outside DHCP that the management cluster\n"+ + "uses for static assignments (VIP, MetalLB pool, node IPs).\n"+ + "The IPAM allocator will never hand these to tenants. Use\n"+ + "start/end IPs — doesn't need to be a power-of-2 block.\n\n"+ + "Example: subnet 10.92.90.0/23, IPs .1 through .10 are\n"+ + "outside DHCP for static use:\n"+ + " Reserved: 10.92.90.1 - 10.92.90.10\n"+ + " VIP: 10.92.90.5, LB pool: .33-.254\n"+ + " Tenant alloc: 10.92.90.33 - 10.92.91.254\n\n"+ + "Tenant Allocation is the sub-range the pool hands out to\n"+ + "tenant clusters. Everything outside it (including the\n"+ "reserved range) is untouched by IPAM."), huh.NewInput(). @@ -818,10 +819,10 @@ func providerNetworkStep(s *wizardState) *huh.Group { huh.NewNote(). Title("Tenant LB Allocation"). Description( - "These settings go into ProviderConfig.spec.network and control\n" + - "how aggressively the platform allocates LoadBalancer IPs to each\n" + - "tenant cluster, plus the hard caps on per-tenant IP usage.\n\n" + - "Defaults match butler-beta's production values — safe starting\n" + + "These settings go into ProviderConfig.spec.network and control\n"+ + "how aggressively the platform allocates LoadBalancer IPs to each\n"+ + "tenant cluster, plus the hard caps on per-tenant IP usage.\n\n"+ + "Defaults match butler-beta's production values — safe starting\n"+ "point that you can tune later via kubectl edit providerconfig."), huh.NewSelect[string](). @@ -873,15 +874,15 @@ func multiTenancyStep(s *wizardState) *huh.Group { huh.NewNote(). Title("Multi-Tenancy"). Description( - "Butler supports two multi-tenancy modes:\n\n" + - "Optional (default):\n" + - " Teams are available but not required. Clusters can be created\n" + - " without belonging to a team. Good for small teams or dev\n" + - " environments where you want flexibility.\n\n" + - "Enforced:\n" + - " Every cluster must belong to a team. Team quotas are enforced.\n" + - " Required for production multi-tenant platforms where teams\n" + - " need resource boundaries and RBAC isolation.\n\n" + + "Butler supports two multi-tenancy modes:\n\n"+ + "Optional (default):\n"+ + " Teams are available but not required. Clusters can be created\n"+ + " without belonging to a team. Good for small teams or dev\n"+ + " environments where you want flexibility.\n\n"+ + "Enforced:\n"+ + " Every cluster must belong to a team. Team quotas are enforced.\n"+ + " Required for production multi-tenant platforms where teams\n"+ + " need resource boundaries and RBAC isolation.\n\n"+ "This setting is changeable later by editing the ButlerConfig CR."), huh.NewSelect[string](). @@ -902,17 +903,17 @@ func exposureModeStep(s *wizardState) *huh.Group { huh.NewNote(). Title("Control Plane Exposure"). Description( - "How should tenant-cluster Kubernetes API servers be reached?\n\n" + - "LoadBalancer (recommended for starters):\n" + - " 1 LoadBalancer IP per tenant cluster. Simplest setup.\n" + - " Burns 1 IP from the MetalLB pool per tenant.\n\n" + - "Ingress:\n" + - " Shared IP via an ingress controller with TLS passthrough.\n" + - " IP-efficient. Requires steward-tcp-proxy and a wildcard DNS\n" + - " record pointing at the ingress.\n\n" + - "Gateway:\n" + - " Shared IP via Gateway API TLSRoute. Modern alternative to\n" + - " Ingress. Requires a configured Gateway resource.\n\n" + + "How should tenant-cluster Kubernetes API servers be reached?\n\n"+ + "LoadBalancer (recommended for starters):\n"+ + " 1 LoadBalancer IP per tenant cluster. Simplest setup.\n"+ + " Burns 1 IP from the MetalLB pool per tenant.\n\n"+ + "Ingress:\n"+ + " Shared IP via an ingress controller with TLS passthrough.\n"+ + " IP-efficient. Requires steward-tcp-proxy and a wildcard DNS\n"+ + " record pointing at the ingress.\n\n"+ + "Gateway:\n"+ + " Shared IP via Gateway API TLSRoute. Modern alternative to\n"+ + " Ingress. Requires a configured Gateway resource.\n\n"+ "This setting is changeable later by editing the ButlerConfig CR."), huh.NewSelect[string](). @@ -991,11 +992,11 @@ func consoleStep(s *wizardState) *huh.Group { huh.NewNote(). Title("Butler Console"). Description( - "Butler Console is the web UI for managing tenant clusters, teams,\n" + - "and platform settings. It's installed automatically.\n\n" + - "By default the console is reachable only via port-forward:\n" + - " kubectl port-forward -n butler-system svc/butler-console-frontend 3000:80\n\n" + - "Enable ingress to expose it at a hostname instead. You'll need\n" + + "Butler Console is the web UI for managing tenant clusters, teams,\n"+ + "and platform settings. It's installed automatically.\n\n"+ + "By default the console is reachable only via port-forward:\n"+ + " kubectl port-forward -n butler-system svc/butler-console-frontend 3000:80\n\n"+ + "Enable ingress to expose it at a hostname instead. You'll need\n"+ "a DNS record pointing at the ingress IP."), huh.NewInput(). diff --git a/internal/tui/wizard/wizard.go b/internal/tui/wizard/wizard.go index c1f1a24..14fdebb 100644 --- a/internal/tui/wizard/wizard.go +++ b/internal/tui/wizard/wizard.go @@ -73,6 +73,13 @@ func Run() (*orchestrator.Config, error) { return nil, fmt.Errorf("wizard cancelled by user") } + // The local provider needs no infrastructure configuration: no + // credentials, discovery, sizing, or networking. Skip straight to the + // laptop defaults. + if s.Provider == "local" { + return buildConfig(s) + } + // Build the discovery client from wizard state and connect to the // provider, fetching root resources concurrently. creds := &stateCredentials{s: s} @@ -166,6 +173,32 @@ func Run() (*orchestrator.Config, error) { // the current main schema. Field-by-field drift from the feat branch wizard // lives here — keep this function in sync with orchestrator/config.go. func buildConfig(s *wizardState) (*orchestrator.Config, error) { + // Local provider: laptop defaults, mirrors the `bootstrap local` subcommand. + // Single control-plane node, Cilium on tenants (not the mgmt cluster), MetalLB, + // CAPI/CAPD; no Longhorn or kube-vip. Unused VM fields satisfy schema minimums. + if s.Provider == "local" { + name := s.ClusterName + if name == "" { + name = "butler-local" + } + return &orchestrator.Config{ + Provider: "local", + Cluster: orchestrator.ClusterConfig{ + Name: name, + Topology: "single-node", + ControlPlane: orchestrator.NodePoolConfig{Replicas: 1, CPU: 2, MemoryMB: 2048, DiskGB: 20}, + }, + Network: orchestrator.NetworkConfig{PodCIDR: "10.244.0.0/16", ServiceCIDR: "10.96.0.0/12"}, + Talos: orchestrator.TalosConfig{Version: "v1.9.3"}, + Addons: orchestrator.AddonsConfig{ + CNI: orchestrator.CNIConfig{Type: "none"}, + Storage: orchestrator.StorageConfig{Type: "none"}, + LoadBalancer: orchestrator.LoadBalancerConfig{Type: "metallb"}, + CAPI: orchestrator.CAPIConfig{Version: "v1.9.4"}, + }, + }, nil + } + cpReplicas, err := parseInt32(s.CPReplicas) if err != nil { return nil, fmt.Errorf("control plane replicas: %w", err) @@ -207,8 +240,8 @@ func buildConfig(s *wizardState) (*orchestrator.Config, error) { TimeServers: splitCSV(s.NTPServers), }, Addons: orchestrator.AddonsConfig{ - CNI: orchestrator.CNIConfig{Type: "cilium"}, - Storage: orchestrator.StorageConfig{Type: "longhorn"}, + CNI: orchestrator.CNIConfig{Type: "cilium"}, + Storage: orchestrator.StorageConfig{Type: "longhorn"}, GitOps: orchestrator.GitOpsConfig{Type: "flux"}, CAPI: orchestrator.CAPIConfig{Enabled: true}, ButlerController: orchestrator.ButlerControllerConfig{Enabled: true}, @@ -390,7 +423,7 @@ func buildConfig(s *wizardState) (*orchestrator.Config, error) { case "aws": cfg.ProviderConfig.AWS = &orchestrator.AWSProviderConfig{ AccessKeyID: s.AWSAccessKey, - SecretAccessKey: s.AWSSecretKey, + SecretAccessKey: s.AWSSecretKey, Region: s.AWSRegion, VPCID: s.AWSVPCID, SubnetID: s.AWSSubnetID, From 7640b3923e6d311fc0c986c03820d2256e31d168 Mon Sep 17 00:00:00 2001 From: atbagan Date: Thu, 11 Jun 2026 14:42:32 -0500 Subject: [PATCH 18/19] feat: pull published images by default and harden local bootstrap - Default `bootstrap local` (subcommand and TUI wizard) to pull published component images from ghcr; add --from-source to build from sibling repos for development. - Tear down the full prior local environment on a fresh bootstrap, including tenant clusters (each runs as its own KIND cluster via CAPD), so re-runs start from a clean slate. - Report the console URL, username, and password at the end of bootstrap and on the TUI completion screen; read the password from in-memory kubeconfig bytes. Hide the talosconfig line for local (no Talos nodes). - Capture image build/load output to a buffer so it does not corrupt the bootstrap TUI; surface it only on failure. - Make local ProviderConfig creation idempotent (tolerate AlreadyExists). --- internal/adm/bootstrap/local.go | 23 +- internal/adm/bootstrap/orchestrator/events.go | 2 + .../bootstrap/orchestrator/orchestrator.go | 203 ++++++++++++++++-- internal/tui/bootstrap/post_bootstrap.go | 20 +- internal/tui/cmd.go | 18 +- 5 files changed, 233 insertions(+), 33 deletions(-) diff --git a/internal/adm/bootstrap/local.go b/internal/adm/bootstrap/local.go index f806cdb..7c988fe 100644 --- a/internal/adm/bootstrap/local.go +++ b/internal/adm/bootstrap/local.go @@ -33,6 +33,7 @@ func NewLocalCmd(logger *log.Logger) *cobra.Command { clusterName string repoRoot string skipCleanup bool + fromSource bool ) cmd := &cobra.Command{ @@ -45,18 +46,25 @@ worker nodes run as containers via the Cluster API Docker provider (CAPD), so no hypervisor or cloud account is required. The KIND cluster is kept after bootstrap so you can create tenant clusters against it. +By default the published component images are pulled from ghcr.io, so nothing needs +to be built and no source checkout is required. + Prerequisites: - • Docker running locally - • kind and kubectl installed - • The butlerdotdev repos checked out as siblings (for building images from source) + • Docker running locally (at least 4 CPUs and 6 GB of memory) + • kubectl installed Example: - butleradm bootstrap local`, + butleradm bootstrap local + +Development: + --from-source builds butler-controller, butler-bootstrap, and steward from the + sibling butlerdotdev repos instead of pulling published images. Use it when + iterating on those components locally.`, RunE: func(cmd *cobra.Command, args []string) error { ctx, cancel := setupSignalHandler(cmd, logger) defer cancel() - if repoRoot == "" { + if fromSource && repoRoot == "" { home, _ := os.UserHomeDir() repoRoot = home + "/code/github.com/butlerdotdev" } @@ -66,7 +74,7 @@ Example: orch := orchestrator.New(logger, orchestrator.Options{ SkipCleanup: skipCleanup, Timeout: 30 * time.Minute, - LocalDev: true, + LocalDev: fromSource, RepoRoot: repoRoot, }) return orch.Run(ctx, cfg) @@ -74,7 +82,8 @@ Example: } cmd.Flags().StringVar(&clusterName, "name", "butler-local", "management cluster name") - cmd.Flags().StringVar(&repoRoot, "repo-root", "", "path to butlerdotdev repos (default: ~/code/github.com/butlerdotdev)") + cmd.Flags().BoolVar(&fromSource, "from-source", false, "build component images from the sibling butlerdotdev repos instead of pulling published images (for development)") + cmd.Flags().StringVar(&repoRoot, "repo-root", "", "path to butlerdotdev repos when using --from-source (default: ~/code/github.com/butlerdotdev)") cmd.Flags().BoolVar(&skipCleanup, "skip-cleanup", false, "no-op for local (the KIND cluster is always kept); accepted for parity") return cmd diff --git a/internal/adm/bootstrap/orchestrator/events.go b/internal/adm/bootstrap/orchestrator/events.go index 35e8c75..e993309 100644 --- a/internal/adm/bootstrap/orchestrator/events.go +++ b/internal/adm/bootstrap/orchestrator/events.go @@ -73,6 +73,8 @@ type ClusterCredentials struct { Talosconfig []byte ControlPlaneIPs []string ConsoleURL string + ConsoleUser string + ConsolePassword string } // EventSink receives events from the orchestrator. Implementations must be diff --git a/internal/adm/bootstrap/orchestrator/orchestrator.go b/internal/adm/bootstrap/orchestrator/orchestrator.go index 9a123df..8039a7a 100644 --- a/internal/adm/bootstrap/orchestrator/orchestrator.go +++ b/internal/adm/bootstrap/orchestrator/orchestrator.go @@ -18,6 +18,7 @@ limitations under the License. package orchestrator import ( + "bytes" "context" "encoding/base64" "encoding/json" @@ -35,6 +36,7 @@ import ( "github.com/butlerdotdev/butler/internal/common/log" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -86,6 +88,11 @@ var ( Version: butlerAPIVersion, Resource: "butlerconfigs", } + tenantClusterGVR = schema.GroupVersionResource{ + Group: butlerAPIGroup, + Version: butlerAPIVersion, + Resource: "tenantclusters", + } ) // Options configures the orchestrator @@ -118,6 +125,9 @@ type Orchestrator struct { // kindKubeconfigPath is the path to the KIND cluster's kubeconfig. For the local // provider this kubeconfig is also the management cluster kubeconfig. kindKubeconfigPath string + // consolePassword caches the console admin password once fetched, so the + // EventComplete emission and the end-of-run summary share one lookup. + consolePassword string } // New creates a new orchestrator @@ -358,12 +368,17 @@ func (o *Orchestrator) Run(ctx context.Context, cfg *Config) error { home, _ := os.UserHomeDir() mgmtKubeconfig := filepath.Join(home, ".butler", cfg.Cluster.Name+"-kubeconfig") - var consolePassword string if o.isLocal { o.logger.Phase("Applying local stability profile") if err := o.configureLocalProfile(ctx, mgmtKubeconfig); err != nil { o.logger.Warn("Failed to apply local stability profile", "error", err) } + } + + // The password was fetched at completion time; fall back to a path-based + // read here if that lookup came up empty. + consolePassword := o.consolePassword + if consolePassword == "" { consolePassword = o.getConsoleAdminPassword(ctx, mgmtKubeconfig) } @@ -401,8 +416,12 @@ func (o *Orchestrator) Run(ctx context.Context, cfg *Config) error { o.logger.Info(" URL: " + creds.consoleURL) } o.logger.Info(" Username: admin") - o.logger.Info(" Password: Run the following command to retrieve:") - o.logger.Info(" kubectl get secret butler-console-admin -n butler-system -o jsonpath='{.data.admin-password}' | base64 -d && echo") + if consolePassword != "" { + o.logger.Info(" Password: " + consolePassword) + } else { + o.logger.Info(" Password: Run the following command to retrieve:") + o.logger.Info(" kubectl get secret butler-console-admin -n butler-system -o jsonpath='{.data.admin-password}' | base64 -d && echo") + } o.logger.Info("") } @@ -433,6 +452,29 @@ func (o *Orchestrator) getConsoleAdminPassword(ctx context.Context, kubeconfigPa return "" } +// consoleAdminPasswordFromBytes reads the bootstrap admin password from the +// butler-console-admin Secret using in-memory kubeconfig bytes (the management +// kubeconfig file may not be written to ~/.butler yet at emit time). Returns "" +// if it cannot be read. +func (o *Orchestrator) consoleAdminPasswordFromBytes(ctx context.Context, kubeconfig []byte) string { + cfg, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig) + if err != nil { + return "" + } + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return "" + } + secret, err := clientset.CoreV1().Secrets(butlerNamespace).Get(ctx, "butler-console-admin", metav1.GetOptions{}) + if err != nil { + return "" + } + if pw, ok := secret.Data["admin-password"]; ok { + return string(pw) + } + return "" +} + // dryRun shows what would be created func (o *Orchestrator) dryRun(cfg *Config) error { o.logger.Info("DRY RUN - showing what would be created") @@ -811,6 +853,78 @@ func (o *Orchestrator) injectHostAliases(ctx context.Context, hostAliases []stri } // createKINDCluster creates a KIND cluster with the specified configuration +// teardownLocalEnvironment removes a prior local environment so a fresh bootstrap +// starts from a known-good state. It deletes the management KIND cluster and every +// tenant cluster the management cluster created (each local tenant runs as its own +// KIND cluster via CAPD). Tenant names are read from the management cluster while it +// is still up, so only Butler-created clusters are removed; unrelated KIND clusters +// on the machine are left untouched and surfaced as a warning. +func (o *Orchestrator) teardownLocalEnvironment(ctx context.Context, provider *cluster.Provider, existing []string) error { + existingSet := make(map[string]bool, len(existing)) + for _, c := range existing { + existingSet[c] = true + } + + // Discover tenant clusters from the management cluster before deleting it. + for _, name := range o.listLocalTenantClusters(ctx, provider) { + if name == "" || name == kindClusterName || !existingSet[name] { + continue + } + o.logger.Info("Deleting tenant cluster", "name", name) + if err := provider.Delete(name, ""); err != nil { + o.logger.Warn("Failed to delete tenant cluster", "name", name, "error", err) + } + delete(existingSet, name) + } + + // Delete the management cluster. + o.logger.Info("Deleting management cluster", "name", kindClusterName) + if err := provider.Delete(kindClusterName, ""); err != nil { + return fmt.Errorf("deleting existing KIND cluster: %w", err) + } + delete(existingSet, kindClusterName) + + // Surface any KIND clusters that were not recognized as Butler tenants (for + // example orphans from an interrupted prior run). They are not deleted because + // they cannot be confirmed as Butler-created. + if len(existingSet) > 0 { + leftover := make([]string, 0, len(existingSet)) + for c := range existingSet { + leftover = append(leftover, c) + } + o.logger.Warn("Other KIND clusters remain and were left untouched", "clusters", strings.Join(leftover, ", ")) + o.logger.Info("If any are leftover Butler tenants, remove with: kind delete cluster --name ") + } + + return nil +} + +// listLocalTenantClusters returns the names of TenantClusters registered on the +// existing management KIND cluster. Each local tenant runs as a KIND cluster with +// the same name. Best-effort: returns nil if the management cluster is unreachable. +func (o *Orchestrator) listLocalTenantClusters(ctx context.Context, provider *cluster.Provider) []string { + kubeconfigPath, err := o.getKINDKubeconfig(provider) + if err != nil { + o.logger.Debug("Could not read management kubeconfig for tenant discovery", "error", err) + return nil + } + _, dyn, err := o.createClients(kubeconfigPath) + if err != nil { + o.logger.Debug("Could not build client for tenant discovery", "error", err) + return nil + } + list, err := dyn.Resource(tenantClusterGVR).Namespace("").List(ctx, metav1.ListOptions{}) + if err != nil { + o.logger.Debug("Could not list tenant clusters for cleanup", "error", err) + return nil + } + names := make([]string, 0, len(list.Items)) + for _, item := range list.Items { + names = append(names, item.GetName()) + } + return names +} + func (o *Orchestrator) createKINDCluster(ctx context.Context, provider *cluster.Provider) (string, error) { // Check if cluster already exists clusters, err := provider.List() @@ -819,6 +933,17 @@ func (o *Orchestrator) createKINDCluster(ctx context.Context, provider *cluster. } for _, c := range clusters { if c == kindClusterName { + // The local provider's KIND cluster is permanent (no pivot), so a + // leftover from a prior run carries stale controller images and + // CRs. Tear it down and recreate clean rather than reuse, so a + // fresh bootstrap is always from a known-good state. + if o.isLocal { + o.logger.Warn("Existing local environment found, tearing it down for a clean bootstrap") + if err := o.teardownLocalEnvironment(ctx, provider, clusters); err != nil { + return "", err + } + break + } o.logger.Warn("KIND cluster already exists, reusing") kubeconfigPath, err := o.getKINDKubeconfig(provider) if err != nil { @@ -1234,20 +1359,26 @@ func (o *Orchestrator) deployControllers(ctx context.Context, clientset *kuberne return fmt.Errorf("deploying controllers: %w", err) } - // The embedded manifest pins a published bootstrap image tag, but for the local - // provider we just built and loaded the controller from source as :latest. Point - // the deployment at that local image so the IsLocal install path runs (otherwise - // the published controller treats local as a VM provider and provisions machines). + // Point the bootstrap deployment at the local-capable butler-bootstrap image so + // the IsLocal install path runs (otherwise the controller treats local as a VM + // provider and provisions machines). In the default pull mode the image is + // fetched from ghcr (Always, so a stranger gets the current published build); in + // --from-source mode it is the :latest image just built and loaded into KIND + // (IfNotPresent, so the local build is used rather than re-pulled). if o.isLocal { + pullPolicy := "Always" + if o.options.LocalDev { + pullPolicy = "IfNotPresent" + } // Recreate strategy: the controller binds a host port, so a rolling update - // would leave the new pod Pending on a port conflict while the old (published) - // pod keeps reconciling. Recreate deletes the old pod first. - patch := []byte(`{"spec":{"strategy":{"$retainKeys":["type"],"type":"Recreate"},"template":{"spec":{"containers":[{"name":"manager","image":"ghcr.io/butlerdotdev/butler-bootstrap:latest","imagePullPolicy":"IfNotPresent"}]}}}}`) + // would leave the new pod Pending on a port conflict while the old pod keeps + // reconciling. Recreate deletes the old pod first. + patch := []byte(fmt.Sprintf(`{"spec":{"strategy":{"$retainKeys":["type"],"type":"Recreate"},"template":{"spec":{"containers":[{"name":"manager","image":"ghcr.io/butlerdotdev/butler-bootstrap:latest","imagePullPolicy":%q}]}}}}`, pullPolicy)) if _, err := clientset.AppsV1().Deployments(butlerNamespace).Patch( ctx, "butler-bootstrap-controller", types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil { return fmt.Errorf("patching bootstrap controller to local image: %w", err) } - o.logger.Info("patched bootstrap controller to local image", "image", "ghcr.io/butlerdotdev/butler-bootstrap:latest") + o.logger.Info("patched bootstrap controller image", "image", "ghcr.io/butlerdotdev/butler-bootstrap:latest", "pullPolicy", pullPolicy) // For local the controller installs addons onto its own cluster using its // in-cluster credentials. Its default RBAC is scoped for remote installs via a @@ -1303,6 +1434,10 @@ func (o *Orchestrator) createProviderConfig(ctx context.Context, client dynamic. _, err := client.Resource(providerConfigGVR).Namespace(butlerNamespace).Create( ctx, pc, metav1.CreateOptions{}) if err != nil { + if apierrors.IsAlreadyExists(err) { + o.logger.Info("ProviderConfig already exists, reusing", "name", pc.GetName()) + return nil + } return fmt.Errorf("creating ProviderConfig: %w", err) } @@ -1900,6 +2035,19 @@ func (o *Orchestrator) watchBootstrap(ctx context.Context, client dynamic.Interf talosconfigBytes = tc } + // The local console is exposed on the host via the KIND NodePort + // mapping, so report the reachable host URL rather than the + // in-cluster Service or a port-forward command. + if o.isLocal { + consoleURL = "http://localhost:8080" + } + + // Surface the sign-in details on the completion screen so the + // operator does not have to go hunting for the password. The + // management kubeconfig file is not written yet, so read from the + // in-memory bytes. + o.consolePassword = o.consoleAdminPasswordFromBytes(ctx, kubeconfigBytes) + creds := &clusterCredentials{ kubeconfig: kubeconfigBytes, talosconfig: talosconfigBytes, @@ -1915,6 +2063,8 @@ func (o *Orchestrator) watchBootstrap(ctx context.Context, client dynamic.Interf Talosconfig: talosconfigBytes, ControlPlaneIPs: controlPlaneIPs, ConsoleURL: consoleURL, + ConsoleUser: "admin", + ConsolePassword: o.consolePassword, }, }) return creds, nil @@ -2103,22 +2253,31 @@ func (o *Orchestrator) buildAndLoadImages(ctx context.Context, provider string) buildCmd = exec.CommandContext(ctx, "docker", "build", "-t", img.image, ".") buildCmd.Dir = img.repoDir } - buildCmd.Stdout = os.Stdout - buildCmd.Stderr = os.Stderr + // Capture build output to a buffer instead of the terminal. Docker's + // buildx progress writes TTY control sequences that corrupt the + // bootstrap TUI's alt-screen. Surface the tail only if the build fails. + var buildOut bytes.Buffer + buildCmd.Stdout = &buildOut + buildCmd.Stderr = &buildOut + + // Force plain progress so a fallback to non-buffered output (or the + // error tail) is line-oriented rather than ANSI cursor movement. + buildCmd.Env = append(os.Environ(), "BUILDKIT_PROGRESS=plain", "DOCKER_BUILDKIT=1") if err := buildCmd.Run(); err != nil { - return fmt.Errorf("building %s: %w", img.name, err) + return fmt.Errorf("building %s: %w\n%s", img.name, err, tailLines(buildOut.String(), 30)) } o.logger.Success("built image", "image", img.image) // Load into KIND o.logger.Info("loading image into KIND", "image", img.image) loadCmd := exec.CommandContext(ctx, "kind", "load", "docker-image", img.image, "--name", kindClusterName) - loadCmd.Stdout = os.Stdout - loadCmd.Stderr = os.Stderr + var loadOut bytes.Buffer + loadCmd.Stdout = &loadOut + loadCmd.Stderr = &loadOut if err := loadCmd.Run(); err != nil { - return fmt.Errorf("loading %s into KIND: %w", img.name, err) + return fmt.Errorf("loading %s into KIND: %w\n%s", img.name, err, tailLines(loadOut.String(), 30)) } o.logger.Success("loaded image into KIND", "image", img.image) } @@ -2126,6 +2285,16 @@ func (o *Orchestrator) buildAndLoadImages(ctx context.Context, provider string) return nil } +// tailLines returns the last n lines of s, for surfacing the relevant tail of +// captured command output in an error without dumping the entire build log. +func tailLines(s string, n int) string { + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return strings.Join(lines, "\n") +} + // buildTalosSpec builds the talos section of the ClusterBootstrap spec. // Emits configPatches for optional overrides: NTP servers (when TimeServers // is set) and primary NIC MTU (when Network.MTU is non-zero). diff --git a/internal/tui/bootstrap/post_bootstrap.go b/internal/tui/bootstrap/post_bootstrap.go index 912d28a..aa854b4 100644 --- a/internal/tui/bootstrap/post_bootstrap.go +++ b/internal/tui/bootstrap/post_bootstrap.go @@ -95,14 +95,30 @@ func (m postBootstrapModel) viewSuccess(width int) string { b.WriteString(fmt.Sprintf(" %s %s\n", labelStyle.Render("Duration:"), valueStyle.Render(m.elapsed.Round(time.Second).String()))) kubeconfigPath := fmt.Sprintf("~/.butler/%s-kubeconfig", m.clusterName) - talosconfigPath := fmt.Sprintf("~/.butler/%s-talosconfig", m.clusterName) b.WriteString(fmt.Sprintf(" %s %s\n", labelStyle.Render("Kubeconfig:"), valueStyle.Render(kubeconfigPath))) - b.WriteString(fmt.Sprintf(" %s %s\n", labelStyle.Render("Talosconfig:"), valueStyle.Render(talosconfigPath))) + // The local provider runs on KIND and has no Talos node config. + if m.provider != "local" { + talosconfigPath := fmt.Sprintf("~/.butler/%s-talosconfig", m.clusterName) + b.WriteString(fmt.Sprintf(" %s %s\n", labelStyle.Render("Talosconfig:"), valueStyle.Render(talosconfigPath))) + } if m.creds != nil && m.creds.ConsoleURL != "" { b.WriteString(fmt.Sprintf(" %s %s\n", labelStyle.Render("Console URL:"), valueStyle.Render(m.creds.ConsoleURL))) } + if m.creds != nil && m.creds.ConsoleUser != "" { + b.WriteString(fmt.Sprintf(" %s %s\n", labelStyle.Render("Username:"), valueStyle.Render(m.creds.ConsoleUser))) + } + + if m.creds != nil { + if m.creds.ConsolePassword != "" { + b.WriteString(fmt.Sprintf(" %s %s\n", labelStyle.Render("Password:"), valueStyle.Render(m.creds.ConsolePassword))) + } else if m.creds.ConsoleURL != "" { + b.WriteString(fmt.Sprintf(" %s %s\n", labelStyle.Render("Password:"), + dimStyle.Render("kubectl get secret butler-console-admin -n butler-system -o jsonpath='{.data.admin-password}' | base64 -d"))) + } + } + if m.creds != nil && len(m.creds.ControlPlaneIPs) > 0 { b.WriteString(fmt.Sprintf(" %s %s\n", labelStyle.Render("CP Endpoint:"), valueStyle.Render(strings.Join(m.creds.ControlPlaneIPs, ", ")))) diff --git a/internal/tui/cmd.go b/internal/tui/cmd.go index 3a8ac70..cadda5c 100644 --- a/internal/tui/cmd.go +++ b/internal/tui/cmd.go @@ -40,6 +40,7 @@ import ( func NewTUICmd(admin bool) *cobra.Command { var kubeconfig string var skipCleanup bool + var fromSource bool cmd := &cobra.Command{ Use: "tui", @@ -71,12 +72,13 @@ Examples: SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { kubeContext, _ := cmd.Flags().GetString("context") - return runDashboardLoop(kubeconfig, kubeContext, admin, skipCleanup) + return runDashboardLoop(kubeconfig, kubeContext, admin, skipCleanup, fromSource) }, } cmd.Flags().StringVar(&kubeconfig, "kubeconfig", "", "path to kubeconfig file") cmd.Flags().BoolVar(&skipCleanup, "skip-cleanup", false, "don't delete KIND cluster on bootstrap failure (for debugging)") + cmd.Flags().BoolVar(&fromSource, "from-source", false, "for a local bootstrap, build component images from the sibling butlerdotdev repos instead of pulling published images (for development)") return cmd } @@ -88,7 +90,7 @@ Examples: // cluster's kubeconfig. This nested-TUI pattern works around Bubbletea's // exclusive grab on stdin — huh forms and bootstrap's own tea.Program can't // literally nest inside the dashboard tea.Program. -func runDashboardLoop(kubeconfig, kubeContext string, admin bool, skipCleanup bool) error { +func runDashboardLoop(kubeconfig, kubeContext string, admin bool, skipCleanup bool, fromSource bool) error { for { // 1. Try to build a Kubernetes client. A missing or invalid // kubeconfig is not a fatal error in admin mode — the app @@ -140,7 +142,7 @@ func runDashboardLoop(kubeconfig, kubeContext string, admin bool, skipCleanup bo continue } - if err := runBootstrap(cfg, skipCleanup); err != nil { + if err := runBootstrap(cfg, skipCleanup, fromSource); err != nil { fmt.Fprintf(os.Stderr, "\nbootstrap error: %v\n", err) fmt.Fprintf(os.Stderr, "Press Enter to return to the dashboard...\n") fmt.Scanln() @@ -158,7 +160,7 @@ func runDashboardLoop(kubeconfig, kubeContext string, admin bool, skipCleanup bo // runBootstrap hands a wizard-assembled Config to the bootstrap TUI. // Signal handling and orchestrator cleanup are delegated to bootstrap.Run. -func runBootstrap(cfg *orchestrator.Config, skipCleanup bool) error { +func runBootstrap(cfg *orchestrator.Config, skipCleanup bool, fromSource bool) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -168,9 +170,11 @@ func runBootstrap(cfg *orchestrator.Config, skipCleanup bool) error { Timeout: 30 * time.Minute, SkipCleanup: skipCleanup, } - // The local provider builds the controller, bootstrap, and steward images from - // the sibling butlerdotdev repos, so it runs in local-dev mode. - if cfg.Provider == "local" { + // The local provider pulls published component images from ghcr by default, so + // the wizard flow needs no source checkout. With --from-source it builds the + // controller, bootstrap, and steward images from the sibling butlerdotdev repos + // instead (for iterating on those components before they are published). + if cfg.Provider == "local" && fromSource { orcOpts.LocalDev = true home, _ := os.UserHomeDir() orcOpts.RepoRoot = filepath.Join(home, "code", "github.com", "butlerdotdev") From f164d17ab59719df20733eedea81e372056adef7 Mon Sep 17 00:00:00 2001 From: atbagan Date: Thu, 11 Jun 2026 20:01:10 -0500 Subject: [PATCH 19/19] style(tui): use -- instead of em-dashes in wizard help text Three wizard help strings used em-dashes; replace with -- to match the repo text convention. --- internal/tui/wizard/forms.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/tui/wizard/forms.go b/internal/tui/wizard/forms.go index bdb5e80..da765d9 100644 --- a/internal/tui/wizard/forms.go +++ b/internal/tui/wizard/forms.go @@ -722,7 +722,7 @@ func ipamStep(s *wizardState) *huh.Group { "tenant creation fails at IP allocation and you have to build the\n"+ "NetworkPool and ProviderConfig.spec.network by hand later.\n\n"+ "Enabling IPAM here makes the bootstrap emit the NetworkPool and\n"+ - "wire the ProviderConfig to use it — the management cluster is\n"+ + "wire the ProviderConfig to use it -- the management cluster is\n"+ "tenant-ready the moment bootstrap completes.\n\n"+ "Cloud providers (AWS, Azure, GCP) use native networking and\n"+ "should leave this disabled."), @@ -746,7 +746,7 @@ func networkPoolStep(s *wizardState) *huh.Group { "Reserved Range: IPs outside DHCP that the management cluster\n"+ "uses for static assignments (VIP, MetalLB pool, node IPs).\n"+ "The IPAM allocator will never hand these to tenants. Use\n"+ - "start/end IPs — doesn't need to be a power-of-2 block.\n\n"+ + "start/end IPs -- doesn't need to be a power-of-2 block.\n\n"+ "Example: subnet 10.92.90.0/23, IPs .1 through .10 are\n"+ "outside DHCP for static use:\n"+ " Reserved: 10.92.90.1 - 10.92.90.10\n"+ @@ -822,7 +822,7 @@ func providerNetworkStep(s *wizardState) *huh.Group { "These settings go into ProviderConfig.spec.network and control\n"+ "how aggressively the platform allocates LoadBalancer IPs to each\n"+ "tenant cluster, plus the hard caps on per-tenant IP usage.\n\n"+ - "Defaults match butler-beta's production values — safe starting\n"+ + "Defaults match butler-beta's production values -- safe starting\n"+ "point that you can tune later via kubectl edit providerconfig."), huh.NewSelect[string]().