From 3bddbf909fe598a3473306dde4189b0beaeac3d1 Mon Sep 17 00:00:00 2001 From: raphgm Date: Thu, 24 Sep 2026 14:56:17 +0100 Subject: [PATCH 1/4] feat(k8s): predict Kubernetes deployment failures against a specific cluster container-preflight k8s reads manifests (files, dirs, or helm/kustomize output on stdin), profiles a cluster with read-only kubectl calls (or a saved snapshot) and checks five rules: k8s.api (no matches for kind), k8s.image (ErrImagePull / ImagePullBackOff), k8s.platform (images that cannot run on eligible node architectures), k8s.schedulable (FailedScheduling with the scheduler's reasons) and k8s.storage (PVCs Pending on missing StorageClasses). Unreachable clusters and missing RBAC become root causes through the existing fact graph. --- cmd/k8s.go | 85 ++++++++ cmd/preflight.go | 6 + pkg/preflight/fact/fact.go | 12 + pkg/preflight/k8s/cluster.go | 265 +++++++++++++++++++++++ pkg/preflight/k8s/k8s_test.go | 284 ++++++++++++++++++++++++ pkg/preflight/k8s/manifest.go | 314 +++++++++++++++++++++++++++ pkg/preflight/k8s/quantity.go | 62 ++++++ pkg/preflight/k8s/rules.go | 397 ++++++++++++++++++++++++++++++++++ pkg/preflight/kubernetes.go | 90 ++++++++ pkg/preflight/preflight.go | 4 +- 10 files changed, 1518 insertions(+), 1 deletion(-) create mode 100644 cmd/k8s.go create mode 100644 pkg/preflight/k8s/cluster.go create mode 100644 pkg/preflight/k8s/k8s_test.go create mode 100644 pkg/preflight/k8s/manifest.go create mode 100644 pkg/preflight/k8s/quantity.go create mode 100644 pkg/preflight/k8s/rules.go create mode 100644 pkg/preflight/kubernetes.go diff --git a/cmd/k8s.go b/cmd/k8s.go new file mode 100644 index 0000000..8b9032c --- /dev/null +++ b/cmd/k8s.go @@ -0,0 +1,85 @@ +package cmd + +import ( + "encoding/json" + "os" + + "github.com/spf13/cobra" + + "github.com/raphgm/container-preflight/internal/executor" + "github.com/raphgm/container-preflight/pkg/preflight" + "github.com/raphgm/container-preflight/pkg/preflight/k8s" +) + +var ( + k8sContext string + k8sCluster string + k8sSaveTo string + k8sOffline bool + k8sFormat string +) + +var k8sCmd = &cobra.Command{ + Use: "k8s [manifests...]", + Short: "Predict what applying Kubernetes manifests to a cluster would fail on", + Long: `Reads manifests (files, directories, or - for stdin, e.g. from +` + "`helm template`" + ` or ` + "`kustomize build`" + `), profiles the target cluster with +read-only kubectl calls, and queries registries for image platforms. It +predicts FailedScheduling, ImagePullBackOff, "no matches for kind", Pending +volume claims and wrong-architecture images before anything is applied. + + container-preflight k8s ./deploy + helm template my-release ./chart | container-preflight k8s - + container-preflight k8s --save-cluster prod.json # capture a cluster + container-preflight k8s ./deploy --cluster prod.json # check against it`, + RunE: func(cmd *cobra.Command, args []string) error { + if k8sSaveTo != "" { + snap := k8s.Capture(cmd.Context(), executor.New(), k8sContext) + f, err := os.Create(k8sSaveTo) + if err != nil { + return err + } + defer f.Close() + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + return enc.Encode(snap) + } + if len(args) == 0 { + args = []string{"."} + } + opts := preflight.K8sOptions{Paths: args, Context: k8sContext, Offline: k8sOffline} + if k8sCluster != "" { + snap, err := k8s.LoadSnapshot(k8sCluster) + if err != nil { + return err + } + opts.Snapshot = snap + } + rep, err := preflight.RunK8s(cmd.Context(), opts) + if err != nil { + return err + } + if k8sFormat == "json" { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(rep); err != nil { + return err + } + } else { + renderPreflight(os.Stdout, rep, useColor()) + } + if rep.Errors() > 0 { + os.Exit(1) + } + return nil + }, +} + +func init() { + rootCmd.AddCommand(k8sCmd) + k8sCmd.Flags().StringVar(&k8sContext, "context", "", "kubeconfig context (default: current)") + k8sCmd.Flags().StringVar(&k8sCluster, "cluster", "", "Check against a saved cluster snapshot instead of a live cluster") + k8sCmd.Flags().StringVar(&k8sSaveTo, "save-cluster", "", "Save a snapshot of the cluster to this file and exit") + k8sCmd.Flags().BoolVar(&k8sOffline, "offline", false, "Skip registry lookups") + k8sCmd.Flags().StringVarP(&k8sFormat, "format", "f", "term", "Output format (term, json)") +} diff --git a/cmd/preflight.go b/cmd/preflight.go index ca146a8..28b63a1 100644 --- a/cmd/preflight.go +++ b/cmd/preflight.go @@ -101,6 +101,9 @@ func renderPreflight(w io.Writer, rep *preflight.Report, color bool) { } h := rep.Host + if h == nil { + h = &host.Profile{} + } fmt.Fprintf(w, "\n%sContainer Preflight · preflight%s %s%s%s\n", c.bold, c.reset, c.dim, rep.Dir, c.reset) if rep.HostName != "" { fmt.Fprintf(w, "%sPredicting for snapshot of %s%s\n", c.yellow, rep.HostName, c.reset) @@ -115,6 +118,9 @@ func renderPreflight(w io.Writer, rep *preflight.Report, color bool) { if h.BuildxInstalled { hostBits = append(hostBits, "buildx "+h.BuildxVersion) } + if c := rep.Cluster; c != nil && c.ServerVersion != "" { + hostBits = append(hostBits, "cluster "+c.Context, "Kubernetes "+c.ServerVersion, fmt.Sprintf("%d node(s): %s", len(c.Nodes), strings.Join(c.Platforms(), ", "))) + } if len(hostBits) > 0 { fmt.Fprintf(w, "%sHost: %s%s\n", c.dim, strings.Join(hostBits, " · "), c.reset) } diff --git a/pkg/preflight/fact/fact.go b/pkg/preflight/fact/fact.go index 089da3d..b7c76a0 100644 --- a/pkg/preflight/fact/fact.go +++ b/pkg/preflight/fact/fact.go @@ -17,6 +17,13 @@ const ( Registry ID = "registry.network" ComposeFile ID = "project.compose" Dockerfile ID = "project.dockerfile" + + Kubectl ID = "k8s.kubectl" + Cluster ID = "k8s.cluster" + ClusterAPIs ID = "k8s.apis" + Nodes ID = "k8s.nodes" + Storage ID = "k8s.storage" + Manifests ID = "project.manifests" ) // parents encodes the causal edges: a fact cannot be known unless its parent @@ -28,6 +35,11 @@ var parents = map[ID]ID{ LocalDaemon: DockerCLI, Compose: DockerCLI, Buildx: DockerCLI, + + Cluster: Kubectl, + ClusterAPIs: Cluster, + Nodes: Cluster, + Storage: Cluster, } // Parent returns the direct causal parent of id, if any. diff --git a/pkg/preflight/k8s/cluster.go b/pkg/preflight/k8s/cluster.go new file mode 100644 index 0000000..ea58317 --- /dev/null +++ b/pkg/preflight/k8s/cluster.go @@ -0,0 +1,265 @@ +package k8s + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + osexec "os/exec" + "strings" + "time" + + "github.com/raphgm/container-preflight/internal/executor" + "github.com/raphgm/container-preflight/pkg/preflight/fact" + "github.com/raphgm/container-preflight/pkg/preflight/registry" +) + +// Cluster is what a specific cluster offers workloads. +type Cluster struct { + Context string + ServerVersion string + Nodes []Node + // Kinds maps "group/Kind" (core group: "/Kind") to true when served. + Kinds map[string]bool + // Versions lists served group/versions ("v1", "apps/v1", ...). + Versions map[string]bool + StorageClasses []string + DefaultClass string +} + +type Node struct { + Name string + Labels map[string]string + Taints []Taint + CPU int64 // allocatable millicores + Memory int64 // allocatable bytes + Extended map[string]int64 + Platform string // os/arch + Unschedulable bool + Ready bool +} + +type Taint struct{ Key, Value, Effect string } + +// commandTimeout bounds each kubectl call; an unreachable API server can +// otherwise hang for minutes. +const commandTimeout = 20 * time.Second + +type timeoutRunner struct{ executor.Runner } + +func (t timeoutRunner) Run(ctx context.Context, name string, args ...string) (string, error) { + cctx, cancel := context.WithTimeout(ctx, commandTimeout) + defer cancel() + out, err := t.Runner.Run(cctx, name, args...) + if err != nil && cctx.Err() == context.DeadlineExceeded { + return out, fmt.Errorf("`%s %s` did not answer within %s", name, strings.Join(args, " "), commandTimeout) + } + return out, err +} + +// Probe reads the cluster with get/list calls only. Anything it cannot read +// becomes a fact failure, so dependent rules report the root cause instead +// of failing individually. +func Probe(ctx context.Context, run executor.Runner, kubeContext string, facts *fact.Set) *Cluster { + run = timeoutRunner{run} + c := &Cluster{Kinds: map[string]bool{}, Versions: map[string]bool{}} + kc := func(args ...string) []string { + if kubeContext != "" { + return append([]string{"--context", kubeContext}, args...) + } + return args + } + + if _, err := run.Run(ctx, "kubectl", "version", "--client", "-o", "json"); err != nil { + facts.Fail(fact.Failure{Fact: fact.Kubectl, Severity: fact.Error, Summary: "kubectl not found", + Detail: errText(err), Fix: "Install kubectl and make sure it is on PATH."}) + return c + } + c.Context = kubeContext + if c.Context == "" { + c.Context, _ = run.Run(ctx, "kubectl", "config", "current-context") + } + + out, err := run.Run(ctx, "kubectl", kc("version", "-o", "json")...) + var ver struct { + ServerVersion struct{ GitVersion string } `json:"serverVersion"` + } + if err == nil { + _ = json.Unmarshal([]byte(out), &ver) + } + if ver.ServerVersion.GitVersion == "" { + summary := "Kubernetes API server is not reachable" + if c.Context == "" { + summary = "kubectl has no current context" + } else { + summary += " (context " + c.Context + ")" + } + facts.Fail(fact.Failure{Fact: fact.Cluster, Severity: fact.Error, Summary: summary, Detail: errText(err), + Fix: "Check `kubectl config get-contexts`, start the cluster (e.g. `kind create cluster`), or pass --context."}) + return c + } + c.ServerVersion = ver.ServerVersion.GitVersion + + if out, err := run.Run(ctx, "kubectl", kc("api-versions")...); err == nil { + for _, v := range strings.Fields(out) { + c.Versions[v] = true + } + } + if out, err := run.Run(ctx, "kubectl", kc("api-resources", "--no-headers")...); err != nil { + facts.Fail(fact.Failure{Fact: fact.ClusterAPIs, Severity: fact.Warning, Summary: "cannot list the cluster's API resources", Detail: errText(err)}) + } else { + for _, line := range strings.Split(out, "\n") { + f := strings.Fields(line) + if len(f) < 4 { + continue + } + // NAME [SHORTNAMES] APIVERSION NAMESPACED KIND + gv, kind := f[len(f)-3], f[len(f)-1] + c.Kinds[group(gv)+"/"+kind] = true + } + } + + if out, err := run.Run(ctx, "kubectl", kc("get", "nodes", "-o", "json")...); err != nil { + facts.Fail(fact.Failure{Fact: fact.Nodes, Severity: fact.Warning, Summary: "cannot list nodes", Detail: errText(err), + Fix: "Scheduling and platform checks need `get nodes` permission (cluster-scoped)."}) + } else if err := c.parseNodes(out); err != nil { + facts.Fail(fact.Failure{Fact: fact.Nodes, Severity: fact.Warning, Summary: "cannot read node list", Detail: err.Error()}) + } + + if out, err := run.Run(ctx, "kubectl", kc("get", "storageclasses", "-o", "json")...); err != nil { + facts.Fail(fact.Failure{Fact: fact.Storage, Severity: fact.Warning, Summary: "cannot list StorageClasses", Detail: errText(err)}) + } else { + var sc struct { + Items []struct { + Metadata struct { + Name string + Annotations map[string]string + } + } + } + if json.Unmarshal([]byte(out), &sc) == nil { + for _, it := range sc.Items { + c.StorageClasses = append(c.StorageClasses, it.Metadata.Name) + if it.Metadata.Annotations["storageclass.kubernetes.io/is-default-class"] == "true" { + c.DefaultClass = it.Metadata.Name + } + } + } + } + return c +} + +func (c *Cluster) parseNodes(out string) error { + var nl struct { + Items []struct { + Metadata struct { + Name string + Labels map[string]string + } + Spec struct { + Unschedulable bool + Taints []Taint + } + Status struct { + Allocatable map[string]string + NodeInfo struct { + Architecture string + OperatingSystem string + } + Conditions []struct{ Type, Status string } + } + } + } + if err := json.Unmarshal([]byte(out), &nl); err != nil { + return err + } + for _, it := range nl.Items { + n := Node{Name: it.Metadata.Name, Labels: it.Metadata.Labels, Taints: it.Spec.Taints, + Unschedulable: it.Spec.Unschedulable, Extended: map[string]int64{}} + n.CPU, _ = quantity(it.Status.Allocatable["cpu"], true) + n.Memory, _ = quantity(it.Status.Allocatable["memory"], false) + for k, v := range it.Status.Allocatable { + if strings.Contains(k, "/") { + n.Extended[k], _ = quantity(v, false) + } + } + n.Platform = registry.Normalize(it.Status.NodeInfo.OperatingSystem + "/" + it.Status.NodeInfo.Architecture) + for _, cond := range it.Status.Conditions { + if cond.Type == "Ready" { + n.Ready = cond.Status == "True" + } + } + c.Nodes = append(c.Nodes, n) + } + return nil +} + +// Platforms lists the distinct platforms of schedulable nodes. +func (c *Cluster) Platforms() []string { + seen := map[string]bool{} + var out []string + for _, n := range c.Nodes { + if n.Ready && !n.Unschedulable && !seen[n.Platform] { + seen[n.Platform] = true + out = append(out, n.Platform) + } + } + return out +} + +// Serves reports whether apiVersion/kind is available. +func (c *Cluster) Serves(apiVersion, kind string) (servedKind, servedVersion bool) { + return c.Kinds[group(apiVersion)+"/"+kind], c.Versions[apiVersion] +} + +func group(apiVersion string) string { + if g, _, ok := strings.Cut(apiVersion, "/"); ok { + return g + } + return "" +} + +// Snapshot is a portable cluster profile. +type Snapshot struct { + Version int `json:"version"` + CapturedAt time.Time `json:"capturedAt"` + Cluster *Cluster `json:"cluster"` + Failures []fact.Failure `json:"failures,omitempty"` +} + +func Capture(ctx context.Context, run executor.Runner, kubeContext string) *Snapshot { + facts := fact.NewSet() + c := Probe(ctx, run, kubeContext, facts) + return &Snapshot{Version: 1, CapturedAt: time.Now().UTC(), Cluster: c, Failures: facts.All()} +} + +func LoadSnapshot(path string) (*Snapshot, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var s Snapshot + if err := json.Unmarshal(b, &s); err != nil || s.Cluster == nil { + return nil, fmt.Errorf("%s: not a cluster snapshot", path) + } + return &s, nil +} + +func (s *Snapshot) Restore() (*Cluster, *fact.Set) { + facts := fact.NewSet() + for _, f := range s.Failures { + facts.Fail(f) + } + return s.Cluster, facts +} + +func errText(err error) string { + if err == nil { + return "" + } + if errors.Is(err, osexec.ErrNotFound) { + return "`kubectl` executable not found on PATH" + } + return strings.TrimPrefix(strings.TrimSpace(err.Error()), "exit status 1: ") +} diff --git a/pkg/preflight/k8s/k8s_test.go b/pkg/preflight/k8s/k8s_test.go new file mode 100644 index 0000000..d2c8234 --- /dev/null +++ b/pkg/preflight/k8s/k8s_test.go @@ -0,0 +1,284 @@ +package k8s + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/raphgm/container-preflight/pkg/preflight/fact" + "github.com/raphgm/container-preflight/pkg/preflight/registry" + "github.com/raphgm/container-preflight/pkg/preflight/rules" +) + +type fakeKubectl map[string]string + +func (f fakeKubectl) Run(_ context.Context, name string, args ...string) (string, error) { + key := strings.Join(args, " ") + for prefix, out := range f { + if strings.HasPrefix(key, prefix) { + if strings.HasPrefix(out, "ERR:") { + return "", errors.New(strings.TrimPrefix(out, "ERR:")) + } + return out, nil + } + } + return "", fmt.Errorf("unexpected: kubectl %s", key) +} + +const nodesJSON = `{"items":[ + {"metadata":{"name":"amd-1","labels":{"kubernetes.io/arch":"amd64","pool":"general"}}, + "spec":{}, + "status":{"allocatable":{"cpu":"4","memory":"16Gi"},"nodeInfo":{"architecture":"amd64","operatingSystem":"linux"},"conditions":[{"type":"Ready","status":"True"}]}}, + {"metadata":{"name":"arm-1","labels":{"kubernetes.io/arch":"arm64","pool":"general"}}, + "spec":{}, + "status":{"allocatable":{"cpu":"2","memory":"8Gi"},"nodeInfo":{"architecture":"arm64","operatingSystem":"linux"},"conditions":[{"type":"Ready","status":"True"}]}}, + {"metadata":{"name":"gpu-1","labels":{"kubernetes.io/arch":"amd64","pool":"gpu"}}, + "spec":{"taints":[{"key":"nvidia.com/gpu","value":"present","effect":"NoSchedule"}]}, + "status":{"allocatable":{"cpu":"8","memory":"32Gi","nvidia.com/gpu":"1"},"nodeInfo":{"architecture":"amd64","operatingSystem":"linux"},"conditions":[{"type":"Ready","status":"True"}]}} +]}` + +func liveCluster() fakeKubectl { + return fakeKubectl{ + "version --client": `{"clientVersion":{}}`, + "config current-context": "kind-mixed", + "version -o json": `{"serverVersion":{"gitVersion":"v1.31.0"}}`, + "api-versions": "v1\napps/v1\nbatch/v1\nnetworking.k8s.io/v1", + "api-resources": "pods po v1 true Pod\ndeployments deploy apps/v1 true Deployment\nstatefulsets sts apps/v1 true StatefulSet\njobs batch/v1 true Job\ningresses ing networking.k8s.io/v1 true Ingress\npersistentvolumeclaims pvc v1 true PersistentVolumeClaim\nservices svc v1 true Service", + "get nodes": nodesJSON, + "get storageclasses": `{"items":[{"metadata":{"name":"standard","annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}]}`, + } +} + +const manifests = `apiVersion: apps/v1 +kind: Deployment +metadata: {name: web} +spec: + replicas: 3 + template: + spec: + containers: + - name: web + image: acme/web-amd64:1 + resources: {requests: {cpu: 500m, memory: 256Mi}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: pinned} +spec: + template: + spec: + nodeSelector: {kubernetes.io/arch: arm64} + containers: + - {name: app, image: acme/web-amd64:1} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: trainer} +spec: + template: + spec: + containers: + - name: t + image: pytorch/pytorch:2 + resources: {limits: {nvidia.com/gpu: 1}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: huge} +spec: + template: + spec: + containers: + - {name: h, image: nginx:1.27, resources: {requests: {memory: 64Gi}}} +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: {name: old} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: {name: sm} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: {name: db} +spec: + template: + spec: + containers: [{name: db, image: postgres:16}] + volumeClaimTemplates: + - metadata: {name: data} + spec: {storageClassName: gp3} +--- +apiVersion: v1 +kind: Pod +metadata: {name: typo} +spec: + containers: [{name: x, image: nginx:9.99.99}] +` + +type fakeRegistry map[string][]string + +func (f fakeRegistry) Resolve(_ context.Context, ref, _ string) (*registry.Image, error) { + ps, ok := f[ref] + if !ok { + return nil, fmt.Errorf("%w: %s", registry.ErrNotFound, ref) + } + return ®istry.Image{Ref: ref, Index: len(ps) > 1, Platforms: ps}, nil +} + +func setup(t *testing.T) *Env { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "all.yaml"), []byte(manifests), 0o644); err != nil { + t.Fatal(err) + } + m, err := Load([]string{dir}) + if err != nil { + t.Fatal(err) + } + facts := fact.NewSet() + c := Probe(context.Background(), liveCluster(), "", facts) + if len(facts.Roots()) != 0 { + t.Fatalf("probe failures: %+v", facts.Roots()) + } + reg := fakeRegistry{ + "acme/web-amd64:1": {"linux/amd64"}, + "pytorch/pytorch:2": {"linux/amd64"}, + "nginx:1.27": {"linux/amd64", "linux/arm64"}, + "postgres:16": {"linux/amd64", "linux/arm64"}, + } + env := &Env{Manifests: m, Cluster: c, Facts: facts, Images: map[string]rules.ImageResult{}} + for _, w := range m.Workloads { + for _, ct := range allContainers(w.Pod) { + img, err := reg.Resolve(context.Background(), ct.Image, "") + env.Images[ct.Image] = rules.ImageResult{Image: img, Err: err} + } + } + return env +} + +func titles(fs []rules.Finding) string { + var out []string + for _, f := range fs { + out = append(out, string(f.Severity)+" "+f.Service+": "+f.Title) + } + return strings.Join(out, "\n") +} + +func expect(t *testing.T, fs []rules.Finding, sev fact.Severity, substr string) rules.Finding { + t.Helper() + for _, f := range fs { + if f.Severity == sev && strings.Contains(f.Service+": "+f.Title, substr) { + return f + } + } + t.Errorf("want %s containing %q in:\n%s", sev, substr, titles(fs)) + return rules.Finding{} +} + +func TestLoadManifests(t *testing.T) { + env := setup(t) + if len(env.Manifests.Objects) != 8 || len(env.Manifests.Workloads) != 6 { + t.Fatalf("objects %d workloads %d", len(env.Manifests.Objects), len(env.Manifests.Workloads)) + } + w := env.Manifests.Workloads[2] + if _, _, ext := w.Pod.Requests(); ext["nvidia.com/gpu"] != 1 { + t.Errorf("GPU limit should imply a request: %+v", ext) + } + if env.Cluster.DefaultClass != "standard" || len(env.Cluster.Nodes) != 3 { + t.Errorf("cluster = %+v", env.Cluster) + } +} + +func TestAPIRule(t *testing.T) { + fs := apiRule.Check(context.Background(), setup(t)) + f := expect(t, fs, fact.Error, "Ingress/old: extensions/v1beta1 Ingress is not available") + if f.Predicts != `no matches for kind "Ingress" in version "extensions/v1beta1"` { + t.Errorf("predicts %q", f.Predicts) + } + expect(t, fs, fact.Error, "ServiceMonitor/sm") + if len(fs) != 2 { + t.Errorf("only two unserved objects:\n%s", titles(fs)) + } +} + +func TestImageRule(t *testing.T) { + fs := imageRule.Check(context.Background(), setup(t)) + expect(t, fs, fact.Error, "Pod/typo: image nginx:9.99.99 does not exist") + if len(fs) != 1 { + t.Errorf("\n%s", titles(fs)) + } +} + +func TestPlatformRule(t *testing.T) { + fs := platformRule.Check(context.Background(), setup(t)) + f := expect(t, fs, fact.Warning, "Deployment/web: image acme/web-amd64:1 fails on 1 of 2 eligible nodes") + if !strings.Contains(strings.Join(f.Evidence, " "), "arm-1 (linux/arm64)") { + t.Errorf("evidence should name the arm node: %v", f.Evidence) + } + expect(t, fs, fact.Error, "Deployment/pinned: image acme/web-amd64:1 cannot run on any node") + if strings.Contains(titles(fs), "trainer") || strings.Contains(titles(fs), "huge") { + t.Errorf("GPU pod is unschedulable (reported elsewhere); multi-arch nginx fits:\n%s", titles(fs)) + } +} + +func TestSchedulableRule(t *testing.T) { + fs := schedulableRule.Check(context.Background(), setup(t)) + gpu := expect(t, fs, fact.Error, "Deployment/trainer: no node can run this pod") + want := "FailedScheduling: 0/3 nodes are available: 1 node(s) had untolerated taint {nvidia.com/gpu: present}, 2 Insufficient nvidia.com/gpu." + if gpu.Predicts != want { + t.Errorf("predicts\n %q\nwant\n %q", gpu.Predicts, want) + } + expect(t, fs, fact.Error, "Deployment/huge") + if len(fs) != 2 { + t.Errorf("\n%s", titles(fs)) + } +} + +func TestStorageRule(t *testing.T) { + env := setup(t) + fs := storageRule.Check(context.Background(), env) + f := expect(t, fs, fact.Error, `StatefulSet/db: claim data uses StorageClass "gp3", which does not exist`) + if !strings.Contains(f.Predicts, `storageclass.storage.k8s.io "gp3" not found`) { + t.Errorf("predicts %q", f.Predicts) + } + env.Cluster.DefaultClass = "" + env.Manifests.Claims = append(env.Manifests.Claims, Claim{Name: "logs"}) + expect(t, storageRule.Check(context.Background(), env), fact.Error, "claim logs sets no StorageClass and the cluster has no default") +} + +func TestUnreachableClusterIsOneRootCause(t *testing.T) { + facts := fact.NewSet() + Probe(context.Background(), fakeKubectl{ + "version --client": `{}`, + "config current-context": "prod", + "version -o json": "ERR:Unable to connect to the server: dial tcp 10.0.0.1:6443: i/o timeout", + }, "", facts) + roots := facts.Roots() + if len(roots) != 1 || roots[0].Fact != fact.Cluster || !strings.Contains(roots[0].Summary, "context prod") { + t.Fatalf("roots = %+v", roots) + } + for _, id := range []fact.ID{fact.Nodes, fact.ClusterAPIs, fact.Storage} { + if r, _ := facts.Root(id); r.Fact != fact.Cluster { + t.Errorf("%s should be blocked by the cluster", id) + } + } +} + +func TestQuantity(t *testing.T) { + cases := []struct { + in string + cpu bool + want int64 + }{{"500m", true, 500}, {"2", true, 2000}, {"0.25", true, 250}, {"1Gi", false, 1 << 30}, {"128Mi", false, 128 << 20}, {"1G", false, 1e9}, {"1e3", false, 1000}} + for _, c := range cases { + if got, err := quantity(c.in, c.cpu); err != nil || got != c.want { + t.Errorf("quantity(%q) = %d, %v; want %d", c.in, got, err, c.want) + } + } +} diff --git a/pkg/preflight/k8s/manifest.go b/pkg/preflight/k8s/manifest.go new file mode 100644 index 0000000..03c8661 --- /dev/null +++ b/pkg/preflight/k8s/manifest.go @@ -0,0 +1,314 @@ +// Package k8s predicts Kubernetes deployment failures before `kubectl apply` +// by joining what manifests require with what a specific cluster offers and +// what registries publish. +package k8s + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// Object is any manifest document, kept for API checks. +type Object struct { + APIVersion string + Kind string + Name string + Namespace string + Location string +} + +// Workload is an object that creates pods. +type Workload struct { + Object + Replicas int + Pod PodSpec + Claims []Claim // volumeClaimTemplates +} + +type PodSpec struct { + NodeSelector map[string]string + Required []NodeSelectorTerm // requiredDuringScheduling node affinity (ORed) + Tolerations []Toleration + Containers []Container // app containers + InitContainers []Container + ImagePullSecrets []string + ServiceAccount string +} + +type Container struct { + Name string + Image string + CPU int64 // requested millicores + Memory int64 // requested bytes + GPU int64 // nvidia.com/gpu + Extended map[string]int64 +} + +type NodeSelectorTerm struct { + Expressions []Requirement +} + +type Requirement struct { + Key string + Operator string + Values []string +} + +type Toleration struct { + Key, Operator, Value, Effect string +} + +type Claim struct { + Name string + StorageClass *string // nil: not set, "" explicitly empty + Location string +} + +// Manifests is everything read from the given files. +type Manifests struct { + Objects []Object + Workloads []Workload + Claims []Claim // standalone PersistentVolumeClaims +} + +// Load reads YAML manifests from files, directories (recursively, *.yaml and +// *.yml) or "-" for stdin, such as `helm template` or `kustomize build` +// output. +func Load(paths []string) (*Manifests, error) { + m := &Manifests{} + for _, p := range paths { + if p == "-" { + b, err := io.ReadAll(os.Stdin) + if err != nil { + return nil, err + } + if err := m.parse("stdin", b); err != nil { + return nil, err + } + continue + } + st, err := os.Stat(p) + if err != nil { + return nil, err + } + var files []string + if st.IsDir() { + err = filepath.WalkDir(p, func(f string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() && (strings.HasSuffix(f, ".yaml") || strings.HasSuffix(f, ".yml")) { + files = append(files, f) + } + return err + }) + if err != nil { + return nil, err + } + sort.Strings(files) + } else { + files = []string{p} + } + for _, f := range files { + b, err := os.ReadFile(f) + if err != nil { + return nil, err + } + if err := m.parse(f, b); err != nil { + return nil, err + } + } + } + if len(m.Objects) == 0 { + return nil, errors.New("no Kubernetes objects found in the given manifests") + } + return m, nil +} + +func (m *Manifests) parse(file string, b []byte) error { + dec := yaml.NewDecoder(bytes.NewReader(b)) + for { + var n yaml.Node + err := dec.Decode(&n) + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return fmt.Errorf("%s: %w", file, err) + } + if len(n.Content) == 0 { + continue + } + var doc map[string]any + if err := n.Decode(&doc); err != nil || doc == nil { + continue + } + m.add(doc, fmt.Sprintf("%s:%d", filepath.Base(file), n.Content[0].Line)) + } +} + +func (m *Manifests) add(doc map[string]any, loc string) { + kind := str(doc["kind"]) + if kind == "" { + return + } + if strings.HasSuffix(kind, "List") { + for _, it := range list(doc["items"]) { + if d, ok := it.(map[string]any); ok { + m.add(d, loc) + } + } + return + } + meta := mp(doc["metadata"]) + obj := Object{APIVersion: str(doc["apiVersion"]), Kind: kind, Name: str(meta["name"]), Namespace: str(meta["namespace"]), Location: loc} + m.Objects = append(m.Objects, obj) + + spec := mp(doc["spec"]) + var tmpl map[string]any + replicas := 1 + switch kind { + case "Pod": + tmpl = map[string]any{"spec": spec} + case "Deployment", "ReplicaSet", "StatefulSet", "DaemonSet", "ReplicationController": + tmpl = mp(spec["template"]) + if r, ok := spec["replicas"].(int); ok { + replicas = r + } + case "Job": + tmpl = mp(spec["template"]) + case "CronJob": + tmpl = mp(mp(mp(spec["jobTemplate"])["spec"])["template"]) + case "PersistentVolumeClaim": + m.Claims = append(m.Claims, claim(obj.Name, spec, loc)) + return + default: + return + } + w := Workload{Object: obj, Replicas: replicas, Pod: podSpec(mp(tmpl["spec"]))} + for _, c := range list(spec["volumeClaimTemplates"]) { + cm := mp(c) + w.Claims = append(w.Claims, claim(str(mp(cm["metadata"])["name"]), mp(cm["spec"]), loc)) + } + m.Workloads = append(m.Workloads, w) +} + +func claim(name string, spec map[string]any, loc string) Claim { + c := Claim{Name: name, Location: loc} + if v, ok := spec["storageClassName"]; ok { + s := str(v) + c.StorageClass = &s + } + return c +} + +func podSpec(s map[string]any) PodSpec { + p := PodSpec{NodeSelector: map[string]string{}, ServiceAccount: str(s["serviceAccountName"])} + for k, v := range mp(s["nodeSelector"]) { + p.NodeSelector[k] = str(v) + } + req := mp(mp(mp(s["affinity"])["nodeAffinity"])["requiredDuringSchedulingIgnoredDuringExecution"]) + for _, t := range list(req["nodeSelectorTerms"]) { + var term NodeSelectorTerm + for _, e := range list(mp(t)["matchExpressions"]) { + em := mp(e) + r := Requirement{Key: str(em["key"]), Operator: str(em["operator"])} + for _, v := range list(em["values"]) { + r.Values = append(r.Values, str(v)) + } + term.Expressions = append(term.Expressions, r) + } + p.Required = append(p.Required, term) + } + for _, t := range list(s["tolerations"]) { + tm := mp(t) + p.Tolerations = append(p.Tolerations, Toleration{Key: str(tm["key"]), Operator: str(tm["operator"]), Value: str(tm["value"]), Effect: str(tm["effect"])}) + } + for _, sec := range list(s["imagePullSecrets"]) { + p.ImagePullSecrets = append(p.ImagePullSecrets, str(mp(sec)["name"])) + } + p.Containers = containers(list(s["containers"])) + p.InitContainers = containers(list(s["initContainers"])) + return p +} + +func containers(items []any) []Container { + var out []Container + for _, c := range items { + cm := mp(c) + ct := Container{Name: str(cm["name"]), Image: str(cm["image"]), Extended: map[string]int64{}} + res := mp(cm["resources"]) + req, lim := mp(res["requests"]), mp(res["limits"]) + // A limit without a request implies an equal request. + get := func(k string) string { + if v, ok := req[k]; ok { + return str(v) + } + return str(lim[k]) + } + ct.CPU, _ = quantity(get("cpu"), true) + ct.Memory, _ = quantity(get("memory"), false) + for _, src := range []map[string]any{lim, req} { + for k, v := range src { + if strings.Contains(k, "/") { + n, _ := quantity(str(v), false) + ct.Extended[k] = n + } + } + } + ct.GPU = ct.Extended["nvidia.com/gpu"] + out = append(out, ct) + } + return out +} + +// Requests are what the scheduler reserves: the larger of the sum of app +// containers and the largest init container. +func (p PodSpec) Requests() (cpu, mem int64, extended map[string]int64) { + extended = map[string]int64{} + for _, c := range p.Containers { + cpu += c.CPU + mem += c.Memory + for k, v := range c.Extended { + extended[k] += v + } + } + for _, c := range p.InitContainers { + cpu = max(cpu, c.CPU) + mem = max(mem, c.Memory) + for k, v := range c.Extended { + extended[k] = max(extended[k], v) + } + } + return cpu, mem, extended +} + +func str(v any) string { + switch t := v.(type) { + case nil: + return "" + case string: + return t + default: + return fmt.Sprint(t) + } +} + +func mp(v any) map[string]any { + if m, ok := v.(map[string]any); ok { + return m + } + return map[string]any{} +} + +func list(v any) []any { + if l, ok := v.([]any); ok { + return l + } + return nil +} diff --git a/pkg/preflight/k8s/quantity.go b/pkg/preflight/k8s/quantity.go new file mode 100644 index 0000000..7333d56 --- /dev/null +++ b/pkg/preflight/k8s/quantity.go @@ -0,0 +1,62 @@ +package k8s + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +// quantity parses a Kubernetes resource quantity into a base value: CPU in +// millicores, everything else in bytes (or plain units). It covers the +// suffixes manifests use; exponent notation is accepted as a plain float. +func quantity(s string, cpu bool) (int64, error) { + s = strings.TrimSpace(s) + if s == "" { + return 0, nil + } + mult := 1.0 + suffixes := []struct { + s string + m float64 + }{ + {"Ki", 1 << 10}, {"Mi", 1 << 20}, {"Gi", 1 << 30}, {"Ti", 1 << 40}, {"Pi", 1 << 50}, {"Ei", 1 << 60}, + {"k", 1e3}, {"M", 1e6}, {"G", 1e9}, {"T", 1e12}, {"P", 1e15}, {"E", 1e18}, + {"m", 1e-3}, {"u", 1e-6}, {"n", 1e-9}, + } + for _, suf := range suffixes { + if strings.HasSuffix(s, suf.s) { + mult, s = suf.m, strings.TrimSuffix(s, suf.s) + break + } + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0, fmt.Errorf("invalid quantity %q", s) + } + v *= mult + if cpu { + v *= 1000 // cores -> millicores + } + return int64(math.Ceil(v - 1e-9)), nil +} + +func formatCPU(milli int64) string { + if milli%1000 == 0 { + return strconv.FormatInt(milli/1000, 10) + } + return strconv.FormatInt(milli, 10) + "m" +} + +func formatBytes(b int64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := int64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp]) +} diff --git a/pkg/preflight/k8s/rules.go b/pkg/preflight/k8s/rules.go new file mode 100644 index 0000000..7c07df0 --- /dev/null +++ b/pkg/preflight/k8s/rules.go @@ -0,0 +1,397 @@ +package k8s + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/raphgm/container-preflight/pkg/preflight/fact" + "github.com/raphgm/container-preflight/pkg/preflight/registry" + "github.com/raphgm/container-preflight/pkg/preflight/rules" +) + +// Env is everything a Kubernetes rule may read. +type Env struct { + Manifests *Manifests + Cluster *Cluster + Facts *fact.Set + // Images maps an image reference to its registry lookup. + Images map[string]rules.ImageResult +} + +type Rule struct { + ID string + Needs []fact.ID + Check func(ctx context.Context, env *Env) []rules.Finding +} + +func All() []Rule { + return []Rule{apiRule, imageRule, platformRule, schedulableRule, storageRule} +} + +var apiRule = Rule{ + ID: "k8s.api", + Needs: []fact.ID{fact.ClusterAPIs}, + Check: func(_ context.Context, env *Env) []rules.Finding { + var out []rules.Finding + seen := map[string]bool{} + for _, o := range env.Manifests.Objects { + key := o.APIVersion + " " + o.Kind + if seen[key] { + continue + } + kind, version := env.Cluster.Serves(o.APIVersion, o.Kind) + if kind && version { + continue + } + seen[key] = true + why := fmt.Sprintf("cluster: %s/%s is not served", o.APIVersion, o.Kind) + fix := "Install the CRD or operator that provides " + o.Kind + " before applying." + if kind { + why = fmt.Sprintf("cluster: %s is served, but not in version %s", o.Kind, o.APIVersion) + fix = "Use an API version this cluster serves (`kubectl api-resources | grep " + o.Kind + "`)." + } + out = append(out, rules.Finding{ + Rule: "k8s.api", Severity: fact.Error, Service: o.Kind + "/" + o.Name, + Title: fmt.Sprintf("%s %s is not available on this cluster", o.APIVersion, o.Kind), + Evidence: []string{why, "cluster: Kubernetes " + env.Cluster.ServerVersion}, + Fix: fix, Location: o.Location, + Predicts: fmt.Sprintf(`no matches for kind "%s" in version "%s"`, o.Kind, o.APIVersion), + }) + } + return out + }, +} + +var imageRule = Rule{ + ID: "k8s.image", + Needs: []fact.ID{fact.Registry}, + Check: func(_ context.Context, env *Env) []rules.Finding { + var out []rules.Finding + for _, w := range env.Manifests.Workloads { + for _, c := range allContainers(w.Pod) { + res, ok := env.Images[c.Image] + if !ok || res.Err == nil { + continue + } + f := rules.Finding{Rule: "k8s.image", Service: w.Kind + "/" + w.Name, Location: w.Location} + switch { + case errors.Is(res.Err, registry.ErrNotFound): + f.Severity, f.Title = fact.Error, fmt.Sprintf("image %s does not exist", c.Image) + f.Evidence = []string{"registry: " + firstLine(res.Err)} + f.Fix = "Fix the tag; check it on the registry's web page." + f.Predicts = "ErrImagePull: manifest unknown / not found" + case errors.Is(res.Err, registry.ErrUnauthorized) && len(w.Pod.ImagePullSecrets) == 0: + f.Severity, f.Title = fact.Warning, fmt.Sprintf("image %s is private or missing, and the pod has no imagePullSecrets", c.Image) + f.Evidence = []string{"registry: " + firstLine(res.Err), "project: no imagePullSecrets on the pod (the ServiceAccount's are not checked)"} + f.Fix = "Add an imagePullSecret with registry credentials, or check the image name." + f.Predicts = "ImagePullBackOff: pull access denied" + default: + continue + } + out = append(out, f) + } + } + return out + }, +} + +var platformRule = Rule{ + ID: "k8s.platform", + Needs: []fact.ID{fact.Nodes, fact.Registry}, + Check: func(_ context.Context, env *Env) []rules.Finding { + var out []rules.Finding + for _, w := range env.Manifests.Workloads { + // Nodes the pod may land on: placement rules, plus the extended + // resources (GPUs) it requests. CPU and memory are left to + // k8s.schedulable because they depend on what else runs there. + _, _, ext := w.Pod.Requests() + var eligible []Node + for _, n := range env.Cluster.Nodes { + if len(placementReasons(w.Pod, n)) > 0 { + continue + } + has := true + for k, v := range ext { + if n.Extended[k] < v { + has = false + } + } + if has { + eligible = append(eligible, n) + } + } + if len(eligible) == 0 { + continue // k8s.schedulable reports it + } + for _, c := range allContainers(w.Pod) { + res, ok := env.Images[c.Image] + if !ok || res.Image == nil { + continue + } + var bad, good []string + for _, n := range eligible { + if supports(res.Image, n.Platform) { + good = append(good, n.Name) + } else { + bad = append(bad, n.Name+" ("+n.Platform+")") + } + } + if len(bad) == 0 { + continue + } + f := rules.Finding{ + Rule: "k8s.platform", Service: w.Kind + "/" + w.Name, Location: w.Location, + Evidence: []string{ + "registry: " + c.Image + " ships " + strings.Join(res.Image.Platforms, ", "), + "cluster: eligible nodes without a matching platform: " + strings.Join(bad, ", "), + }, + } + predicts := "exec format error / CrashLoopBackOff" + if res.Image.Index { + predicts = "ErrImagePull: no matching manifest for the node's platform" + } + f.Predicts = predicts + if len(good) == 0 { + f.Severity = fact.Error + f.Title = fmt.Sprintf("image %s cannot run on any node this pod can be scheduled to", c.Image) + f.Fix = "Build a multi-arch image (docker buildx build --platform …) or add nodes of a supported architecture." + } else { + f.Severity = fact.Warning + f.Title = fmt.Sprintf("image %s fails on %d of %d eligible nodes (mixed architectures)", c.Image, len(bad), len(eligible)) + f.Fix = fmt.Sprintf("Pin the pod with nodeSelector kubernetes.io/arch, or publish a multi-arch image. Nodes that work: %s.", strings.Join(good, ", ")) + } + out = append(out, f) + } + } + return out + }, +} + +var schedulableRule = Rule{ + ID: "k8s.schedulable", + Needs: []fact.ID{fact.Nodes}, + Check: func(_ context.Context, env *Env) []rules.Finding { + var out []rules.Finding + nodes := env.Cluster.Nodes + for _, w := range env.Manifests.Workloads { + if w.Kind == "DaemonSet" { + continue // zero matching nodes means zero pods, not a failure + } + cpu, mem, ext := w.Pod.Requests() + counts := map[string]int{} + fits := 0 + for _, n := range nodes { + reasons := placementReasons(w.Pod, n) + if n.CPU > 0 && cpu > n.CPU { + reasons = append(reasons, "Insufficient cpu") + } + if n.Memory > 0 && mem > n.Memory { + reasons = append(reasons, "Insufficient memory") + } + for k, v := range ext { + if v > n.Extended[k] { + reasons = append(reasons, "Insufficient "+k) + } + } + if len(reasons) == 0 { + fits++ + } + for _, r := range dedupe(reasons) { + counts[r]++ + } + } + if fits > 0 || len(nodes) == 0 { + continue + } + var parts []string + for r, n := range counts { + parts = append(parts, fmt.Sprintf("%d %s", n, r)) + } + sort.Strings(parts) + msg := fmt.Sprintf("0/%d nodes are available: %s.", len(nodes), strings.Join(parts, ", ")) + out = append(out, rules.Finding{ + Rule: "k8s.schedulable", Severity: fact.Error, Service: w.Kind + "/" + w.Name, Location: w.Location, + Title: "no node can run this pod", + Evidence: []string{ + fmt.Sprintf("project: requests cpu %s, memory %s%s", formatCPU(cpu), formatBytes(mem), extendedText(ext)), + "cluster: " + msg, + "capacity is compared with node allocatable; pods already running are not counted", + }, + Fix: "Relax the selector/affinity, add a toleration, lower the requests, or add a matching node.", + Predicts: "FailedScheduling: " + msg, + }) + } + return out + }, +} + +var storageRule = Rule{ + ID: "k8s.storage", + Needs: []fact.ID{fact.Storage}, + Check: func(_ context.Context, env *Env) []rules.Finding { + c := env.Cluster + known := map[string]bool{} + for _, s := range c.StorageClasses { + known[s] = true + } + var out []rules.Finding + check := func(owner string, cl Claim) { + f := rules.Finding{Rule: "k8s.storage", Severity: fact.Error, Service: owner, Location: cl.Location} + switch { + case cl.StorageClass == nil && c.DefaultClass == "": + f.Title = fmt.Sprintf("claim %s sets no StorageClass and the cluster has no default", cl.Name) + f.Evidence = []string{"cluster: StorageClasses " + listOrNone(c.StorageClasses) + ", none marked default"} + f.Fix = "Set storageClassName, or mark a StorageClass as default." + f.Predicts = "PersistentVolumeClaim Pending: no persistent volumes available for this claim and no storage class is set" + case cl.StorageClass != nil && *cl.StorageClass != "" && !known[*cl.StorageClass]: + f.Title = fmt.Sprintf("claim %s uses StorageClass %q, which does not exist", cl.Name, *cl.StorageClass) + f.Evidence = []string{"cluster: StorageClasses " + listOrNone(c.StorageClasses)} + f.Fix = "Use one of the cluster's StorageClasses, or install the provisioner." + f.Predicts = fmt.Sprintf(`PersistentVolumeClaim Pending: storageclass.storage.k8s.io "%s" not found`, *cl.StorageClass) + default: + return + } + out = append(out, f) + } + for _, cl := range env.Manifests.Claims { + check("PersistentVolumeClaim/"+cl.Name, cl) + } + for _, w := range env.Manifests.Workloads { + for _, cl := range w.Claims { + check(w.Kind+"/"+w.Name, cl) + } + } + return out + }, +} + +// placementReasons lists why the scheduler would skip node n for this pod, +// ignoring resources, in the scheduler's own wording. +func placementReasons(p PodSpec, n Node) []string { + var r []string + if !n.Ready { + r = append(r, "node(s) were not ready") + } + if n.Unschedulable { + r = append(r, "node(s) were unschedulable") + } + if !matchesSelector(p, n) { + r = append(r, "node(s) didn't match Pod's node affinity/selector") + } + for _, t := range n.Taints { + if (t.Effect == "NoSchedule" || t.Effect == "NoExecute") && !tolerates(p.Tolerations, t) { + r = append(r, fmt.Sprintf("node(s) had untolerated taint {%s: %s}", t.Key, t.Value)) + } + } + return r +} + +func matchesSelector(p PodSpec, n Node) bool { + for k, v := range p.NodeSelector { + if n.Labels[k] != v { + return false + } + } + if len(p.Required) == 0 { + return true + } + for _, term := range p.Required { // terms are ORed + ok := true + for _, e := range term.Expressions { // expressions are ANDed + v, has := n.Labels[e.Key] + switch e.Operator { + case "In": + ok = ok && has && contains(e.Values, v) + case "NotIn": + ok = ok && !(has && contains(e.Values, v)) + case "Exists": + ok = ok && has + case "DoesNotExist": + ok = ok && !has + } + } + if ok { + return true + } + } + return false +} + +func tolerates(ts []Toleration, t Taint) bool { + for _, tol := range ts { + if tol.Effect != "" && tol.Effect != t.Effect { + continue + } + switch { + case tol.Operator == "Exists" && (tol.Key == "" || tol.Key == t.Key): + return true + case (tol.Operator == "" || tol.Operator == "Equal") && tol.Key == t.Key && tol.Value == t.Value: + return true + } + } + return false +} + +func supports(img *registry.Image, platform string) bool { + for _, p := range img.Platforms { + if registry.Compatible(p, platform) { + return true + } + } + return false +} + +func allContainers(p PodSpec) []Container { + return append(append([]Container{}, p.InitContainers...), p.Containers...) +} + +func extendedText(ext map[string]int64) string { + var parts []string + for k, v := range ext { + parts = append(parts, fmt.Sprintf(", %s %d", k, v)) + } + sort.Strings(parts) + return strings.Join(parts, "") +} + +func listOrNone(xs []string) string { + if len(xs) == 0 { + return "(none)" + } + return strings.Join(xs, ", ") +} + +func dedupe(xs []string) []string { + seen := map[string]bool{} + var out []string + for _, x := range xs { + if !seen[x] { + seen[x] = true + out = append(out, x) + } + } + return out +} + +func contains(xs []string, s string) bool { + for _, x := range xs { + if x == s { + return true + } + } + return false +} + +func firstLine(err error) string { + s := err.Error() + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + if len(s) > 200 { + s = s[:200] + "…" + } + return s +} diff --git a/pkg/preflight/kubernetes.go b/pkg/preflight/kubernetes.go new file mode 100644 index 0000000..ee58699 --- /dev/null +++ b/pkg/preflight/kubernetes.go @@ -0,0 +1,90 @@ +package preflight + +import ( + "context" + "sort" + "time" + + "github.com/raphgm/container-preflight/internal/executor" + "github.com/raphgm/container-preflight/pkg/preflight/fact" + "github.com/raphgm/container-preflight/pkg/preflight/k8s" + "github.com/raphgm/container-preflight/pkg/preflight/registry" + "github.com/raphgm/container-preflight/pkg/preflight/rules" +) + +type K8sOptions struct { + Paths []string + Context string + Offline bool + Snapshot *k8s.Snapshot + Runner executor.Runner + Resolver registry.Resolver +} + +// RunK8s predicts what applying the manifests to a cluster would fail on. +func RunK8s(ctx context.Context, opts K8sOptions) (*Report, error) { + start := time.Now() + if opts.Runner == nil { + opts.Runner = executor.New() + } + if opts.Resolver == nil { + opts.Resolver = registry.NewRemote() + } + m, err := k8s.Load(opts.Paths) + if err != nil { + return nil, err + } + + var cluster *k8s.Cluster + var facts *fact.Set + if opts.Snapshot != nil { + cluster, facts = opts.Snapshot.Restore() + } else { + facts = fact.NewSet() + cluster = k8s.Probe(ctx, opts.Runner, opts.Context, facts) + } + + env := &k8s.Env{Manifests: m, Cluster: cluster, Facts: facts, Images: map[string]rules.ImageResult{}} + if opts.Offline { + facts.Fail(fact.Failure{Fact: fact.Registry, Severity: fact.Info, Summary: "Registry checks skipped (--offline)"}) + } else { + platform := "linux/amd64" + if ps := cluster.Platforms(); len(ps) > 0 { + platform = ps[0] + } + var pulls []rules.Pull + seen := map[string]bool{} + for _, w := range m.Workloads { + for _, c := range append(append([]k8s.Container{}, w.Pod.InitContainers...), w.Pod.Containers...) { + if c.Image != "" && !seen[c.Image] { + seen[c.Image] = true + pulls = append(pulls, rules.Pull{Ref: c.Image, Platform: platform}) + } + } + } + byKey := resolveImages(ctx, opts.Resolver, pulls, facts) + for _, p := range pulls { + env.Images[p.Ref] = byKey[rules.ImageKey(p.Ref, p.Platform)] + } + } + + rep := &Report{Project: "kubernetes manifests", Cluster: cluster, Timestamp: start} + blocked := map[fact.ID][]string{} + for _, r := range k8s.All() { + if root, ok := blockingRoot(facts, r.Needs); ok { + blocked[root.Fact] = append(blocked[root.Fact], r.ID) + continue + } + found := r.Check(ctx, env) + if len(found) == 0 { + rep.Passed = append(rep.Passed, r.ID) + } + rep.Findings = append(rep.Findings, found...) + } + for _, root := range facts.Roots() { + rep.Roots = append(rep.Roots, RootCause{Failure: root, Blocks: blocked[root.Fact]}) + } + sort.SliceStable(rep.Findings, func(i, j int) bool { return rank(rep.Findings[i].Severity) < rank(rep.Findings[j].Severity) }) + rep.Duration = time.Since(start) + return rep, nil +} diff --git a/pkg/preflight/preflight.go b/pkg/preflight/preflight.go index c3d2fa7..2639ca4 100644 --- a/pkg/preflight/preflight.go +++ b/pkg/preflight/preflight.go @@ -14,6 +14,7 @@ import ( "github.com/raphgm/container-preflight/internal/executor" "github.com/raphgm/container-preflight/pkg/preflight/fact" "github.com/raphgm/container-preflight/pkg/preflight/host" + "github.com/raphgm/container-preflight/pkg/preflight/k8s" "github.com/raphgm/container-preflight/pkg/preflight/learn" "github.com/raphgm/container-preflight/pkg/preflight/project" "github.com/raphgm/container-preflight/pkg/preflight/registry" @@ -48,7 +49,8 @@ type Report struct { Project string `json:"project"` Dir string `json:"dir"` HostName string `json:"hostName,omitempty"` - Host *host.Profile `json:"host"` + Host *host.Profile `json:"host,omitempty"` + Cluster *k8s.Cluster `json:"cluster,omitempty"` Roots []RootCause `json:"rootCauses"` Findings []rules.Finding `json:"findings"` Passed []string `json:"passed"` From 9d507a219ae4ad3652c58aa53994bd5ac43fb44d Mon Sep 17 00:00:00 2001 From: raphgm Date: Thu, 24 Sep 2026 15:03:53 +0100 Subject: [PATCH 2/4] feat(k8s): validate on minikube; emulation-aware platform rule; served apiVersion hint Applying the example manifests to minikube confirmed 4 of 5 predictions (3 verbatim). The miss: an amd64-only image ran on the arm64 node through Docker Desktop's emulation. Local clusters now get a warning instead of an error. The API rule names the served apiVersion for a kind that moved groups. --- docs/validation/2026-09-24-minikube-k8s.md | 27 +++++++++++ examples/k8s/app.yaml | 56 ++++++++++++++++++++++ pkg/preflight/k8s/cluster.go | 29 ++++++++++- pkg/preflight/k8s/k8s_test.go | 9 +++- pkg/preflight/k8s/rules.go | 12 ++++- 5 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 docs/validation/2026-09-24-minikube-k8s.md create mode 100644 examples/k8s/app.yaml diff --git a/docs/validation/2026-09-24-minikube-k8s.md b/docs/validation/2026-09-24-minikube-k8s.md new file mode 100644 index 0000000..911c800 --- /dev/null +++ b/docs/validation/2026-09-24-minikube-k8s.md @@ -0,0 +1,27 @@ +# Validation run: Kubernetes rules on minikube + +Date: 2026-09-24 +Cluster: minikube (Docker driver on Docker Desktop, Apple silicon), Kubernetes v1.37.0, one +`linux/arm64` node, default StorageClass `standard`. +Manifests: `examples/k8s/app.yaml`. + +`container-preflight k8s examples/k8s` was run first. The manifests were then applied to a +throw-away namespace, and the events Kubernetes reported were compared with the predictions. + +| Object | Prediction | Real outcome | Verdict | +|---|---|---|---| +| `Ingress/old-ingress` | `no matches for kind "Ingress" in version "extensions/v1beta1"` | `kubectl apply`: identical | ✅ exact | +| `Pod/typo` (`nginx:9.99.99`) | `ErrImagePull` | `ErrImagePull`, then `ImagePullBackOff` (`failed to resolve reference … not found`) | ✅ | +| `Deployment/huge` (64 GiB request) | `FailedScheduling: 0/1 nodes are available: 1 Insufficient memory.` | identical, followed by a preemption note | ✅ exact | +| `StatefulSet/db` (StorageClass `gp3`) | PVC Pending: `storageclass.storage.k8s.io "gp3" not found` | `ProvisioningFailed` with that exact text; PVC `Pending`; pod `FailedScheduling: … unbound immediate PersistentVolumeClaims` | ✅ exact | +| `Deployment/web` (`pause-amd64`, amd64 only) | first version: cannot run on any node → `exec format error` | pods `Running` | ❌ false positive | + +The false positive has the same cause as the Colima case. minikube's node runs inside Docker +Desktop's VM, which runs amd64 binaries under Rosetta, so the node inherits emulation that the +cluster API does not report. Local clusters (minikube, kind, Docker Desktop, k3d, Rancher +Desktop, OrbStack, Colima) now get a warning: "will only run under emulation". Other clusters keep +the error, because production arm64 nodes normally have no emulation. The Ingress fix also names +the served version (`networking.k8s.io/v1`) instead of suggesting a CRD. + +Result: 4 of 5 predictions confirmed, 3 word for word. After the fix the tool's output matches +what the cluster did. diff --git a/examples/k8s/app.yaml b/examples/k8s/app.yaml new file mode 100644 index 0000000..f2387d4 --- /dev/null +++ b/examples/k8s/app.yaml @@ -0,0 +1,56 @@ +# Deliberately broken manifests for trying `container-preflight k8s`. +apiVersion: apps/v1 +kind: Deployment +metadata: {name: web} +spec: + replicas: 2 + selector: {matchLabels: {app: web}} + template: + metadata: {labels: {app: web}} + spec: + containers: + - name: web + image: gcr.io/google-containers/pause-amd64:3.2 # amd64 only + resources: {requests: {cpu: 100m, memory: 64Mi}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: huge} +spec: + selector: {matchLabels: {app: huge}} + template: + metadata: {labels: {app: huge}} + spec: + containers: + - name: huge + image: busybox:1.36 + command: ["sleep", "3600"] + resources: {requests: {memory: 64Gi}} +--- +apiVersion: v1 +kind: Pod +metadata: {name: typo} +spec: + containers: [{name: x, image: nginx:9.99.99}] +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: {name: old-ingress} +spec: {} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: {name: db} +spec: + serviceName: db + selector: {matchLabels: {app: db}} + template: + metadata: {labels: {app: db}} + spec: + containers: [{name: db, image: busybox:1.36, command: ["sleep", "3600"]}] + volumeClaimTemplates: + - metadata: {name: data} + spec: + storageClassName: gp3 + accessModes: [ReadWriteOnce] + resources: {requests: {storage: 1Gi}} diff --git a/pkg/preflight/k8s/cluster.go b/pkg/preflight/k8s/cluster.go index ea58317..c3bf73e 100644 --- a/pkg/preflight/k8s/cluster.go +++ b/pkg/preflight/k8s/cluster.go @@ -23,7 +23,9 @@ type Cluster struct { // Kinds maps "group/Kind" (core group: "/Kind") to true when served. Kinds map[string]bool // Versions lists served group/versions ("v1", "apps/v1", ...). - Versions map[string]bool + Versions map[string]bool + // Preferred maps a Kind to the apiVersion the cluster prefers for it. + Preferred map[string]string StorageClasses []string DefaultClass string } @@ -63,7 +65,7 @@ func (t timeoutRunner) Run(ctx context.Context, name string, args ...string) (st // of failing individually. func Probe(ctx context.Context, run executor.Runner, kubeContext string, facts *fact.Set) *Cluster { run = timeoutRunner{run} - c := &Cluster{Kinds: map[string]bool{}, Versions: map[string]bool{}} + c := &Cluster{Kinds: map[string]bool{}, Versions: map[string]bool{}, Preferred: map[string]string{}} kc := func(args ...string) []string { if kubeContext != "" { return append([]string{"--context", kubeContext}, args...) @@ -117,6 +119,9 @@ func Probe(ctx context.Context, run executor.Runner, kubeContext string, facts * // NAME [SHORTNAMES] APIVERSION NAMESPACED KIND gv, kind := f[len(f)-3], f[len(f)-1] c.Kinds[group(gv)+"/"+kind] = true + if _, dup := c.Preferred[kind]; !dup { + c.Preferred[kind] = gv + } } } @@ -195,6 +200,20 @@ func (c *Cluster) parseNodes(out string) error { return nil } +// Local reports whether the cluster runs inside a developer's Docker engine +// (minikube, kind, Docker Desktop, k3d, Rancher Desktop, OrbStack). Their +// nodes inherit the engine VM's emulation, so foreign-architecture images +// usually run, slowly; production nodes usually cannot run them at all. +func (c *Cluster) Local() bool { + ctx := strings.ToLower(c.Context) + for _, p := range []string{"minikube", "kind-", "docker-desktop", "k3d-", "rancher-desktop", "orbstack", "colima"} { + if strings.HasPrefix(ctx, p) { + return true + } + } + return false +} + // Platforms lists the distinct platforms of schedulable nodes. func (c *Cluster) Platforms() []string { seen := map[string]bool{} @@ -213,6 +232,12 @@ func (c *Cluster) Serves(apiVersion, kind string) (servedKind, servedVersion boo return c.Kinds[group(apiVersion)+"/"+kind], c.Versions[apiVersion] } +// ServedAs returns the apiVersion under which the cluster serves kind, when +// a manifest used another group or version for it. +func (c *Cluster) ServedAs(kind string) string { + return c.Preferred[kind] +} + func group(apiVersion string) string { if g, _, ok := strings.Cut(apiVersion, "/"); ok { return g diff --git a/pkg/preflight/k8s/k8s_test.go b/pkg/preflight/k8s/k8s_test.go index d2c8234..0cd814f 100644 --- a/pkg/preflight/k8s/k8s_test.go +++ b/pkg/preflight/k8s/k8s_test.go @@ -44,7 +44,7 @@ const nodesJSON = `{"items":[ func liveCluster() fakeKubectl { return fakeKubectl{ "version --client": `{"clientVersion":{}}`, - "config current-context": "kind-mixed", + "config current-context": "prod-eks", "version -o json": `{"serverVersion":{"gitVersion":"v1.31.0"}}`, "api-versions": "v1\napps/v1\nbatch/v1\nnetworking.k8s.io/v1", "api-resources": "pods po v1 true Pod\ndeployments deploy apps/v1 true Deployment\nstatefulsets sts apps/v1 true StatefulSet\njobs batch/v1 true Job\ningresses ing networking.k8s.io/v1 true Ingress\npersistentvolumeclaims pvc v1 true PersistentVolumeClaim\nservices svc v1 true Service", @@ -198,6 +198,9 @@ func TestLoadManifests(t *testing.T) { func TestAPIRule(t *testing.T) { fs := apiRule.Check(context.Background(), setup(t)) f := expect(t, fs, fact.Error, "Ingress/old: extensions/v1beta1 Ingress is not available") + if f.Fix != "Change apiVersion to networking.k8s.io/v1." { + t.Errorf("fix %q", f.Fix) + } if f.Predicts != `no matches for kind "Ingress" in version "extensions/v1beta1"` { t.Errorf("predicts %q", f.Predicts) } @@ -222,6 +225,10 @@ func TestPlatformRule(t *testing.T) { t.Errorf("evidence should name the arm node: %v", f.Evidence) } expect(t, fs, fact.Error, "Deployment/pinned: image acme/web-amd64:1 cannot run on any node") + + local := setup(t) + local.Cluster.Context = "minikube" + expect(t, platformRule.Check(context.Background(), local), fact.Warning, "Deployment/pinned: image acme/web-amd64:1 has no build for this cluster's nodes and will only run under emulation") if strings.Contains(titles(fs), "trainer") || strings.Contains(titles(fs), "huge") { t.Errorf("GPU pod is unschedulable (reported elsewhere); multi-arch nginx fits:\n%s", titles(fs)) } diff --git a/pkg/preflight/k8s/rules.go b/pkg/preflight/k8s/rules.go index 7c07df0..1286412 100644 --- a/pkg/preflight/k8s/rules.go +++ b/pkg/preflight/k8s/rules.go @@ -49,6 +49,10 @@ var apiRule = Rule{ seen[key] = true why := fmt.Sprintf("cluster: %s/%s is not served", o.APIVersion, o.Kind) fix := "Install the CRD or operator that provides " + o.Kind + " before applying." + if alt := env.Cluster.ServedAs(o.Kind); alt != "" { + why = fmt.Sprintf("cluster: %s is served as %s, not %s", o.Kind, alt, o.APIVersion) + fix = "Change apiVersion to " + alt + "." + } if kind { why = fmt.Sprintf("cluster: %s is served, but not in version %s", o.Kind, o.APIVersion) fix = "Use an API version this cluster serves (`kubectl api-resources | grep " + o.Kind + "`)." @@ -154,7 +158,13 @@ var platformRule = Rule{ predicts = "ErrImagePull: no matching manifest for the node's platform" } f.Predicts = predicts - if len(good) == 0 { + if len(good) == 0 && env.Cluster.Local() { + f.Severity = fact.Warning + f.Title = fmt.Sprintf("image %s has no build for this cluster's nodes and will only run under emulation", c.Image) + f.Evidence = append(f.Evidence, "cluster: "+env.Cluster.Context+" is a local cluster; its nodes usually inherit the Docker VM's amd64/arm64 emulation") + f.Fix = "Expect it to be slow here, and to fail on real clusters without emulation. Publish a multi-arch image." + f.Predicts = "runs under emulation here; exec format error on nodes without it" + } else if len(good) == 0 { f.Severity = fact.Error f.Title = fmt.Sprintf("image %s cannot run on any node this pod can be scheduled to", c.Image) f.Fix = "Build a multi-arch image (docker buildx build --platform …) or add nodes of a supported architecture." From d8e00c061cf68610e1f0fa495b3ac24d7e9075ea Mon Sep 17 00:00:00 2001 From: raphgm Date: Thu, 24 Sep 2026 15:04:50 +0100 Subject: [PATCH 3/4] examples: small Helm chart for trying container-preflight k8s --- examples/k8s-chart/Chart.yaml | 4 ++++ examples/k8s-chart/templates/app.yaml | 20 ++++++++++++++++++++ examples/k8s-chart/values.yaml | 3 +++ 3 files changed, 27 insertions(+) create mode 100644 examples/k8s-chart/Chart.yaml create mode 100644 examples/k8s-chart/templates/app.yaml create mode 100644 examples/k8s-chart/values.yaml diff --git a/examples/k8s-chart/Chart.yaml b/examples/k8s-chart/Chart.yaml new file mode 100644 index 0000000..3823eba --- /dev/null +++ b/examples/k8s-chart/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +name: demo +description: Small chart with deliberate problems, for trying `container-preflight k8s -`. +version: 0.1.0 diff --git a/examples/k8s-chart/templates/app.yaml b/examples/k8s-chart/templates/app.yaml new file mode 100644 index 0000000..e2b36de --- /dev/null +++ b/examples/k8s-chart/templates/app.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: {name: {{ .Release.Name }}-web} +spec: + selector: {matchLabels: {app: web}} + template: + metadata: {labels: {app: web}} + spec: + containers: + - name: web + image: {{ .Values.image }} + resources: {requests: {memory: {{ .Values.memory }}}} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: {{ .Release.Name }}-data} +spec: + storageClassName: {{ .Values.storageClass }} + accessModes: [ReadWriteOnce] + resources: {requests: {storage: 1Gi}} diff --git a/examples/k8s-chart/values.yaml b/examples/k8s-chart/values.yaml new file mode 100644 index 0000000..595e7be --- /dev/null +++ b/examples/k8s-chart/values.yaml @@ -0,0 +1,3 @@ +image: nginx:9.99.99 # tag that does not exist +memory: 64Gi # more than a laptop cluster has +storageClass: gp3 # AWS class, absent on local clusters From 7f3c304aabf71177a29c61193cbf7484bf6efbd1 Mon Sep 17 00:00:00 2001 From: raphgm Date: Thu, 24 Sep 2026 15:07:05 +0100 Subject: [PATCH 4/4] docs: second Kubernetes validation round (scheduling reasons, snapshots, root cause) --- docs/validation/2026-09-24-minikube-k8s.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/validation/2026-09-24-minikube-k8s.md b/docs/validation/2026-09-24-minikube-k8s.md index 911c800..23dae8b 100644 --- a/docs/validation/2026-09-24-minikube-k8s.md +++ b/docs/validation/2026-09-24-minikube-k8s.md @@ -25,3 +25,21 @@ the served version (`networking.k8s.io/v1`) instead of suggesting a CRD. Result: 4 of 5 predictions confirmed, 3 word for word. After the fix the tool's output matches what the cluster did. + +## Second round: scheduling reasons, snapshots, root cause + +| Pod | Predicted | Real `FailedScheduling` event | +|---|---|---| +| `wrong-pool` (`nodeSelector: pool=gpu`) | `0/1 nodes are available: 1 node(s) didn't match Pod's node affinity/selector.` | identical ✅ | +| `wants-gpu` (`nvidia.com/gpu: 1`) | `0/1 nodes are available: 1 Insufficient nvidia.com/gpu.` | identical ✅ | +| `arm-only-affinity` (required affinity `arch In [amd64]`) | `0/1 nodes are available: 1 node(s) didn't match Pod's node affinity/selector.` | identical ✅ | + +- **Snapshot.** `--save-cluster` wrote an 8 KB profile. Checking against it with + `--cluster`, and no cluster access, produced the same three predictions. +- **Unreachable cluster.** A context pointing at an address that does not answer was reported + once, as "Kubernetes API server is not reachable (context ghost)", blocking `k8s.api`, + `k8s.platform`, `k8s.schedulable` and `k8s.storage`. `k8s.image` still ran because it needs + only the registry. + +Combined result on minikube: 7 of 8 predictions confirmed word for word. The remaining one is +the emulation case, fixed above.