diff --git a/cmd/k8s.go b/cmd/k8s.go new file mode 100644 index 0000000..8b9032c --- /dev/null +++ b/cmd/k8s.go @@ -0,0 +1,85 @@ +package cmd + +import ( + "encoding/json" + "os" + + "github.com/spf13/cobra" + + "github.com/raphgm/container-preflight/internal/executor" + "github.com/raphgm/container-preflight/pkg/preflight" + "github.com/raphgm/container-preflight/pkg/preflight/k8s" +) + +var ( + k8sContext string + k8sCluster string + k8sSaveTo string + k8sOffline bool + k8sFormat string +) + +var k8sCmd = &cobra.Command{ + Use: "k8s [manifests...]", + Short: "Predict what applying Kubernetes manifests to a cluster would fail on", + Long: `Reads manifests (files, directories, or - for stdin, e.g. from +` + "`helm template`" + ` or ` + "`kustomize build`" + `), profiles the target cluster with +read-only kubectl calls, and queries registries for image platforms. It +predicts FailedScheduling, ImagePullBackOff, "no matches for kind", Pending +volume claims and wrong-architecture images before anything is applied. + + container-preflight k8s ./deploy + helm template my-release ./chart | container-preflight k8s - + container-preflight k8s --save-cluster prod.json # capture a cluster + container-preflight k8s ./deploy --cluster prod.json # check against it`, + RunE: func(cmd *cobra.Command, args []string) error { + if k8sSaveTo != "" { + snap := k8s.Capture(cmd.Context(), executor.New(), k8sContext) + f, err := os.Create(k8sSaveTo) + if err != nil { + return err + } + defer f.Close() + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + return enc.Encode(snap) + } + if len(args) == 0 { + args = []string{"."} + } + opts := preflight.K8sOptions{Paths: args, Context: k8sContext, Offline: k8sOffline} + if k8sCluster != "" { + snap, err := k8s.LoadSnapshot(k8sCluster) + if err != nil { + return err + } + opts.Snapshot = snap + } + rep, err := preflight.RunK8s(cmd.Context(), opts) + if err != nil { + return err + } + if k8sFormat == "json" { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(rep); err != nil { + return err + } + } else { + renderPreflight(os.Stdout, rep, useColor()) + } + if rep.Errors() > 0 { + os.Exit(1) + } + return nil + }, +} + +func init() { + rootCmd.AddCommand(k8sCmd) + k8sCmd.Flags().StringVar(&k8sContext, "context", "", "kubeconfig context (default: current)") + k8sCmd.Flags().StringVar(&k8sCluster, "cluster", "", "Check against a saved cluster snapshot instead of a live cluster") + k8sCmd.Flags().StringVar(&k8sSaveTo, "save-cluster", "", "Save a snapshot of the cluster to this file and exit") + k8sCmd.Flags().BoolVar(&k8sOffline, "offline", false, "Skip registry lookups") + k8sCmd.Flags().StringVarP(&k8sFormat, "format", "f", "term", "Output format (term, json)") +} diff --git a/cmd/preflight.go b/cmd/preflight.go index ca146a8..28b63a1 100644 --- a/cmd/preflight.go +++ b/cmd/preflight.go @@ -101,6 +101,9 @@ func renderPreflight(w io.Writer, rep *preflight.Report, color bool) { } h := rep.Host + if h == nil { + h = &host.Profile{} + } fmt.Fprintf(w, "\n%sContainer Preflight · preflight%s %s%s%s\n", c.bold, c.reset, c.dim, rep.Dir, c.reset) if rep.HostName != "" { fmt.Fprintf(w, "%sPredicting for snapshot of %s%s\n", c.yellow, rep.HostName, c.reset) @@ -115,6 +118,9 @@ func renderPreflight(w io.Writer, rep *preflight.Report, color bool) { if h.BuildxInstalled { hostBits = append(hostBits, "buildx "+h.BuildxVersion) } + if c := rep.Cluster; c != nil && c.ServerVersion != "" { + hostBits = append(hostBits, "cluster "+c.Context, "Kubernetes "+c.ServerVersion, fmt.Sprintf("%d node(s): %s", len(c.Nodes), strings.Join(c.Platforms(), ", "))) + } if len(hostBits) > 0 { fmt.Fprintf(w, "%sHost: %s%s\n", c.dim, strings.Join(hostBits, " · "), c.reset) } diff --git a/docs/validation/2026-09-24-minikube-k8s.md b/docs/validation/2026-09-24-minikube-k8s.md new file mode 100644 index 0000000..23dae8b --- /dev/null +++ b/docs/validation/2026-09-24-minikube-k8s.md @@ -0,0 +1,45 @@ +# Validation run: Kubernetes rules on minikube + +Date: 2026-09-24 +Cluster: minikube (Docker driver on Docker Desktop, Apple silicon), Kubernetes v1.37.0, one +`linux/arm64` node, default StorageClass `standard`. +Manifests: `examples/k8s/app.yaml`. + +`container-preflight k8s examples/k8s` was run first. The manifests were then applied to a +throw-away namespace, and the events Kubernetes reported were compared with the predictions. + +| Object | Prediction | Real outcome | Verdict | +|---|---|---|---| +| `Ingress/old-ingress` | `no matches for kind "Ingress" in version "extensions/v1beta1"` | `kubectl apply`: identical | ✅ exact | +| `Pod/typo` (`nginx:9.99.99`) | `ErrImagePull` | `ErrImagePull`, then `ImagePullBackOff` (`failed to resolve reference … not found`) | ✅ | +| `Deployment/huge` (64 GiB request) | `FailedScheduling: 0/1 nodes are available: 1 Insufficient memory.` | identical, followed by a preemption note | ✅ exact | +| `StatefulSet/db` (StorageClass `gp3`) | PVC Pending: `storageclass.storage.k8s.io "gp3" not found` | `ProvisioningFailed` with that exact text; PVC `Pending`; pod `FailedScheduling: … unbound immediate PersistentVolumeClaims` | ✅ exact | +| `Deployment/web` (`pause-amd64`, amd64 only) | first version: cannot run on any node → `exec format error` | pods `Running` | ❌ false positive | + +The false positive has the same cause as the Colima case. minikube's node runs inside Docker +Desktop's VM, which runs amd64 binaries under Rosetta, so the node inherits emulation that the +cluster API does not report. Local clusters (minikube, kind, Docker Desktop, k3d, Rancher +Desktop, OrbStack, Colima) now get a warning: "will only run under emulation". Other clusters keep +the error, because production arm64 nodes normally have no emulation. The Ingress fix also names +the served version (`networking.k8s.io/v1`) instead of suggesting a CRD. + +Result: 4 of 5 predictions confirmed, 3 word for word. After the fix the tool's output matches +what the cluster did. + +## Second round: scheduling reasons, snapshots, root cause + +| Pod | Predicted | Real `FailedScheduling` event | +|---|---|---| +| `wrong-pool` (`nodeSelector: pool=gpu`) | `0/1 nodes are available: 1 node(s) didn't match Pod's node affinity/selector.` | identical ✅ | +| `wants-gpu` (`nvidia.com/gpu: 1`) | `0/1 nodes are available: 1 Insufficient nvidia.com/gpu.` | identical ✅ | +| `arm-only-affinity` (required affinity `arch In [amd64]`) | `0/1 nodes are available: 1 node(s) didn't match Pod's node affinity/selector.` | identical ✅ | + +- **Snapshot.** `--save-cluster` wrote an 8 KB profile. Checking against it with + `--cluster`, and no cluster access, produced the same three predictions. +- **Unreachable cluster.** A context pointing at an address that does not answer was reported + once, as "Kubernetes API server is not reachable (context ghost)", blocking `k8s.api`, + `k8s.platform`, `k8s.schedulable` and `k8s.storage`. `k8s.image` still ran because it needs + only the registry. + +Combined result on minikube: 7 of 8 predictions confirmed word for word. The remaining one is +the emulation case, fixed above. diff --git a/examples/k8s-chart/Chart.yaml b/examples/k8s-chart/Chart.yaml new file mode 100644 index 0000000..3823eba --- /dev/null +++ b/examples/k8s-chart/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +name: demo +description: Small chart with deliberate problems, for trying `container-preflight k8s -`. +version: 0.1.0 diff --git a/examples/k8s-chart/templates/app.yaml b/examples/k8s-chart/templates/app.yaml new file mode 100644 index 0000000..e2b36de --- /dev/null +++ b/examples/k8s-chart/templates/app.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: {name: {{ .Release.Name }}-web} +spec: + selector: {matchLabels: {app: web}} + template: + metadata: {labels: {app: web}} + spec: + containers: + - name: web + image: {{ .Values.image }} + resources: {requests: {memory: {{ .Values.memory }}}} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: {{ .Release.Name }}-data} +spec: + storageClassName: {{ .Values.storageClass }} + accessModes: [ReadWriteOnce] + resources: {requests: {storage: 1Gi}} diff --git a/examples/k8s-chart/values.yaml b/examples/k8s-chart/values.yaml new file mode 100644 index 0000000..595e7be --- /dev/null +++ b/examples/k8s-chart/values.yaml @@ -0,0 +1,3 @@ +image: nginx:9.99.99 # tag that does not exist +memory: 64Gi # more than a laptop cluster has +storageClass: gp3 # AWS class, absent on local clusters diff --git a/examples/k8s/app.yaml b/examples/k8s/app.yaml new file mode 100644 index 0000000..f2387d4 --- /dev/null +++ b/examples/k8s/app.yaml @@ -0,0 +1,56 @@ +# Deliberately broken manifests for trying `container-preflight k8s`. +apiVersion: apps/v1 +kind: Deployment +metadata: {name: web} +spec: + replicas: 2 + selector: {matchLabels: {app: web}} + template: + metadata: {labels: {app: web}} + spec: + containers: + - name: web + image: gcr.io/google-containers/pause-amd64:3.2 # amd64 only + resources: {requests: {cpu: 100m, memory: 64Mi}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: huge} +spec: + selector: {matchLabels: {app: huge}} + template: + metadata: {labels: {app: huge}} + spec: + containers: + - name: huge + image: busybox:1.36 + command: ["sleep", "3600"] + resources: {requests: {memory: 64Gi}} +--- +apiVersion: v1 +kind: Pod +metadata: {name: typo} +spec: + containers: [{name: x, image: nginx:9.99.99}] +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: {name: old-ingress} +spec: {} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: {name: db} +spec: + serviceName: db + selector: {matchLabels: {app: db}} + template: + metadata: {labels: {app: db}} + spec: + containers: [{name: db, image: busybox:1.36, command: ["sleep", "3600"]}] + volumeClaimTemplates: + - metadata: {name: data} + spec: + storageClassName: gp3 + accessModes: [ReadWriteOnce] + resources: {requests: {storage: 1Gi}} diff --git a/pkg/preflight/fact/fact.go b/pkg/preflight/fact/fact.go index 089da3d..b7c76a0 100644 --- a/pkg/preflight/fact/fact.go +++ b/pkg/preflight/fact/fact.go @@ -17,6 +17,13 @@ const ( Registry ID = "registry.network" ComposeFile ID = "project.compose" Dockerfile ID = "project.dockerfile" + + Kubectl ID = "k8s.kubectl" + Cluster ID = "k8s.cluster" + ClusterAPIs ID = "k8s.apis" + Nodes ID = "k8s.nodes" + Storage ID = "k8s.storage" + Manifests ID = "project.manifests" ) // parents encodes the causal edges: a fact cannot be known unless its parent @@ -28,6 +35,11 @@ var parents = map[ID]ID{ LocalDaemon: DockerCLI, Compose: DockerCLI, Buildx: DockerCLI, + + Cluster: Kubectl, + ClusterAPIs: Cluster, + Nodes: Cluster, + Storage: Cluster, } // Parent returns the direct causal parent of id, if any. diff --git a/pkg/preflight/k8s/cluster.go b/pkg/preflight/k8s/cluster.go new file mode 100644 index 0000000..c3bf73e --- /dev/null +++ b/pkg/preflight/k8s/cluster.go @@ -0,0 +1,290 @@ +package k8s + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + osexec "os/exec" + "strings" + "time" + + "github.com/raphgm/container-preflight/internal/executor" + "github.com/raphgm/container-preflight/pkg/preflight/fact" + "github.com/raphgm/container-preflight/pkg/preflight/registry" +) + +// Cluster is what a specific cluster offers workloads. +type Cluster struct { + Context string + ServerVersion string + Nodes []Node + // Kinds maps "group/Kind" (core group: "/Kind") to true when served. + Kinds map[string]bool + // Versions lists served group/versions ("v1", "apps/v1", ...). + Versions map[string]bool + // Preferred maps a Kind to the apiVersion the cluster prefers for it. + Preferred map[string]string + StorageClasses []string + DefaultClass string +} + +type Node struct { + Name string + Labels map[string]string + Taints []Taint + CPU int64 // allocatable millicores + Memory int64 // allocatable bytes + Extended map[string]int64 + Platform string // os/arch + Unschedulable bool + Ready bool +} + +type Taint struct{ Key, Value, Effect string } + +// commandTimeout bounds each kubectl call; an unreachable API server can +// otherwise hang for minutes. +const commandTimeout = 20 * time.Second + +type timeoutRunner struct{ executor.Runner } + +func (t timeoutRunner) Run(ctx context.Context, name string, args ...string) (string, error) { + cctx, cancel := context.WithTimeout(ctx, commandTimeout) + defer cancel() + out, err := t.Runner.Run(cctx, name, args...) + if err != nil && cctx.Err() == context.DeadlineExceeded { + return out, fmt.Errorf("`%s %s` did not answer within %s", name, strings.Join(args, " "), commandTimeout) + } + return out, err +} + +// Probe reads the cluster with get/list calls only. Anything it cannot read +// becomes a fact failure, so dependent rules report the root cause instead +// of failing individually. +func Probe(ctx context.Context, run executor.Runner, kubeContext string, facts *fact.Set) *Cluster { + run = timeoutRunner{run} + c := &Cluster{Kinds: map[string]bool{}, Versions: map[string]bool{}, Preferred: map[string]string{}} + kc := func(args ...string) []string { + if kubeContext != "" { + return append([]string{"--context", kubeContext}, args...) + } + return args + } + + if _, err := run.Run(ctx, "kubectl", "version", "--client", "-o", "json"); err != nil { + facts.Fail(fact.Failure{Fact: fact.Kubectl, Severity: fact.Error, Summary: "kubectl not found", + Detail: errText(err), Fix: "Install kubectl and make sure it is on PATH."}) + return c + } + c.Context = kubeContext + if c.Context == "" { + c.Context, _ = run.Run(ctx, "kubectl", "config", "current-context") + } + + out, err := run.Run(ctx, "kubectl", kc("version", "-o", "json")...) + var ver struct { + ServerVersion struct{ GitVersion string } `json:"serverVersion"` + } + if err == nil { + _ = json.Unmarshal([]byte(out), &ver) + } + if ver.ServerVersion.GitVersion == "" { + summary := "Kubernetes API server is not reachable" + if c.Context == "" { + summary = "kubectl has no current context" + } else { + summary += " (context " + c.Context + ")" + } + facts.Fail(fact.Failure{Fact: fact.Cluster, Severity: fact.Error, Summary: summary, Detail: errText(err), + Fix: "Check `kubectl config get-contexts`, start the cluster (e.g. `kind create cluster`), or pass --context."}) + return c + } + c.ServerVersion = ver.ServerVersion.GitVersion + + if out, err := run.Run(ctx, "kubectl", kc("api-versions")...); err == nil { + for _, v := range strings.Fields(out) { + c.Versions[v] = true + } + } + if out, err := run.Run(ctx, "kubectl", kc("api-resources", "--no-headers")...); err != nil { + facts.Fail(fact.Failure{Fact: fact.ClusterAPIs, Severity: fact.Warning, Summary: "cannot list the cluster's API resources", Detail: errText(err)}) + } else { + for _, line := range strings.Split(out, "\n") { + f := strings.Fields(line) + if len(f) < 4 { + continue + } + // NAME [SHORTNAMES] APIVERSION NAMESPACED KIND + gv, kind := f[len(f)-3], f[len(f)-1] + c.Kinds[group(gv)+"/"+kind] = true + if _, dup := c.Preferred[kind]; !dup { + c.Preferred[kind] = gv + } + } + } + + if out, err := run.Run(ctx, "kubectl", kc("get", "nodes", "-o", "json")...); err != nil { + facts.Fail(fact.Failure{Fact: fact.Nodes, Severity: fact.Warning, Summary: "cannot list nodes", Detail: errText(err), + Fix: "Scheduling and platform checks need `get nodes` permission (cluster-scoped)."}) + } else if err := c.parseNodes(out); err != nil { + facts.Fail(fact.Failure{Fact: fact.Nodes, Severity: fact.Warning, Summary: "cannot read node list", Detail: err.Error()}) + } + + if out, err := run.Run(ctx, "kubectl", kc("get", "storageclasses", "-o", "json")...); err != nil { + facts.Fail(fact.Failure{Fact: fact.Storage, Severity: fact.Warning, Summary: "cannot list StorageClasses", Detail: errText(err)}) + } else { + var sc struct { + Items []struct { + Metadata struct { + Name string + Annotations map[string]string + } + } + } + if json.Unmarshal([]byte(out), &sc) == nil { + for _, it := range sc.Items { + c.StorageClasses = append(c.StorageClasses, it.Metadata.Name) + if it.Metadata.Annotations["storageclass.kubernetes.io/is-default-class"] == "true" { + c.DefaultClass = it.Metadata.Name + } + } + } + } + return c +} + +func (c *Cluster) parseNodes(out string) error { + var nl struct { + Items []struct { + Metadata struct { + Name string + Labels map[string]string + } + Spec struct { + Unschedulable bool + Taints []Taint + } + Status struct { + Allocatable map[string]string + NodeInfo struct { + Architecture string + OperatingSystem string + } + Conditions []struct{ Type, Status string } + } + } + } + if err := json.Unmarshal([]byte(out), &nl); err != nil { + return err + } + for _, it := range nl.Items { + n := Node{Name: it.Metadata.Name, Labels: it.Metadata.Labels, Taints: it.Spec.Taints, + Unschedulable: it.Spec.Unschedulable, Extended: map[string]int64{}} + n.CPU, _ = quantity(it.Status.Allocatable["cpu"], true) + n.Memory, _ = quantity(it.Status.Allocatable["memory"], false) + for k, v := range it.Status.Allocatable { + if strings.Contains(k, "/") { + n.Extended[k], _ = quantity(v, false) + } + } + n.Platform = registry.Normalize(it.Status.NodeInfo.OperatingSystem + "/" + it.Status.NodeInfo.Architecture) + for _, cond := range it.Status.Conditions { + if cond.Type == "Ready" { + n.Ready = cond.Status == "True" + } + } + c.Nodes = append(c.Nodes, n) + } + return nil +} + +// Local reports whether the cluster runs inside a developer's Docker engine +// (minikube, kind, Docker Desktop, k3d, Rancher Desktop, OrbStack). Their +// nodes inherit the engine VM's emulation, so foreign-architecture images +// usually run, slowly; production nodes usually cannot run them at all. +func (c *Cluster) Local() bool { + ctx := strings.ToLower(c.Context) + for _, p := range []string{"minikube", "kind-", "docker-desktop", "k3d-", "rancher-desktop", "orbstack", "colima"} { + if strings.HasPrefix(ctx, p) { + return true + } + } + return false +} + +// Platforms lists the distinct platforms of schedulable nodes. +func (c *Cluster) Platforms() []string { + seen := map[string]bool{} + var out []string + for _, n := range c.Nodes { + if n.Ready && !n.Unschedulable && !seen[n.Platform] { + seen[n.Platform] = true + out = append(out, n.Platform) + } + } + return out +} + +// Serves reports whether apiVersion/kind is available. +func (c *Cluster) Serves(apiVersion, kind string) (servedKind, servedVersion bool) { + return c.Kinds[group(apiVersion)+"/"+kind], c.Versions[apiVersion] +} + +// ServedAs returns the apiVersion under which the cluster serves kind, when +// a manifest used another group or version for it. +func (c *Cluster) ServedAs(kind string) string { + return c.Preferred[kind] +} + +func group(apiVersion string) string { + if g, _, ok := strings.Cut(apiVersion, "/"); ok { + return g + } + return "" +} + +// Snapshot is a portable cluster profile. +type Snapshot struct { + Version int `json:"version"` + CapturedAt time.Time `json:"capturedAt"` + Cluster *Cluster `json:"cluster"` + Failures []fact.Failure `json:"failures,omitempty"` +} + +func Capture(ctx context.Context, run executor.Runner, kubeContext string) *Snapshot { + facts := fact.NewSet() + c := Probe(ctx, run, kubeContext, facts) + return &Snapshot{Version: 1, CapturedAt: time.Now().UTC(), Cluster: c, Failures: facts.All()} +} + +func LoadSnapshot(path string) (*Snapshot, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var s Snapshot + if err := json.Unmarshal(b, &s); err != nil || s.Cluster == nil { + return nil, fmt.Errorf("%s: not a cluster snapshot", path) + } + return &s, nil +} + +func (s *Snapshot) Restore() (*Cluster, *fact.Set) { + facts := fact.NewSet() + for _, f := range s.Failures { + facts.Fail(f) + } + return s.Cluster, facts +} + +func errText(err error) string { + if err == nil { + return "" + } + if errors.Is(err, osexec.ErrNotFound) { + return "`kubectl` executable not found on PATH" + } + return strings.TrimPrefix(strings.TrimSpace(err.Error()), "exit status 1: ") +} diff --git a/pkg/preflight/k8s/k8s_test.go b/pkg/preflight/k8s/k8s_test.go new file mode 100644 index 0000000..0cd814f --- /dev/null +++ b/pkg/preflight/k8s/k8s_test.go @@ -0,0 +1,291 @@ +package k8s + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/raphgm/container-preflight/pkg/preflight/fact" + "github.com/raphgm/container-preflight/pkg/preflight/registry" + "github.com/raphgm/container-preflight/pkg/preflight/rules" +) + +type fakeKubectl map[string]string + +func (f fakeKubectl) Run(_ context.Context, name string, args ...string) (string, error) { + key := strings.Join(args, " ") + for prefix, out := range f { + if strings.HasPrefix(key, prefix) { + if strings.HasPrefix(out, "ERR:") { + return "", errors.New(strings.TrimPrefix(out, "ERR:")) + } + return out, nil + } + } + return "", fmt.Errorf("unexpected: kubectl %s", key) +} + +const nodesJSON = `{"items":[ + {"metadata":{"name":"amd-1","labels":{"kubernetes.io/arch":"amd64","pool":"general"}}, + "spec":{}, + "status":{"allocatable":{"cpu":"4","memory":"16Gi"},"nodeInfo":{"architecture":"amd64","operatingSystem":"linux"},"conditions":[{"type":"Ready","status":"True"}]}}, + {"metadata":{"name":"arm-1","labels":{"kubernetes.io/arch":"arm64","pool":"general"}}, + "spec":{}, + "status":{"allocatable":{"cpu":"2","memory":"8Gi"},"nodeInfo":{"architecture":"arm64","operatingSystem":"linux"},"conditions":[{"type":"Ready","status":"True"}]}}, + {"metadata":{"name":"gpu-1","labels":{"kubernetes.io/arch":"amd64","pool":"gpu"}}, + "spec":{"taints":[{"key":"nvidia.com/gpu","value":"present","effect":"NoSchedule"}]}, + "status":{"allocatable":{"cpu":"8","memory":"32Gi","nvidia.com/gpu":"1"},"nodeInfo":{"architecture":"amd64","operatingSystem":"linux"},"conditions":[{"type":"Ready","status":"True"}]}} +]}` + +func liveCluster() fakeKubectl { + return fakeKubectl{ + "version --client": `{"clientVersion":{}}`, + "config current-context": "prod-eks", + "version -o json": `{"serverVersion":{"gitVersion":"v1.31.0"}}`, + "api-versions": "v1\napps/v1\nbatch/v1\nnetworking.k8s.io/v1", + "api-resources": "pods po v1 true Pod\ndeployments deploy apps/v1 true Deployment\nstatefulsets sts apps/v1 true StatefulSet\njobs batch/v1 true Job\ningresses ing networking.k8s.io/v1 true Ingress\npersistentvolumeclaims pvc v1 true PersistentVolumeClaim\nservices svc v1 true Service", + "get nodes": nodesJSON, + "get storageclasses": `{"items":[{"metadata":{"name":"standard","annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}]}`, + } +} + +const manifests = `apiVersion: apps/v1 +kind: Deployment +metadata: {name: web} +spec: + replicas: 3 + template: + spec: + containers: + - name: web + image: acme/web-amd64:1 + resources: {requests: {cpu: 500m, memory: 256Mi}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: pinned} +spec: + template: + spec: + nodeSelector: {kubernetes.io/arch: arm64} + containers: + - {name: app, image: acme/web-amd64:1} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: trainer} +spec: + template: + spec: + containers: + - name: t + image: pytorch/pytorch:2 + resources: {limits: {nvidia.com/gpu: 1}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: huge} +spec: + template: + spec: + containers: + - {name: h, image: nginx:1.27, resources: {requests: {memory: 64Gi}}} +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: {name: old} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: {name: sm} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: {name: db} +spec: + template: + spec: + containers: [{name: db, image: postgres:16}] + volumeClaimTemplates: + - metadata: {name: data} + spec: {storageClassName: gp3} +--- +apiVersion: v1 +kind: Pod +metadata: {name: typo} +spec: + containers: [{name: x, image: nginx:9.99.99}] +` + +type fakeRegistry map[string][]string + +func (f fakeRegistry) Resolve(_ context.Context, ref, _ string) (*registry.Image, error) { + ps, ok := f[ref] + if !ok { + return nil, fmt.Errorf("%w: %s", registry.ErrNotFound, ref) + } + return ®istry.Image{Ref: ref, Index: len(ps) > 1, Platforms: ps}, nil +} + +func setup(t *testing.T) *Env { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "all.yaml"), []byte(manifests), 0o644); err != nil { + t.Fatal(err) + } + m, err := Load([]string{dir}) + if err != nil { + t.Fatal(err) + } + facts := fact.NewSet() + c := Probe(context.Background(), liveCluster(), "", facts) + if len(facts.Roots()) != 0 { + t.Fatalf("probe failures: %+v", facts.Roots()) + } + reg := fakeRegistry{ + "acme/web-amd64:1": {"linux/amd64"}, + "pytorch/pytorch:2": {"linux/amd64"}, + "nginx:1.27": {"linux/amd64", "linux/arm64"}, + "postgres:16": {"linux/amd64", "linux/arm64"}, + } + env := &Env{Manifests: m, Cluster: c, Facts: facts, Images: map[string]rules.ImageResult{}} + for _, w := range m.Workloads { + for _, ct := range allContainers(w.Pod) { + img, err := reg.Resolve(context.Background(), ct.Image, "") + env.Images[ct.Image] = rules.ImageResult{Image: img, Err: err} + } + } + return env +} + +func titles(fs []rules.Finding) string { + var out []string + for _, f := range fs { + out = append(out, string(f.Severity)+" "+f.Service+": "+f.Title) + } + return strings.Join(out, "\n") +} + +func expect(t *testing.T, fs []rules.Finding, sev fact.Severity, substr string) rules.Finding { + t.Helper() + for _, f := range fs { + if f.Severity == sev && strings.Contains(f.Service+": "+f.Title, substr) { + return f + } + } + t.Errorf("want %s containing %q in:\n%s", sev, substr, titles(fs)) + return rules.Finding{} +} + +func TestLoadManifests(t *testing.T) { + env := setup(t) + if len(env.Manifests.Objects) != 8 || len(env.Manifests.Workloads) != 6 { + t.Fatalf("objects %d workloads %d", len(env.Manifests.Objects), len(env.Manifests.Workloads)) + } + w := env.Manifests.Workloads[2] + if _, _, ext := w.Pod.Requests(); ext["nvidia.com/gpu"] != 1 { + t.Errorf("GPU limit should imply a request: %+v", ext) + } + if env.Cluster.DefaultClass != "standard" || len(env.Cluster.Nodes) != 3 { + t.Errorf("cluster = %+v", env.Cluster) + } +} + +func TestAPIRule(t *testing.T) { + fs := apiRule.Check(context.Background(), setup(t)) + f := expect(t, fs, fact.Error, "Ingress/old: extensions/v1beta1 Ingress is not available") + if f.Fix != "Change apiVersion to networking.k8s.io/v1." { + t.Errorf("fix %q", f.Fix) + } + if f.Predicts != `no matches for kind "Ingress" in version "extensions/v1beta1"` { + t.Errorf("predicts %q", f.Predicts) + } + expect(t, fs, fact.Error, "ServiceMonitor/sm") + if len(fs) != 2 { + t.Errorf("only two unserved objects:\n%s", titles(fs)) + } +} + +func TestImageRule(t *testing.T) { + fs := imageRule.Check(context.Background(), setup(t)) + expect(t, fs, fact.Error, "Pod/typo: image nginx:9.99.99 does not exist") + if len(fs) != 1 { + t.Errorf("\n%s", titles(fs)) + } +} + +func TestPlatformRule(t *testing.T) { + fs := platformRule.Check(context.Background(), setup(t)) + f := expect(t, fs, fact.Warning, "Deployment/web: image acme/web-amd64:1 fails on 1 of 2 eligible nodes") + if !strings.Contains(strings.Join(f.Evidence, " "), "arm-1 (linux/arm64)") { + t.Errorf("evidence should name the arm node: %v", f.Evidence) + } + expect(t, fs, fact.Error, "Deployment/pinned: image acme/web-amd64:1 cannot run on any node") + + local := setup(t) + local.Cluster.Context = "minikube" + expect(t, platformRule.Check(context.Background(), local), fact.Warning, "Deployment/pinned: image acme/web-amd64:1 has no build for this cluster's nodes and will only run under emulation") + if strings.Contains(titles(fs), "trainer") || strings.Contains(titles(fs), "huge") { + t.Errorf("GPU pod is unschedulable (reported elsewhere); multi-arch nginx fits:\n%s", titles(fs)) + } +} + +func TestSchedulableRule(t *testing.T) { + fs := schedulableRule.Check(context.Background(), setup(t)) + gpu := expect(t, fs, fact.Error, "Deployment/trainer: no node can run this pod") + want := "FailedScheduling: 0/3 nodes are available: 1 node(s) had untolerated taint {nvidia.com/gpu: present}, 2 Insufficient nvidia.com/gpu." + if gpu.Predicts != want { + t.Errorf("predicts\n %q\nwant\n %q", gpu.Predicts, want) + } + expect(t, fs, fact.Error, "Deployment/huge") + if len(fs) != 2 { + t.Errorf("\n%s", titles(fs)) + } +} + +func TestStorageRule(t *testing.T) { + env := setup(t) + fs := storageRule.Check(context.Background(), env) + f := expect(t, fs, fact.Error, `StatefulSet/db: claim data uses StorageClass "gp3", which does not exist`) + if !strings.Contains(f.Predicts, `storageclass.storage.k8s.io "gp3" not found`) { + t.Errorf("predicts %q", f.Predicts) + } + env.Cluster.DefaultClass = "" + env.Manifests.Claims = append(env.Manifests.Claims, Claim{Name: "logs"}) + expect(t, storageRule.Check(context.Background(), env), fact.Error, "claim logs sets no StorageClass and the cluster has no default") +} + +func TestUnreachableClusterIsOneRootCause(t *testing.T) { + facts := fact.NewSet() + Probe(context.Background(), fakeKubectl{ + "version --client": `{}`, + "config current-context": "prod", + "version -o json": "ERR:Unable to connect to the server: dial tcp 10.0.0.1:6443: i/o timeout", + }, "", facts) + roots := facts.Roots() + if len(roots) != 1 || roots[0].Fact != fact.Cluster || !strings.Contains(roots[0].Summary, "context prod") { + t.Fatalf("roots = %+v", roots) + } + for _, id := range []fact.ID{fact.Nodes, fact.ClusterAPIs, fact.Storage} { + if r, _ := facts.Root(id); r.Fact != fact.Cluster { + t.Errorf("%s should be blocked by the cluster", id) + } + } +} + +func TestQuantity(t *testing.T) { + cases := []struct { + in string + cpu bool + want int64 + }{{"500m", true, 500}, {"2", true, 2000}, {"0.25", true, 250}, {"1Gi", false, 1 << 30}, {"128Mi", false, 128 << 20}, {"1G", false, 1e9}, {"1e3", false, 1000}} + for _, c := range cases { + if got, err := quantity(c.in, c.cpu); err != nil || got != c.want { + t.Errorf("quantity(%q) = %d, %v; want %d", c.in, got, err, c.want) + } + } +} diff --git a/pkg/preflight/k8s/manifest.go b/pkg/preflight/k8s/manifest.go new file mode 100644 index 0000000..03c8661 --- /dev/null +++ b/pkg/preflight/k8s/manifest.go @@ -0,0 +1,314 @@ +// Package k8s predicts Kubernetes deployment failures before `kubectl apply` +// by joining what manifests require with what a specific cluster offers and +// what registries publish. +package k8s + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// Object is any manifest document, kept for API checks. +type Object struct { + APIVersion string + Kind string + Name string + Namespace string + Location string +} + +// Workload is an object that creates pods. +type Workload struct { + Object + Replicas int + Pod PodSpec + Claims []Claim // volumeClaimTemplates +} + +type PodSpec struct { + NodeSelector map[string]string + Required []NodeSelectorTerm // requiredDuringScheduling node affinity (ORed) + Tolerations []Toleration + Containers []Container // app containers + InitContainers []Container + ImagePullSecrets []string + ServiceAccount string +} + +type Container struct { + Name string + Image string + CPU int64 // requested millicores + Memory int64 // requested bytes + GPU int64 // nvidia.com/gpu + Extended map[string]int64 +} + +type NodeSelectorTerm struct { + Expressions []Requirement +} + +type Requirement struct { + Key string + Operator string + Values []string +} + +type Toleration struct { + Key, Operator, Value, Effect string +} + +type Claim struct { + Name string + StorageClass *string // nil: not set, "" explicitly empty + Location string +} + +// Manifests is everything read from the given files. +type Manifests struct { + Objects []Object + Workloads []Workload + Claims []Claim // standalone PersistentVolumeClaims +} + +// Load reads YAML manifests from files, directories (recursively, *.yaml and +// *.yml) or "-" for stdin, such as `helm template` or `kustomize build` +// output. +func Load(paths []string) (*Manifests, error) { + m := &Manifests{} + for _, p := range paths { + if p == "-" { + b, err := io.ReadAll(os.Stdin) + if err != nil { + return nil, err + } + if err := m.parse("stdin", b); err != nil { + return nil, err + } + continue + } + st, err := os.Stat(p) + if err != nil { + return nil, err + } + var files []string + if st.IsDir() { + err = filepath.WalkDir(p, func(f string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() && (strings.HasSuffix(f, ".yaml") || strings.HasSuffix(f, ".yml")) { + files = append(files, f) + } + return err + }) + if err != nil { + return nil, err + } + sort.Strings(files) + } else { + files = []string{p} + } + for _, f := range files { + b, err := os.ReadFile(f) + if err != nil { + return nil, err + } + if err := m.parse(f, b); err != nil { + return nil, err + } + } + } + if len(m.Objects) == 0 { + return nil, errors.New("no Kubernetes objects found in the given manifests") + } + return m, nil +} + +func (m *Manifests) parse(file string, b []byte) error { + dec := yaml.NewDecoder(bytes.NewReader(b)) + for { + var n yaml.Node + err := dec.Decode(&n) + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return fmt.Errorf("%s: %w", file, err) + } + if len(n.Content) == 0 { + continue + } + var doc map[string]any + if err := n.Decode(&doc); err != nil || doc == nil { + continue + } + m.add(doc, fmt.Sprintf("%s:%d", filepath.Base(file), n.Content[0].Line)) + } +} + +func (m *Manifests) add(doc map[string]any, loc string) { + kind := str(doc["kind"]) + if kind == "" { + return + } + if strings.HasSuffix(kind, "List") { + for _, it := range list(doc["items"]) { + if d, ok := it.(map[string]any); ok { + m.add(d, loc) + } + } + return + } + meta := mp(doc["metadata"]) + obj := Object{APIVersion: str(doc["apiVersion"]), Kind: kind, Name: str(meta["name"]), Namespace: str(meta["namespace"]), Location: loc} + m.Objects = append(m.Objects, obj) + + spec := mp(doc["spec"]) + var tmpl map[string]any + replicas := 1 + switch kind { + case "Pod": + tmpl = map[string]any{"spec": spec} + case "Deployment", "ReplicaSet", "StatefulSet", "DaemonSet", "ReplicationController": + tmpl = mp(spec["template"]) + if r, ok := spec["replicas"].(int); ok { + replicas = r + } + case "Job": + tmpl = mp(spec["template"]) + case "CronJob": + tmpl = mp(mp(mp(spec["jobTemplate"])["spec"])["template"]) + case "PersistentVolumeClaim": + m.Claims = append(m.Claims, claim(obj.Name, spec, loc)) + return + default: + return + } + w := Workload{Object: obj, Replicas: replicas, Pod: podSpec(mp(tmpl["spec"]))} + for _, c := range list(spec["volumeClaimTemplates"]) { + cm := mp(c) + w.Claims = append(w.Claims, claim(str(mp(cm["metadata"])["name"]), mp(cm["spec"]), loc)) + } + m.Workloads = append(m.Workloads, w) +} + +func claim(name string, spec map[string]any, loc string) Claim { + c := Claim{Name: name, Location: loc} + if v, ok := spec["storageClassName"]; ok { + s := str(v) + c.StorageClass = &s + } + return c +} + +func podSpec(s map[string]any) PodSpec { + p := PodSpec{NodeSelector: map[string]string{}, ServiceAccount: str(s["serviceAccountName"])} + for k, v := range mp(s["nodeSelector"]) { + p.NodeSelector[k] = str(v) + } + req := mp(mp(mp(s["affinity"])["nodeAffinity"])["requiredDuringSchedulingIgnoredDuringExecution"]) + for _, t := range list(req["nodeSelectorTerms"]) { + var term NodeSelectorTerm + for _, e := range list(mp(t)["matchExpressions"]) { + em := mp(e) + r := Requirement{Key: str(em["key"]), Operator: str(em["operator"])} + for _, v := range list(em["values"]) { + r.Values = append(r.Values, str(v)) + } + term.Expressions = append(term.Expressions, r) + } + p.Required = append(p.Required, term) + } + for _, t := range list(s["tolerations"]) { + tm := mp(t) + p.Tolerations = append(p.Tolerations, Toleration{Key: str(tm["key"]), Operator: str(tm["operator"]), Value: str(tm["value"]), Effect: str(tm["effect"])}) + } + for _, sec := range list(s["imagePullSecrets"]) { + p.ImagePullSecrets = append(p.ImagePullSecrets, str(mp(sec)["name"])) + } + p.Containers = containers(list(s["containers"])) + p.InitContainers = containers(list(s["initContainers"])) + return p +} + +func containers(items []any) []Container { + var out []Container + for _, c := range items { + cm := mp(c) + ct := Container{Name: str(cm["name"]), Image: str(cm["image"]), Extended: map[string]int64{}} + res := mp(cm["resources"]) + req, lim := mp(res["requests"]), mp(res["limits"]) + // A limit without a request implies an equal request. + get := func(k string) string { + if v, ok := req[k]; ok { + return str(v) + } + return str(lim[k]) + } + ct.CPU, _ = quantity(get("cpu"), true) + ct.Memory, _ = quantity(get("memory"), false) + for _, src := range []map[string]any{lim, req} { + for k, v := range src { + if strings.Contains(k, "/") { + n, _ := quantity(str(v), false) + ct.Extended[k] = n + } + } + } + ct.GPU = ct.Extended["nvidia.com/gpu"] + out = append(out, ct) + } + return out +} + +// Requests are what the scheduler reserves: the larger of the sum of app +// containers and the largest init container. +func (p PodSpec) Requests() (cpu, mem int64, extended map[string]int64) { + extended = map[string]int64{} + for _, c := range p.Containers { + cpu += c.CPU + mem += c.Memory + for k, v := range c.Extended { + extended[k] += v + } + } + for _, c := range p.InitContainers { + cpu = max(cpu, c.CPU) + mem = max(mem, c.Memory) + for k, v := range c.Extended { + extended[k] = max(extended[k], v) + } + } + return cpu, mem, extended +} + +func str(v any) string { + switch t := v.(type) { + case nil: + return "" + case string: + return t + default: + return fmt.Sprint(t) + } +} + +func mp(v any) map[string]any { + if m, ok := v.(map[string]any); ok { + return m + } + return map[string]any{} +} + +func list(v any) []any { + if l, ok := v.([]any); ok { + return l + } + return nil +} diff --git a/pkg/preflight/k8s/quantity.go b/pkg/preflight/k8s/quantity.go new file mode 100644 index 0000000..7333d56 --- /dev/null +++ b/pkg/preflight/k8s/quantity.go @@ -0,0 +1,62 @@ +package k8s + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +// quantity parses a Kubernetes resource quantity into a base value: CPU in +// millicores, everything else in bytes (or plain units). It covers the +// suffixes manifests use; exponent notation is accepted as a plain float. +func quantity(s string, cpu bool) (int64, error) { + s = strings.TrimSpace(s) + if s == "" { + return 0, nil + } + mult := 1.0 + suffixes := []struct { + s string + m float64 + }{ + {"Ki", 1 << 10}, {"Mi", 1 << 20}, {"Gi", 1 << 30}, {"Ti", 1 << 40}, {"Pi", 1 << 50}, {"Ei", 1 << 60}, + {"k", 1e3}, {"M", 1e6}, {"G", 1e9}, {"T", 1e12}, {"P", 1e15}, {"E", 1e18}, + {"m", 1e-3}, {"u", 1e-6}, {"n", 1e-9}, + } + for _, suf := range suffixes { + if strings.HasSuffix(s, suf.s) { + mult, s = suf.m, strings.TrimSuffix(s, suf.s) + break + } + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0, fmt.Errorf("invalid quantity %q", s) + } + v *= mult + if cpu { + v *= 1000 // cores -> millicores + } + return int64(math.Ceil(v - 1e-9)), nil +} + +func formatCPU(milli int64) string { + if milli%1000 == 0 { + return strconv.FormatInt(milli/1000, 10) + } + return strconv.FormatInt(milli, 10) + "m" +} + +func formatBytes(b int64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := int64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp]) +} diff --git a/pkg/preflight/k8s/rules.go b/pkg/preflight/k8s/rules.go new file mode 100644 index 0000000..1286412 --- /dev/null +++ b/pkg/preflight/k8s/rules.go @@ -0,0 +1,407 @@ +package k8s + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/raphgm/container-preflight/pkg/preflight/fact" + "github.com/raphgm/container-preflight/pkg/preflight/registry" + "github.com/raphgm/container-preflight/pkg/preflight/rules" +) + +// Env is everything a Kubernetes rule may read. +type Env struct { + Manifests *Manifests + Cluster *Cluster + Facts *fact.Set + // Images maps an image reference to its registry lookup. + Images map[string]rules.ImageResult +} + +type Rule struct { + ID string + Needs []fact.ID + Check func(ctx context.Context, env *Env) []rules.Finding +} + +func All() []Rule { + return []Rule{apiRule, imageRule, platformRule, schedulableRule, storageRule} +} + +var apiRule = Rule{ + ID: "k8s.api", + Needs: []fact.ID{fact.ClusterAPIs}, + Check: func(_ context.Context, env *Env) []rules.Finding { + var out []rules.Finding + seen := map[string]bool{} + for _, o := range env.Manifests.Objects { + key := o.APIVersion + " " + o.Kind + if seen[key] { + continue + } + kind, version := env.Cluster.Serves(o.APIVersion, o.Kind) + if kind && version { + continue + } + seen[key] = true + why := fmt.Sprintf("cluster: %s/%s is not served", o.APIVersion, o.Kind) + fix := "Install the CRD or operator that provides " + o.Kind + " before applying." + if alt := env.Cluster.ServedAs(o.Kind); alt != "" { + why = fmt.Sprintf("cluster: %s is served as %s, not %s", o.Kind, alt, o.APIVersion) + fix = "Change apiVersion to " + alt + "." + } + if kind { + why = fmt.Sprintf("cluster: %s is served, but not in version %s", o.Kind, o.APIVersion) + fix = "Use an API version this cluster serves (`kubectl api-resources | grep " + o.Kind + "`)." + } + out = append(out, rules.Finding{ + Rule: "k8s.api", Severity: fact.Error, Service: o.Kind + "/" + o.Name, + Title: fmt.Sprintf("%s %s is not available on this cluster", o.APIVersion, o.Kind), + Evidence: []string{why, "cluster: Kubernetes " + env.Cluster.ServerVersion}, + Fix: fix, Location: o.Location, + Predicts: fmt.Sprintf(`no matches for kind "%s" in version "%s"`, o.Kind, o.APIVersion), + }) + } + return out + }, +} + +var imageRule = Rule{ + ID: "k8s.image", + Needs: []fact.ID{fact.Registry}, + Check: func(_ context.Context, env *Env) []rules.Finding { + var out []rules.Finding + for _, w := range env.Manifests.Workloads { + for _, c := range allContainers(w.Pod) { + res, ok := env.Images[c.Image] + if !ok || res.Err == nil { + continue + } + f := rules.Finding{Rule: "k8s.image", Service: w.Kind + "/" + w.Name, Location: w.Location} + switch { + case errors.Is(res.Err, registry.ErrNotFound): + f.Severity, f.Title = fact.Error, fmt.Sprintf("image %s does not exist", c.Image) + f.Evidence = []string{"registry: " + firstLine(res.Err)} + f.Fix = "Fix the tag; check it on the registry's web page." + f.Predicts = "ErrImagePull: manifest unknown / not found" + case errors.Is(res.Err, registry.ErrUnauthorized) && len(w.Pod.ImagePullSecrets) == 0: + f.Severity, f.Title = fact.Warning, fmt.Sprintf("image %s is private or missing, and the pod has no imagePullSecrets", c.Image) + f.Evidence = []string{"registry: " + firstLine(res.Err), "project: no imagePullSecrets on the pod (the ServiceAccount's are not checked)"} + f.Fix = "Add an imagePullSecret with registry credentials, or check the image name." + f.Predicts = "ImagePullBackOff: pull access denied" + default: + continue + } + out = append(out, f) + } + } + return out + }, +} + +var platformRule = Rule{ + ID: "k8s.platform", + Needs: []fact.ID{fact.Nodes, fact.Registry}, + Check: func(_ context.Context, env *Env) []rules.Finding { + var out []rules.Finding + for _, w := range env.Manifests.Workloads { + // Nodes the pod may land on: placement rules, plus the extended + // resources (GPUs) it requests. CPU and memory are left to + // k8s.schedulable because they depend on what else runs there. + _, _, ext := w.Pod.Requests() + var eligible []Node + for _, n := range env.Cluster.Nodes { + if len(placementReasons(w.Pod, n)) > 0 { + continue + } + has := true + for k, v := range ext { + if n.Extended[k] < v { + has = false + } + } + if has { + eligible = append(eligible, n) + } + } + if len(eligible) == 0 { + continue // k8s.schedulable reports it + } + for _, c := range allContainers(w.Pod) { + res, ok := env.Images[c.Image] + if !ok || res.Image == nil { + continue + } + var bad, good []string + for _, n := range eligible { + if supports(res.Image, n.Platform) { + good = append(good, n.Name) + } else { + bad = append(bad, n.Name+" ("+n.Platform+")") + } + } + if len(bad) == 0 { + continue + } + f := rules.Finding{ + Rule: "k8s.platform", Service: w.Kind + "/" + w.Name, Location: w.Location, + Evidence: []string{ + "registry: " + c.Image + " ships " + strings.Join(res.Image.Platforms, ", "), + "cluster: eligible nodes without a matching platform: " + strings.Join(bad, ", "), + }, + } + predicts := "exec format error / CrashLoopBackOff" + if res.Image.Index { + predicts = "ErrImagePull: no matching manifest for the node's platform" + } + f.Predicts = predicts + if len(good) == 0 && env.Cluster.Local() { + f.Severity = fact.Warning + f.Title = fmt.Sprintf("image %s has no build for this cluster's nodes and will only run under emulation", c.Image) + f.Evidence = append(f.Evidence, "cluster: "+env.Cluster.Context+" is a local cluster; its nodes usually inherit the Docker VM's amd64/arm64 emulation") + f.Fix = "Expect it to be slow here, and to fail on real clusters without emulation. Publish a multi-arch image." + f.Predicts = "runs under emulation here; exec format error on nodes without it" + } else if len(good) == 0 { + f.Severity = fact.Error + f.Title = fmt.Sprintf("image %s cannot run on any node this pod can be scheduled to", c.Image) + f.Fix = "Build a multi-arch image (docker buildx build --platform …) or add nodes of a supported architecture." + } else { + f.Severity = fact.Warning + f.Title = fmt.Sprintf("image %s fails on %d of %d eligible nodes (mixed architectures)", c.Image, len(bad), len(eligible)) + f.Fix = fmt.Sprintf("Pin the pod with nodeSelector kubernetes.io/arch, or publish a multi-arch image. Nodes that work: %s.", strings.Join(good, ", ")) + } + out = append(out, f) + } + } + return out + }, +} + +var schedulableRule = Rule{ + ID: "k8s.schedulable", + Needs: []fact.ID{fact.Nodes}, + Check: func(_ context.Context, env *Env) []rules.Finding { + var out []rules.Finding + nodes := env.Cluster.Nodes + for _, w := range env.Manifests.Workloads { + if w.Kind == "DaemonSet" { + continue // zero matching nodes means zero pods, not a failure + } + cpu, mem, ext := w.Pod.Requests() + counts := map[string]int{} + fits := 0 + for _, n := range nodes { + reasons := placementReasons(w.Pod, n) + if n.CPU > 0 && cpu > n.CPU { + reasons = append(reasons, "Insufficient cpu") + } + if n.Memory > 0 && mem > n.Memory { + reasons = append(reasons, "Insufficient memory") + } + for k, v := range ext { + if v > n.Extended[k] { + reasons = append(reasons, "Insufficient "+k) + } + } + if len(reasons) == 0 { + fits++ + } + for _, r := range dedupe(reasons) { + counts[r]++ + } + } + if fits > 0 || len(nodes) == 0 { + continue + } + var parts []string + for r, n := range counts { + parts = append(parts, fmt.Sprintf("%d %s", n, r)) + } + sort.Strings(parts) + msg := fmt.Sprintf("0/%d nodes are available: %s.", len(nodes), strings.Join(parts, ", ")) + out = append(out, rules.Finding{ + Rule: "k8s.schedulable", Severity: fact.Error, Service: w.Kind + "/" + w.Name, Location: w.Location, + Title: "no node can run this pod", + Evidence: []string{ + fmt.Sprintf("project: requests cpu %s, memory %s%s", formatCPU(cpu), formatBytes(mem), extendedText(ext)), + "cluster: " + msg, + "capacity is compared with node allocatable; pods already running are not counted", + }, + Fix: "Relax the selector/affinity, add a toleration, lower the requests, or add a matching node.", + Predicts: "FailedScheduling: " + msg, + }) + } + return out + }, +} + +var storageRule = Rule{ + ID: "k8s.storage", + Needs: []fact.ID{fact.Storage}, + Check: func(_ context.Context, env *Env) []rules.Finding { + c := env.Cluster + known := map[string]bool{} + for _, s := range c.StorageClasses { + known[s] = true + } + var out []rules.Finding + check := func(owner string, cl Claim) { + f := rules.Finding{Rule: "k8s.storage", Severity: fact.Error, Service: owner, Location: cl.Location} + switch { + case cl.StorageClass == nil && c.DefaultClass == "": + f.Title = fmt.Sprintf("claim %s sets no StorageClass and the cluster has no default", cl.Name) + f.Evidence = []string{"cluster: StorageClasses " + listOrNone(c.StorageClasses) + ", none marked default"} + f.Fix = "Set storageClassName, or mark a StorageClass as default." + f.Predicts = "PersistentVolumeClaim Pending: no persistent volumes available for this claim and no storage class is set" + case cl.StorageClass != nil && *cl.StorageClass != "" && !known[*cl.StorageClass]: + f.Title = fmt.Sprintf("claim %s uses StorageClass %q, which does not exist", cl.Name, *cl.StorageClass) + f.Evidence = []string{"cluster: StorageClasses " + listOrNone(c.StorageClasses)} + f.Fix = "Use one of the cluster's StorageClasses, or install the provisioner." + f.Predicts = fmt.Sprintf(`PersistentVolumeClaim Pending: storageclass.storage.k8s.io "%s" not found`, *cl.StorageClass) + default: + return + } + out = append(out, f) + } + for _, cl := range env.Manifests.Claims { + check("PersistentVolumeClaim/"+cl.Name, cl) + } + for _, w := range env.Manifests.Workloads { + for _, cl := range w.Claims { + check(w.Kind+"/"+w.Name, cl) + } + } + return out + }, +} + +// placementReasons lists why the scheduler would skip node n for this pod, +// ignoring resources, in the scheduler's own wording. +func placementReasons(p PodSpec, n Node) []string { + var r []string + if !n.Ready { + r = append(r, "node(s) were not ready") + } + if n.Unschedulable { + r = append(r, "node(s) were unschedulable") + } + if !matchesSelector(p, n) { + r = append(r, "node(s) didn't match Pod's node affinity/selector") + } + for _, t := range n.Taints { + if (t.Effect == "NoSchedule" || t.Effect == "NoExecute") && !tolerates(p.Tolerations, t) { + r = append(r, fmt.Sprintf("node(s) had untolerated taint {%s: %s}", t.Key, t.Value)) + } + } + return r +} + +func matchesSelector(p PodSpec, n Node) bool { + for k, v := range p.NodeSelector { + if n.Labels[k] != v { + return false + } + } + if len(p.Required) == 0 { + return true + } + for _, term := range p.Required { // terms are ORed + ok := true + for _, e := range term.Expressions { // expressions are ANDed + v, has := n.Labels[e.Key] + switch e.Operator { + case "In": + ok = ok && has && contains(e.Values, v) + case "NotIn": + ok = ok && !(has && contains(e.Values, v)) + case "Exists": + ok = ok && has + case "DoesNotExist": + ok = ok && !has + } + } + if ok { + return true + } + } + return false +} + +func tolerates(ts []Toleration, t Taint) bool { + for _, tol := range ts { + if tol.Effect != "" && tol.Effect != t.Effect { + continue + } + switch { + case tol.Operator == "Exists" && (tol.Key == "" || tol.Key == t.Key): + return true + case (tol.Operator == "" || tol.Operator == "Equal") && tol.Key == t.Key && tol.Value == t.Value: + return true + } + } + return false +} + +func supports(img *registry.Image, platform string) bool { + for _, p := range img.Platforms { + if registry.Compatible(p, platform) { + return true + } + } + return false +} + +func allContainers(p PodSpec) []Container { + return append(append([]Container{}, p.InitContainers...), p.Containers...) +} + +func extendedText(ext map[string]int64) string { + var parts []string + for k, v := range ext { + parts = append(parts, fmt.Sprintf(", %s %d", k, v)) + } + sort.Strings(parts) + return strings.Join(parts, "") +} + +func listOrNone(xs []string) string { + if len(xs) == 0 { + return "(none)" + } + return strings.Join(xs, ", ") +} + +func dedupe(xs []string) []string { + seen := map[string]bool{} + var out []string + for _, x := range xs { + if !seen[x] { + seen[x] = true + out = append(out, x) + } + } + return out +} + +func contains(xs []string, s string) bool { + for _, x := range xs { + if x == s { + return true + } + } + return false +} + +func firstLine(err error) string { + s := err.Error() + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + if len(s) > 200 { + s = s[:200] + "…" + } + return s +} diff --git a/pkg/preflight/kubernetes.go b/pkg/preflight/kubernetes.go new file mode 100644 index 0000000..ee58699 --- /dev/null +++ b/pkg/preflight/kubernetes.go @@ -0,0 +1,90 @@ +package preflight + +import ( + "context" + "sort" + "time" + + "github.com/raphgm/container-preflight/internal/executor" + "github.com/raphgm/container-preflight/pkg/preflight/fact" + "github.com/raphgm/container-preflight/pkg/preflight/k8s" + "github.com/raphgm/container-preflight/pkg/preflight/registry" + "github.com/raphgm/container-preflight/pkg/preflight/rules" +) + +type K8sOptions struct { + Paths []string + Context string + Offline bool + Snapshot *k8s.Snapshot + Runner executor.Runner + Resolver registry.Resolver +} + +// RunK8s predicts what applying the manifests to a cluster would fail on. +func RunK8s(ctx context.Context, opts K8sOptions) (*Report, error) { + start := time.Now() + if opts.Runner == nil { + opts.Runner = executor.New() + } + if opts.Resolver == nil { + opts.Resolver = registry.NewRemote() + } + m, err := k8s.Load(opts.Paths) + if err != nil { + return nil, err + } + + var cluster *k8s.Cluster + var facts *fact.Set + if opts.Snapshot != nil { + cluster, facts = opts.Snapshot.Restore() + } else { + facts = fact.NewSet() + cluster = k8s.Probe(ctx, opts.Runner, opts.Context, facts) + } + + env := &k8s.Env{Manifests: m, Cluster: cluster, Facts: facts, Images: map[string]rules.ImageResult{}} + if opts.Offline { + facts.Fail(fact.Failure{Fact: fact.Registry, Severity: fact.Info, Summary: "Registry checks skipped (--offline)"}) + } else { + platform := "linux/amd64" + if ps := cluster.Platforms(); len(ps) > 0 { + platform = ps[0] + } + var pulls []rules.Pull + seen := map[string]bool{} + for _, w := range m.Workloads { + for _, c := range append(append([]k8s.Container{}, w.Pod.InitContainers...), w.Pod.Containers...) { + if c.Image != "" && !seen[c.Image] { + seen[c.Image] = true + pulls = append(pulls, rules.Pull{Ref: c.Image, Platform: platform}) + } + } + } + byKey := resolveImages(ctx, opts.Resolver, pulls, facts) + for _, p := range pulls { + env.Images[p.Ref] = byKey[rules.ImageKey(p.Ref, p.Platform)] + } + } + + rep := &Report{Project: "kubernetes manifests", Cluster: cluster, Timestamp: start} + blocked := map[fact.ID][]string{} + for _, r := range k8s.All() { + if root, ok := blockingRoot(facts, r.Needs); ok { + blocked[root.Fact] = append(blocked[root.Fact], r.ID) + continue + } + found := r.Check(ctx, env) + if len(found) == 0 { + rep.Passed = append(rep.Passed, r.ID) + } + rep.Findings = append(rep.Findings, found...) + } + for _, root := range facts.Roots() { + rep.Roots = append(rep.Roots, RootCause{Failure: root, Blocks: blocked[root.Fact]}) + } + sort.SliceStable(rep.Findings, func(i, j int) bool { return rank(rep.Findings[i].Severity) < rank(rep.Findings[j].Severity) }) + rep.Duration = time.Since(start) + return rep, nil +} diff --git a/pkg/preflight/preflight.go b/pkg/preflight/preflight.go index c3d2fa7..2639ca4 100644 --- a/pkg/preflight/preflight.go +++ b/pkg/preflight/preflight.go @@ -14,6 +14,7 @@ import ( "github.com/raphgm/container-preflight/internal/executor" "github.com/raphgm/container-preflight/pkg/preflight/fact" "github.com/raphgm/container-preflight/pkg/preflight/host" + "github.com/raphgm/container-preflight/pkg/preflight/k8s" "github.com/raphgm/container-preflight/pkg/preflight/learn" "github.com/raphgm/container-preflight/pkg/preflight/project" "github.com/raphgm/container-preflight/pkg/preflight/registry" @@ -48,7 +49,8 @@ type Report struct { Project string `json:"project"` Dir string `json:"dir"` HostName string `json:"hostName,omitempty"` - Host *host.Profile `json:"host"` + Host *host.Profile `json:"host,omitempty"` + Cluster *k8s.Cluster `json:"cluster,omitempty"` Roots []RootCause `json:"rootCauses"` Findings []rules.Finding `json:"findings"` Passed []string `json:"passed"`