diff --git a/charts/cozystack/Chart.yaml b/charts/cozystack/Chart.yaml index d5a7afa2..0025676e 100644 --- a/charts/cozystack/Chart.yaml +++ b/charts/cozystack/Chart.yaml @@ -12,9 +12,9 @@ templateOptions: fileValues: [] jsonValues: [] literalValues: [] - talosVersion: "v1.12" + talosVersion: "v1.14" withSecrets: "secrets.yaml" - kubernetesVersion: "v1.34.3" + kubernetesVersion: "v1.36.5" full: false applyOptions: preserve: false diff --git a/charts/cozystack/templates/_helpers.tpl b/charts/cozystack/templates/_helpers.tpl index 807bbbcc..bd34eaf9 100644 --- a/charts/cozystack/templates/_helpers.tpl +++ b/charts/cozystack/templates/_helpers.tpl @@ -6,57 +6,44 @@ {{- end }} {{- end }} +{{- /* Kubelet configuration keys at .indent: the preset's built-ins plus + extraKubeletExtraArgs. extraKubeletExtraArgs MUST NOT collide with + the built-ins — yaml.v3 (used by Talos config decode and by the + upgrade-time body write-back) rejects duplicate map keys, so a + silent merge would emit a config that cannot decode. Fail at render + with a precise hint naming the offending key; operators wanting a + different default fork the preset. */ -}} +{{- define "talos.config.kubelet_config" }} +{{- range $k, $_ := .root.Values.extraKubeletExtraArgs }} +{{- if or (eq $k "cpuManagerPolicy") (eq $k "maxPods") }} +{{- fail (printf "values.yaml: extraKubeletExtraArgs.%s collides with the cozystack preset's built-in kubelet.extraConfig — keys never override (yaml.v3 rejects duplicate map keys on decode). Remove the entry from extraKubeletExtraArgs, or fork the chart preset if you need a different default." $k) }} +{{- end }} +{{- end }} +{{- "cpuManagerPolicy: static" | nindent .indent }} +{{- "maxPods: 512" | nindent .indent }} +{{- with .root.Values.extraKubeletExtraArgs }} +{{- toYaml . | nindent $.indent }} +{{- end }} +{{- end }} + {{- /* Shared machine section: type, nodeLabels (controlplane), kubelet, sysctls, kernel, certSANs, files, install */ -}} {{- define "talos.config.machine.common" }} machine: - {{- if eq .MachineType "controlplane" }} + {{- $kubedocs := include "talm.contract.kubedocs" . }} + {{- if and (eq .MachineType "controlplane") (not $kubedocs) }} nodeLabels: node.kubernetes.io/exclude-from-external-load-balancers: $patch: delete {{- end }} type: {{ .MachineType }} + {{- if not $kubedocs }} kubelet: nodeIP: validSubnets: - {{- if .Values.advertisedSubnets }} - {{- toYaml .Values.advertisedSubnets | nindent 8 }} - {{- else }} - {{- /* Fall back to the subnet of the node's default-gateway-bearing - link. cidrNetwork masks host bits so the emitted YAML is the - canonical network form (192.168.201.0/24) rather than the - host form (192.168.201.10/24). Dedupe after masking because - a link with a secondary address in the same subnet would - otherwise produce duplicate list entries. */ -}} - {{- $addrs := fromJsonArray (include "talm.discovered.default_addresses_by_gateway" .) }} - {{- if not $addrs }} - {{- fail "values.yaml: `advertisedSubnets` was left empty and talm could not derive a default from discovery. No default-gateway-bearing link was found on the node. This field is a cluster-wide subnet selector fed to kubelet and etcd; `talm template` is invoked once per node and cannot merge per-node values into one cluster value. Either set advertisedSubnets explicitly in values.yaml, or ensure the node has a default route before running `talm template`." }} - {{- end }} - {{- $subnets := list }} - {{- range $addrs }} - {{- $subnets = append $subnets (. | cidrNetwork) }} - {{- end }} - {{- range uniq $subnets }} - - {{ . }} - {{- end }} - {{- end }} - {{- /* extraKubeletExtraArgs MUST NOT collide with the preset's - built-in extraConfig keys — yaml.v3 (used by Talos config - decode and by the upgrade-time body write-back) rejects - duplicate map keys, so a silent merge would emit a config - that cannot decode. Fail at render with a precise hint - naming the offending key; operators wanting a different - default fork the preset. */ -}} - {{- range $k, $_ := .Values.extraKubeletExtraArgs }} - {{- if or (eq $k "cpuManagerPolicy") (eq $k "maxPods") }} - {{- fail (printf "values.yaml: extraKubeletExtraArgs.%s collides with the cozystack preset's built-in kubelet.extraConfig — keys never override (yaml.v3 rejects duplicate map keys on decode). Remove the entry from extraKubeletExtraArgs, or fork the chart preset if you need a different default." $k) }} - {{- end }} - {{- end }} + {{- include "talm.advertised_subnets" (dict "root" . "indent" 8) }} extraConfig: - cpuManagerPolicy: static - maxPods: 512 - {{- with .Values.extraKubeletExtraArgs }} - {{- toYaml . | nindent 6 }} - {{- end }} + {{- include "talos.config.kubelet_config" (dict "root" . "indent" 6) }} + {{- end }} {{- /* extraSysctls MUST NOT collide with the preset's built-in sysctls; same rationale as extraKubeletExtraArgs. $builtinSysctls is the single source of truth for the preset-owned keys — keep @@ -196,6 +183,185 @@ machine: {{- end }} {{- end }} +{{- define "talos.config.cluster_name" -}} +{{ include "talm.validate.dns1123subdomain" (dict "value" (.Values.clusterName | default .Chart.Name) "field" "clusterName") | quote }} +{{- end }} + +{{- define "talos.config.endpoint" -}} +{{ required "values.yaml: `endpoint` must be set to the cluster control-plane URL (e.g. https://:6443). This field is cluster-wide: every node's kubelet and kube-proxy dials it, so it cannot be auto-derived from the current node's IP -- `talm template` runs once per node and has no way to reconcile per-node IPs into a single shared endpoint. For multi-node setups use a VIP (cozystack floatingIP) or an external load balancer; for single-node clusters the node's routable IP works." .Values.endpoint | quote }} +{{- end }} + +{{- /* Control-plane component flags at .indent: the preset's built-ins, + a collision guard against them, then the operator's extra args. + Expects a dict with "root" (the chart context) and "indent". */ -}} +{{- define "talos.config.controller_manager_args" }} +{{- $v := .root.Values }} +{{- "bind-address: 0.0.0.0" | nindent .indent }} +{{- if $v.allocateNodeCIDRs }} +{{- "allocate-node-cidrs: true" | nindent .indent }} +{{- printf "cluster-cidr: %q" (join "," $v.podSubnets) | nindent .indent }} +{{- else }} +{{- "allocate-node-cidrs: false" | nindent .indent }} +{{- end }} +{{- range $k, $_ := $v.extraControllerManagerArgs }} +{{- /* cluster-cidr is preset-owned only when allocateNodeCIDRs is on + (that is the only branch that emits it); with it off the + operator may set cluster-cidr freely. */}} +{{- if or (eq $k "bind-address") (eq $k "allocate-node-cidrs") (and (eq $k "cluster-cidr") $v.allocateNodeCIDRs) }} +{{- fail (printf "values.yaml: extraControllerManagerArgs.%s collides with the cozystack preset's built-in controllerManager.extraArgs; drop it from extraControllerManagerArgs" $k) }} +{{- end }} +{{- end }} +{{- include "talm.render.component_args" (dict "args" $v.extraControllerManagerArgs "field" "extraControllerManagerArgs" "indent" .indent) }} +{{- end }} + +{{- define "talos.config.scheduler_args" }} +{{- $v := .root.Values }} +{{- "bind-address: 0.0.0.0" | nindent .indent }} +{{- range $k, $_ := $v.extraSchedulerArgs }} +{{- if eq $k "bind-address" }} +{{- fail (printf "values.yaml: extraSchedulerArgs.%s collides with the cozystack preset's built-in scheduler.extraArgs; drop it from extraSchedulerArgs" $k) }} +{{- end }} +{{- end }} +{{- include "talm.render.component_args" (dict "args" $v.extraSchedulerArgs "field" "extraSchedulerArgs" "indent" .indent) }} +{{- end }} + +{{- /* Empty when there is nothing to pass, so the caller can skip the + extraArgs key. "oidcFlags" puts OIDC on the command line; from + contract v1.14 it goes to KubeAuthenticationConfig instead. */ -}} +{{- define "talos.config.api_server_args" }} +{{- $v := .root.Values }} +{{- $oidcSet := and $v.oidcIssuerUrl (ne $v.oidcIssuerUrl "") }} +{{- if and $oidcSet .oidcFlags }} +{{- printf "oidc-issuer-url: \"%s\"" $v.oidcIssuerUrl | nindent .indent }} +{{- "oidc-client-id: \"kubernetes\"" | nindent .indent }} +{{- "oidc-username-claim: \"preferred_username\"" | nindent .indent }} +{{- "oidc-groups-claim: \"groups\"" | nindent .indent }} +{{- end }} +{{- range $k, $_ := $v.extraApiServerArgs }} +{{- if and $oidcSet (has $k (list "oidc-issuer-url" "oidc-client-id" "oidc-username-claim" "oidc-groups-claim")) }} +{{- fail (printf "values.yaml: extraApiServerArgs.%s collides with the cozystack preset's built-in apiServer OIDC args (active because oidcIssuerUrl is set); drop it from extraApiServerArgs" $k) }} +{{- end }} +{{- end }} +{{- include "talm.render.component_args" (dict "args" $v.extraApiServerArgs "field" "extraApiServerArgs" "indent" .indent) }} +{{- end }} + +{{- /* Cluster settings on contract v1.14+. Only etcd stays in v1alpha1; + the rest are typed documents the engine merges into the ones the + generator emits, so they carry only what this preset sets and + delete what it turns off. */ -}} +{{- define "talos.config.cluster.kubedocs" }} +{{- $controlplane := eq .MachineType "controlplane" }} +{{- if $controlplane }} +cluster: + etcd: + advertisedSubnets: + {{- include "talm.advertised_subnets" (dict "root" . "indent" 6) }} + {{- with (.Values.etcd | default dict).quotaBackendBytes }} + extraArgs: + quota-backend-bytes: {{ . | quote }} + {{- end }} +{{- end }} +--- +apiVersion: v1alpha1 +kind: KubeClusterConfig +clusterName: {{ include "talos.config.cluster_name" . }} +endpoint: {{ include "talos.config.endpoint" . }} +--- +apiVersion: v1alpha1 +kind: KubeNodeConfig +nodeIP: + validSubnets: + {{- include "talm.advertised_subnets" (dict "root" . "indent" 4) }} +{{- if $controlplane }} +labels: + node.kubernetes.io/exclude-from-external-load-balancers: + $patch: delete +{{- /* allowSchedulingOnControlPlanes: the generator taints control planes. */}} +taints: + node-role.kubernetes.io/control-plane: + $patch: delete +{{- end }} +--- +apiVersion: v1alpha1 +kind: KubeletConfig +config: + {{- include "talos.config.kubelet_config" (dict "root" . "indent" 2) }} +--- +apiVersion: v1alpha1 +kind: KubeNetworkConfig +dnsDomain: {{ include "talm.validate.dns1123subdomain" (dict "value" .Values.clusterDomain "field" "clusterDomain") | quote }} +podSubnets: + {{- toYaml .Values.podSubnets | nindent 2 }} +serviceSubnets: + {{- toYaml .Values.serviceSubnets | nindent 2 }} +{{- if $controlplane }} +--- +apiVersion: v1alpha1 +kind: KubeFlannelCNIConfig +$patch: delete +--- +apiVersion: v1alpha1 +kind: DiscoveryServiceConfig +name: default +$patch: delete +--- +apiVersion: v1alpha1 +kind: KubeProxyConfig +enabled: false +--- +apiVersion: v1alpha1 +kind: KubeControllerManagerConfig +extraArgs: + {{- include "talos.config.controller_manager_args" (dict "root" . "indent" 2) }} +--- +apiVersion: v1alpha1 +kind: KubeSchedulerConfig +extraArgs: + {{- include "talos.config.scheduler_args" (dict "root" . "indent" 2) }} +--- +apiVersion: v1alpha1 +kind: KubeAPIServerConfig +{{- include "talm.guard.apiserver_args_kubedocs" . }} +{{- with include "talos.config.api_server_args" (dict "root" . "indent" 2) }} +extraArgs: + {{- . }} +{{- end }} +certExtraSANs: + - 127.0.0.1 + {{- with .Values.certSANs }} + {{- toYaml . | nindent 2 }} + {{- end }} +{{- with .Values.oidcIssuerUrl }} +{{- /* The command-line OIDC flags as a JWT authenticator, same semantics: + a username claim other than email gets the "#" prefix the + flags default to. configuration replaces the generator's document + whole, so the anonymous health-check access it grants is repeated. */}} +--- +apiVersion: v1alpha1 +kind: KubeAuthenticationConfig +configuration: + anonymous: + enabled: true + conditions: + - path: /livez + - path: /readyz + - path: /healthz + jwt: + - issuer: + url: {{ . | quote }} + audiences: + - kubernetes + claimMappings: + username: + claim: preferred_username + prefix: {{ printf "%s#" . | quote }} + groups: + claim: groups + prefix: "" +{{- end }} +{{- end }} +{{- end }} + {{- /* Shared cluster section */ -}} {{- define "talos.config.cluster" }} cluster: @@ -207,54 +373,21 @@ cluster: {{- toYaml .Values.podSubnets | nindent 6 }} serviceSubnets: {{- toYaml .Values.serviceSubnets | nindent 6 }} - clusterName: {{ include "talm.validate.dns1123subdomain" (dict "value" (.Values.clusterName | default .Chart.Name) "field" "clusterName") | quote }} + clusterName: {{ include "talos.config.cluster_name" . }} controlPlane: - endpoint: {{ required "values.yaml: `endpoint` must be set to the cluster control-plane URL (e.g. https://:6443). This field is cluster-wide: every node's kubelet and kube-proxy dials it, so it cannot be auto-derived from the current node's IP -- `talm template` runs once per node and has no way to reconcile per-node IPs into a single shared endpoint. For multi-node setups use a VIP (cozystack floatingIP) or an external load balancer; for single-node clusters the node's routable IP works." .Values.endpoint | quote }} + endpoint: {{ include "talos.config.endpoint" . }} {{- if eq .MachineType "controlplane" }} allowSchedulingOnControlPlanes: true controllerManager: extraArgs: - bind-address: 0.0.0.0 - {{- if .Values.allocateNodeCIDRs }} - allocate-node-cidrs: true - cluster-cidr: "{{ join "," .Values.podSubnets }}" - {{- else }} - allocate-node-cidrs: false - {{- end }} - {{- range $k, $_ := .Values.extraControllerManagerArgs }} - {{- /* cluster-cidr is preset-owned only when allocateNodeCIDRs is on - (that is the only branch that emits it); with it off the - operator may set cluster-cidr freely. */}} - {{- if or (eq $k "bind-address") (eq $k "allocate-node-cidrs") (and (eq $k "cluster-cidr") $.Values.allocateNodeCIDRs) }} - {{- fail (printf "values.yaml: extraControllerManagerArgs.%s collides with the cozystack preset's built-in controllerManager.extraArgs; drop it from extraControllerManagerArgs" $k) }} - {{- end }} - {{- end }} - {{- include "talm.render.component_args" (dict "args" .Values.extraControllerManagerArgs "field" "extraControllerManagerArgs") }} + {{- include "talos.config.controller_manager_args" (dict "root" . "indent" 6) }} scheduler: extraArgs: - bind-address: 0.0.0.0 - {{- range $k, $_ := .Values.extraSchedulerArgs }} - {{- if eq $k "bind-address" }} - {{- fail (printf "values.yaml: extraSchedulerArgs.%s collides with the cozystack preset's built-in scheduler.extraArgs; drop it from extraSchedulerArgs" $k) }} - {{- end }} - {{- end }} - {{- include "talm.render.component_args" (dict "args" .Values.extraSchedulerArgs "field" "extraSchedulerArgs") }} + {{- include "talos.config.scheduler_args" (dict "root" . "indent" 6) }} apiServer: - {{- $oidcSet := and .Values.oidcIssuerUrl (ne .Values.oidcIssuerUrl "") }} - {{- if or $oidcSet .Values.extraApiServerArgs }} + {{- if include "talos.config.api_server_args" (dict "root" . "indent" 6 "oidcFlags" true) }} extraArgs: - {{- if $oidcSet }} - oidc-issuer-url: "{{ .Values.oidcIssuerUrl }}" - oidc-client-id: "kubernetes" - oidc-username-claim: "preferred_username" - oidc-groups-claim: "groups" - {{- end }} - {{- range $k, $_ := .Values.extraApiServerArgs }} - {{- if and $oidcSet (has $k (list "oidc-issuer-url" "oidc-client-id" "oidc-username-claim" "oidc-groups-claim")) }} - {{- fail (printf "values.yaml: extraApiServerArgs.%s collides with the cozystack preset's built-in apiServer OIDC args (active because oidcIssuerUrl is set); drop it from extraApiServerArgs" $k) }} - {{- end }} - {{- end }} - {{- include "talm.render.component_args" (dict "args" .Values.extraApiServerArgs "field" "extraApiServerArgs") }} + {{- include "talos.config.api_server_args" (dict "root" . "indent" 6 "oidcFlags" true) }} {{- end }} certSANs: - 127.0.0.1 @@ -267,23 +400,7 @@ cluster: enabled: false etcd: advertisedSubnets: - {{- if .Values.advertisedSubnets }} - {{- toYaml .Values.advertisedSubnets | nindent 6 }} - {{- else }} - {{- /* Fall back to the subnet of the node's default-gateway-bearing - link; cidrNetwork masks host bits to emit canonical network - form. Dedupe handled the same way as validSubnets above. - Empty discovery already errored via validSubnets' required() - guard, so we reach this block only when at least one address - was resolved. */ -}} - {{- $subnets := list }} - {{- range fromJsonArray (include "talm.discovered.default_addresses_by_gateway" .) }} - {{- $subnets = append $subnets (. | cidrNetwork) }} - {{- end }} - {{- range uniq $subnets }} - - {{ . }} - {{- end }} - {{- end }} + {{- include "talm.advertised_subnets" (dict "root" . "indent" 6) }} {{- /* etcd backend quota, tunable via values. Raises etcd's 2GiB default backend ceiling so a LINSTOR-heavy control plane — thousands of DRBD-resource CRDs in aggregate — does not trip @@ -478,7 +595,11 @@ environment: {{- define "talos.config.multidoc" }} {{- include "talos.config.machine.common" . }} +{{- if include "talm.contract.kubedocs" . }} +{{- include "talos.config.cluster.kubedocs" . }} +{{- else }} {{- include "talos.config.cluster" . }} +{{- end }} {{- include "talm.config.registries.multidoc" . }} {{- include "talos.config.network.multidoc" . }} {{- include "talos.config.zfs" . }} diff --git a/charts/cozystack/values.yaml b/charts/cozystack/values.yaml index 361650ce..634618cc 100644 --- a/charts/cozystack/values.yaml +++ b/charts/cozystack/values.yaml @@ -77,17 +77,18 @@ floatingIP: "" # non-existent link and the cluster endpoint will be unreachable. # Example: vipLink: eth0.4000 vipLink: "" -image: "ghcr.io/cozystack/cozystack/talos:v1.12.6" +image: "ghcr.io/cozystack/cozystack/talos:v1.14.2" podSubnets: - 10.244.0.0/16 serviceSubnets: - 10.96.0.0/16 -# Optional override for machine.kubelet.nodeIP.validSubnets and -# cluster.etcd.advertisedSubnets. When left empty the chart derives -# the value from the node's IPv4-default-gateway-bearing link at -# render time (via talm.discovered.default_addresses_by_gateway), so -# the generated machine config matches the node's actual network +# Optional override for the kubelet node IP subnets +# (machine.kubelet.nodeIP.validSubnets; KubeNodeConfig nodeIP.validSubnets +# from Talos v1.14) and cluster.etcd.advertisedSubnets. When left empty +# the chart derives the value from the node's IPv4-default-gateway-bearing +# link at render time (via talm.discovered.default_addresses_by_gateway), +# so the generated machine config matches the node's actual network # without any values.yaml edit. Set this only when you want to pin a # specific subnet — typically for multi-homed nodes where the # IPv4-default-gateway link is not the subnet you want kubelet/etcd @@ -178,12 +179,15 @@ etcd: # - bar=baz extraKernelModules: [] -# Extra kubelet args added to the preset's kubelet.extraConfig -# (cpuManagerPolicy: static, maxPods: 512). Operator keys must be -# DISJOINT from the built-in set; a collision fails the render with -# a hint pointing at the offending key. Example: +# Extra kubelet args added to the preset's built-in kubelet settings +# (cpuManagerPolicy: static, maxPods: 512), which land in +# machine.kubelet.extraConfig, or KubeletConfig config from Talos v1.14. +# Operator keys must be DISJOINT from the built-in set; a collision +# fails the render with a hint pointing at the offending key. Keys and +# values follow KubeletConfiguration. Example: # extraKubeletExtraArgs: -# feature-gates: "NodeSwap=true" +# featureGates: +# NodeSwap: true extraKubeletExtraArgs: {} # Extra kernel sysctls added to machine.sysctls alongside the @@ -232,7 +236,11 @@ timeServers: [] # a key that collides with a preset built-in fails the render — for # controllerManager (bind-address / allocate-node-cidrs / cluster-cidr), # scheduler (bind-address), and apiServer (the oidc-* args, but only -# when oidcIssuerUrl is set). Example: +# when oidcIssuerUrl is set). From Talos v1.14 kube-apiserver takes no +# authentication or authorization flags, so any oidc-*, anonymous-auth, +# authentication-config or authorization-config/-mode/-webhook-* key +# fails the render; oidcIssuerUrl becomes a KubeAuthenticationConfig +# document instead. Example: # extraApiServerArgs: { max-requests-inflight: "2000" } extraApiServerArgs: {} extraControllerManagerArgs: {} diff --git a/charts/generic/Chart.yaml b/charts/generic/Chart.yaml index e00d02bb..af1af4b2 100644 --- a/charts/generic/Chart.yaml +++ b/charts/generic/Chart.yaml @@ -15,11 +15,10 @@ templateOptions: # Pin both: an empty key either renders what the node picks (with a warning) # or stops the render on newer contracts. The Kubernetes version is the # default of the pinned Talos line, so a fresh project lands on what that - # line ships and a re-sync does not move the control plane. Replace it with - # the version your cluster runs. - talosVersion: "v1.13" + # line ships. Replace it with the version your cluster runs. + talosVersion: "v1.14" withSecrets: "secrets.yaml" - kubernetesVersion: "v1.36.2" + kubernetesVersion: "v1.37.0" full: false applyOptions: preserve: false diff --git a/charts/generic/templates/_helpers.tpl b/charts/generic/templates/_helpers.tpl index e63553a6..0353019a 100644 --- a/charts/generic/templates/_helpers.tpl +++ b/charts/generic/templates/_helpers.tpl @@ -10,34 +10,16 @@ {{- define "talos.config.machine.common" }} machine: type: {{ .MachineType }} + {{- if not (include "talm.contract.kubedocs" .) }} kubelet: nodeIP: validSubnets: - {{- if .Values.advertisedSubnets }} - {{- toYaml .Values.advertisedSubnets | nindent 8 }} - {{- else }} - {{- /* Fall back to the subnet of the node's default-gateway-bearing - link. cidrNetwork masks host bits so the emitted YAML is the - canonical network form (192.168.201.0/24) rather than the - host form (192.168.201.10/24). Dedupe after masking because - a link with a secondary address in the same subnet would - otherwise produce duplicate list entries. */ -}} - {{- $addrs := fromJsonArray (include "talm.discovered.default_addresses_by_gateway" .) }} - {{- if not $addrs }} - {{- fail "values.yaml: `advertisedSubnets` was left empty and talm could not derive a default from discovery. No default-gateway-bearing link was found on the node. This field is a cluster-wide subnet selector fed to kubelet and etcd; `talm template` is invoked once per node and cannot merge per-node values into one cluster value. Either set advertisedSubnets explicitly in values.yaml, or ensure the node has a default route before running `talm template`." }} - {{- end }} - {{- $subnets := list }} - {{- range $addrs }} - {{- $subnets = append $subnets (. | cidrNetwork) }} - {{- end }} - {{- range uniq $subnets }} - - {{ . }} - {{- end }} - {{- end }} + {{- include "talm.advertised_subnets" (dict "root" . "indent" 8) }} {{- with .Values.extraKubeletExtraArgs }} extraConfig: {{- toYaml . | nindent 6 }} {{- end }} + {{- end }} {{- with .Values.certSANs }} certSANs: {{- toYaml . | nindent 2 }} @@ -85,6 +67,82 @@ machine: {{- end }} {{- end }} +{{- define "talos.config.cluster_name" -}} +{{ include "talm.validate.dns1123subdomain" (dict "value" (.Values.clusterName | default .Chart.Name) "field" "clusterName") | quote }} +{{- end }} + +{{- define "talos.config.endpoint" -}} +{{ required "values.yaml: `endpoint` must be set to the cluster control-plane URL (e.g. https://:6443). This field is cluster-wide: every node's kubelet and kube-proxy dials it, so it cannot be auto-derived from the current node's IP -- `talm template` runs once per node and has no way to reconcile per-node IPs into a single shared endpoint. For multi-node setups use a VIP or an external load balancer; for single-node clusters the node's routable IP works." .Values.endpoint | quote }} +{{- end }} + +{{- /* Cluster settings on contract v1.14+. Only etcd stays in v1alpha1; + the rest are typed documents the engine merges into the ones the + generator emits, so they carry only what this chart sets. */ -}} +{{- define "talos.config.cluster.kubedocs" }} +{{- if eq .MachineType "controlplane" }} + +cluster: + etcd: + advertisedSubnets: + {{- include "talm.advertised_subnets" (dict "root" . "indent" 6) }} +{{- end }} +--- +apiVersion: v1alpha1 +kind: KubeClusterConfig +clusterName: {{ include "talos.config.cluster_name" . }} +endpoint: {{ include "talos.config.endpoint" . }} +--- +apiVersion: v1alpha1 +kind: KubeNodeConfig +nodeIP: + validSubnets: + {{- include "talm.advertised_subnets" (dict "root" . "indent" 4) }} +{{- with .Values.extraKubeletExtraArgs }} +--- +apiVersion: v1alpha1 +kind: KubeletConfig +config: + {{- toYaml . | nindent 2 }} +{{- end }} +--- +apiVersion: v1alpha1 +kind: KubeNetworkConfig +podSubnets: + {{- toYaml .Values.podSubnets | nindent 2 }} +serviceSubnets: + {{- toYaml .Values.serviceSubnets | nindent 2 }} +{{- if eq .MachineType "controlplane" }} +{{- with .Values.extraControllerManagerArgs }} +--- +apiVersion: v1alpha1 +kind: KubeControllerManagerConfig +extraArgs: + {{- include "talm.render.component_args" (dict "args" . "field" "extraControllerManagerArgs" "indent" 2) }} +{{- end }} +{{- with .Values.extraSchedulerArgs }} +--- +apiVersion: v1alpha1 +kind: KubeSchedulerConfig +extraArgs: + {{- include "talm.render.component_args" (dict "args" . "field" "extraSchedulerArgs" "indent" 2) }} +{{- end }} +{{- include "talm.guard.apiserver_args_kubedocs" . }} +{{- if or .Values.extraApiServerArgs .Values.certSANs }} +--- +apiVersion: v1alpha1 +kind: KubeAPIServerConfig +{{- with .Values.extraApiServerArgs }} +extraArgs: + {{- include "talm.render.component_args" (dict "args" . "field" "extraApiServerArgs" "indent" 2) }} +{{- end }} +{{- with .Values.certSANs }} +certExtraSANs: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} + {{- /* Shared cluster section */ -}} {{- define "talos.config.cluster" }} @@ -94,9 +152,9 @@ cluster: {{- toYaml .Values.podSubnets | nindent 6 }} serviceSubnets: {{- toYaml .Values.serviceSubnets | nindent 6 }} - clusterName: {{ include "talm.validate.dns1123subdomain" (dict "value" (.Values.clusterName | default .Chart.Name) "field" "clusterName") | quote }} + clusterName: {{ include "talos.config.cluster_name" . }} controlPlane: - endpoint: {{ required "values.yaml: `endpoint` must be set to the cluster control-plane URL (e.g. https://:6443). This field is cluster-wide: every node's kubelet and kube-proxy dials it, so it cannot be auto-derived from the current node's IP -- `talm template` runs once per node and has no way to reconcile per-node IPs into a single shared endpoint. For multi-node setups use a VIP or an external load balancer; for single-node clusters the node's routable IP works." .Values.endpoint | quote }} + endpoint: {{ include "talos.config.endpoint" . }} {{- if eq .MachineType "controlplane" }} {{- with .Values.extraControllerManagerArgs }} controllerManager: @@ -124,23 +182,7 @@ cluster: {{- end }} etcd: advertisedSubnets: - {{- if .Values.advertisedSubnets }} - {{- toYaml .Values.advertisedSubnets | nindent 6 }} - {{- else }} - {{- /* Fall back to the subnet of the node's default-gateway-bearing - link; cidrNetwork masks host bits to emit canonical network - form. Dedupe handled the same way as validSubnets above. - Empty discovery already errored via validSubnets' required() - guard, so we reach this block only when at least one address - was resolved. */ -}} - {{- $subnets := list }} - {{- range fromJsonArray (include "talm.discovered.default_addresses_by_gateway" .) }} - {{- $subnets = append $subnets (. | cidrNetwork) }} - {{- end }} - {{- range uniq $subnets }} - - {{ . }} - {{- end }} - {{- end }} + {{- include "talm.advertised_subnets" (dict "root" . "indent" 6) }} {{- end }} {{- end }} @@ -291,7 +333,11 @@ cluster: {{- define "talos.config.multidoc" }} {{- include "talos.config.machine.common" . }} +{{- if include "talm.contract.kubedocs" . }} +{{- include "talos.config.cluster.kubedocs" . }} +{{- else }} {{- include "talos.config.cluster" . }} +{{- end }} {{- include "talm.config.registries.multidoc" . }} {{- include "talos.config.network.multidoc" . }} {{- end }} diff --git a/charts/generic/values.yaml b/charts/generic/values.yaml index 12bd405f..a66d8db8 100644 --- a/charts/generic/values.yaml +++ b/charts/generic/values.yaml @@ -71,11 +71,12 @@ podSubnets: serviceSubnets: - 10.96.0.0/16 -# Optional override for machine.kubelet.nodeIP.validSubnets and -# cluster.etcd.advertisedSubnets. When left empty the chart derives -# the value from the node's IPv4-default-gateway-bearing link at -# render time (via talm.discovered.default_addresses_by_gateway), so -# the generated machine config matches the node's actual network +# Optional override for the kubelet node IP subnets +# (machine.kubelet.nodeIP.validSubnets; KubeNodeConfig nodeIP.validSubnets +# from Talos v1.14) and cluster.etcd.advertisedSubnets. When left empty +# the chart derives the value from the node's IPv4-default-gateway-bearing +# link at render time (via talm.discovered.default_addresses_by_gateway), +# so the generated machine config matches the node's actual network # without any values.yaml edit. Set this only when you want to pin a # specific subnet — typically for multi-homed nodes where the # IPv4-default-gateway link is not the subnet you want kubelet/etcd @@ -103,11 +104,15 @@ certSANs: [] # - bar=baz extraKernelModules: [] -# Kubelet command-line args, rendered as machine.kubelet.extraConfig. -# Talos expects string values. Example: +# Kubelet configuration (KubeletConfiguration fields, not command-line +# flags), rendered as machine.kubelet.extraConfig +# (KubeletConfig config from Talos v1.14). +# Keys and values follow KubeletConfiguration: maxPods is a number, +# feature gates go under featureGates. Example: # extraKubeletExtraArgs: -# maxPods: "256" -# feature-gates: "NodeSwap=true" +# maxPods: 256 +# featureGates: +# NodeSwap: true extraKubeletExtraArgs: {} # Kernel sysctls, rendered as machine.sysctls. Values must be YAML diff --git a/charts/talm/templates/_helpers.tpl b/charts/talm/templates/_helpers.tpl index 59f0122a..70cb1eab 100644 --- a/charts/talm/templates/_helpers.tpl +++ b/charts/talm/templates/_helpers.tpl @@ -360,9 +360,11 @@ busPath: {{ .spec.busPath }} are the only thing standing between a values typo and a component that either fails to start or starts with a nonsense flag, and six copies of them would drift. Expects a dict with "args" and "field" - (the values.yaml key name, used in messages). */ -}} + (the values.yaml key name, used in messages), and optionally "indent" + (default 6, the v1alpha1 cluster..extraArgs depth). */ -}} {{- define "talm.render.component_args" }} {{- $field := .field }} +{{- $indent := .indent | default 6 | int }} {{- range $k, $v := .args }} {{- if kindIs "invalid" $v }} {{- fail (printf "values.yaml: %s.%s has no value. A bare `key:` renders as the literal string \"\" onto the component command line; give it a value or drop the key." $field $k) }} @@ -372,7 +374,52 @@ busPath: {{ .spec.busPath }} {{- if or (kindIs "map" $v) (kindIs "slice" $v) }} {{- fail (printf "values.yaml: %s.%s is a %s, but a component flag takes a single scalar value. Write it as the string the component expects." $field $k (kindOf $v)) }} {{- end }} - {{ $k }}: {{ $v | toString | quote }} +{{- printf "%s: %s" $k ($v | toString | quote) | nindent $indent }} +{{- end }} +{{- end }} + +{{- /* Non-empty from contract v1.14, where Talos keeps the Kubernetes + settings in typed documents (KubeClusterConfig, KubeletConfig, + KubeNodeConfig, ...) and refuses the v1alpha1 fields next to them. */ -}} +{{- define "talm.contract.kubedocs" -}} +{{- if and .TalosVersion (not (semverCompare "<1.14.0-0" .TalosVersion)) }}true{{ end }} +{{- end }} + +{{- /* From contract v1.14 Talos refuses these kube-apiserver flags in + KubeAPIServerConfig and takes them only as typed documents; fail at + render rather than at apply. */ -}} +{{- define "talm.guard.apiserver_args_kubedocs" }} +{{- range $k, $_ := .Values.extraApiServerArgs }} +{{- range list "oidc-" "anonymous-auth" "authentication-config" "authorization-config" "authorization-mode" "authorization-webhook-" }} +{{- if hasPrefix . $k }} +{{- fail (printf "values.yaml: extraApiServerArgs.%s is not accepted on Talos v1.14 and later, which configures kube-apiserver authentication and authorization through the KubeAuthenticationConfig and KubeAuthorizerConfig documents. Drop it from extraApiServerArgs and add that document as a per-node patch, or pin templateOptions.talosVersion to v1.13 or lower." $k) }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} + +{{- /* advertisedSubnets as YAML list items at .indent, falling back to the + subnet of the node's default-gateway-bearing link. cidrNetwork masks + host bits so the list holds the canonical network form + (192.168.201.0/24, not 192.168.201.10/24); uniq drops the duplicate a + secondary address in the same subnet would add. Expects a dict with + "root" (the chart context) and "indent". */ -}} +{{- define "talm.advertised_subnets" }} +{{- $root := .root }} +{{- if $root.Values.advertisedSubnets }} +{{- toYaml $root.Values.advertisedSubnets | nindent .indent }} +{{- else }} +{{- $addrs := fromJsonArray (include "talm.discovered.default_addresses_by_gateway" $root) }} +{{- if not $addrs }} +{{- fail "values.yaml: `advertisedSubnets` was left empty and talm could not derive a default from discovery. No default-gateway-bearing link was found on the node. This field is a cluster-wide subnet selector fed to kubelet and etcd; `talm template` is invoked once per node and cannot merge per-node values into one cluster value. Either set advertisedSubnets explicitly in values.yaml, or ensure the node has a default route before running `talm template`." }} +{{- end }} +{{- $subnets := list }} +{{- range $addrs }} +{{- $subnets = append $subnets (. | cidrNetwork) }} +{{- end }} +{{- range uniq $subnets }} +{{- printf "- %s" . | nindent $.indent }} +{{- end }} {{- end }} {{- end }} diff --git a/docs/configuration/node-files.md b/docs/configuration/node-files.md index e661142e..ef07f144 100644 --- a/docs/configuration/node-files.md +++ b/docs/configuration/node-files.md @@ -73,7 +73,7 @@ A node file can carry Talos config below its modeline (for example, a custom `ho ## Idempotent applies -Repeated `talm apply` runs against an already-configured node do not duplicate entries. Before the strategic merge runs, the engine prunes from the body every primitive-list entry the rendered template already carries (e.g. certSANs, nameservers, validSubnets). For object arrays the upstream patcher merges by identity (machine.network.interfaces by `interface:` or `deviceSelector:`, vlans by `vlanId:`, apiServer admissionControl by `name:`), the prune descends into matched pairs and dedupes the inner primitive lists too — so re-applying after `talm template -I` does not double interface addresses, vlan addresses, or admission-control exemption namespaces. For object arrays without an upstream identity merge (extraVolumes, kernel.modules, wireguard.peers, ...), body items that deep-equal a rendered counterpart are dropped, covering the dominant full-restate case. Fields tagged `merge:"replace"` upstream are passed through verbatim — pruning them would let the upstream replace silently drop the rendered entries on a partial edit. This covers v1alpha1 root paths `cluster.network.podSubnets`, `cluster.network.serviceSubnets`, `cluster.apiServer.auditPolicy`, and the typed `NetworkRuleConfig` paths `ingress` and `portSelector.ports`. +Repeated `talm apply` runs against an already-configured node do not duplicate entries. Before the strategic merge runs, the engine prunes from the body every primitive-list entry the rendered template already carries (e.g. certSANs, nameservers, validSubnets). For object arrays the upstream patcher merges by identity (machine.network.interfaces by `interface:` or `deviceSelector:`, vlans by `vlanId:`, apiServer admissionControl by `name:`), the prune descends into matched pairs and dedupes the inner primitive lists too — so re-applying after `talm template -I` does not double interface addresses, vlan addresses, or admission-control exemption namespaces. For object arrays without an upstream identity merge (extraVolumes, kernel.modules, wireguard.peers, ...), body items that deep-equal a rendered counterpart are dropped, covering the dominant full-restate case. Fields tagged `merge:"replace"` upstream are passed through verbatim — pruning them would let the upstream replace silently drop the rendered entries on a partial edit. This covers v1alpha1 root paths `cluster.network.podSubnets`, `cluster.network.serviceSubnets`, `cluster.apiServer.auditPolicy`, the typed `NetworkRuleConfig` paths `ingress` and `portSelector.ports`, `advertise`, `importRoutes` and `neighbors` of `BGPInstanceConfig`, `overrides` of `CRIBaseRuntimeSpecConfig`, and from Talos v1.14 `podSubnets` and `serviceSubnets` of `KubeNetworkConfig`, `pod` of `KubeStaticPodConfig` and `configuration` of `KubeAuthenticationConfig`, `KubeAuditPolicyConfig` and `KubeCredentialProviderConfig`. The typed paths are matched per kind: `configuration` of `KubeAdmissionControlConfig` merges and is pruned as usual. Component `extraArgs` (kubelet, apiserver, controller-manager, scheduler, proxy, etcd, in v1alpha1 and in the typed documents) merge key by key and replace the whole value of each key the body sets, so the prune drops only keys the body restates exactly; a list-valued flag such as `feature-gates` edited in the node file keeps the rendered entries the body lists. ## What `talm template` does not do diff --git a/docs/configuration/talos-versions.md b/docs/configuration/talos-versions.md index 761cfde9..c49ca153 100644 --- a/docs/configuration/talos-versions.md +++ b/docs/configuration/talos-versions.md @@ -6,20 +6,22 @@ Leaving `templateOptions.talosVersion` empty means the version of Talos the `talm` binary was built against. It is not frozen at a release: upgrade talm, and an unpinned project follows it to whatever Talos that build carries. -That is why the shipped presets pin it. The charts write Kubernetes settings as v1alpha1 fields (`machine.kubelet`, `machine.nodeLabels`), and from v1.14 Talos keeps each of those in a document of its own and rejects a config that carries both shapes. Rendering an unpinned project against v1.14 therefore produces a config the node refuses, with `kubelet config is already set in v1alpha1 config (.machine.kubelet)` and siblings. Network settings made the same move earlier, in v1.12. +That is why the shipped presets pin it. The charts choose the config shape from the pinned version: from v1.14 Talos keeps the Kubernetes settings in documents of their own and rejects a config that also carries the v1alpha1 fields, and the charts only know to emit those documents when the version says so. An unpinned project renders the v1alpha1 shape against whatever contract the binary carries, and on v1.14 the node would refuse it with `kubelet config is already set in v1alpha1 config (.machine.kubelet)` and siblings. Network settings made the same move earlier, in v1.12. Rather than emit that config, the render stops. With both keys empty the Kubernetes version is reported first, since the bundle cannot be serialized without one; pin it and the contract conflict is reported next: ```text rendered config mixes v1alpha1 fields with the documents that superseded them: ... -hint: templateOptions.talosVersion is unset, so the render targets the Talos version talm was built from. - Pin it to v1.13 or lower in Chart.yaml; above that contract Talos keeps these settings in documents of their own, - which the charts do not emit. +hint: templateOptions.talosVersion is unset, so the render targets the Talos version talm was built from; above v1.13 + Talos keeps these settings in documents of their own, and the charts emit them only for a pinned version. Pin it + in Chart.yaml to the Talos version the nodes run, or to v1.13 or lower to keep the v1alpha1 shape. ``` -A chart can also fail earlier than either guard: the cozystack preset deletes a default node label with `$patch: delete`, and on a contract where `machine.nodeLabels` has moved into `KubeNodeConfig` that path no longer exists, so patching stops with `failed to delete path ...: lookup failed` before any version check runs. The preset happens to pin `v1.12`, which predates that move and sidesteps it. +A project pinned at v1.14 or newer can hit the same error from its own templates: templates vendored by an earlier talm still write the v1alpha1 fields. The hint then points at `talm init --update --preset `, which re-syncs them; it also offers `Chart.yaml` and `values.yaml`, so review those changes before accepting them (see [Keeping charts in sync](../operations/upgrading.md)). -Pin `templateOptions.talosVersion` in `Chart.yaml`. The rule below still holds — the contract must not be newer than the Talos running on the node — and until the charts emit the typed documents there is a second bound on top of it: keep the pin at or below `v1.13`. Nodes running v1.14 are served by a v1.13 contract, which is the supported direction; the reverse is not. Projects created from the shipped presets already carry a pin. +`talm apply` reports the conflict for a node file rendered before the project moved to v1.14, with a hint of its own: the file still carries `machine.kubelet`, `cluster.clusterName` and the rest, and laid over the v1.14 render they would reach the node as the mix it refuses. `talm template` alone does not catch it, because it renders from the templates and does not read the node file's body. `talm template -I` rewrites the file from the templates and keeps only its leading comments, so save any hand-written body first (per-node `LinkConfig` overlays, pinned settings), re-render, then put the saved settings back, moving v1alpha1 Kubernetes fields into the matching document. + +Pin `templateOptions.talosVersion` in `Chart.yaml`, and keep it no newer than the Talos running on the node (see [Version compatibility](#version-compatibility)). Projects created from the shipped presets already carry a pin. ## The Kubernetes version @@ -31,7 +33,7 @@ From v1.14 the contract takes that choice away. The Kubernetes settings live in Pin it in `Chart.yaml` when you want to choose the version yourself, which is what both shipped presets do. A pinned version reaches every component, kubelet included, so they never split across releases. -The presets do not agree on a value. Generic pins `v1.36.2`, the default of the Talos line it targets, so a fresh project lands on what that line ships and a re-sync does not move the control plane. Cozystack pins `v1.34.3` against an older Talos line. Either way the pin is a starting point: set the key to the version your own cluster runs rather than inheriting the preset's. +The presets do not agree on a value. Generic pins `v1.37.0`, the default of the Talos v1.14 line it targets, so a fresh project lands on what that line ships. Cozystack pins `v1.36.5`. Either way the pin is a starting point: set the key to the version your own cluster runs rather than inheriting the preset's. A project created before those pins existed carries empty values for this key and for `talosVersion`. On the first render after upgrading talm it stops with an error naming the key to set; add both pins to its `Chart.yaml` and it renders again. @@ -41,7 +43,25 @@ Selected via `Chart.yaml` (`templateOptions.talosVersion`) or `--talos-version`: - **Talos < v1.12** — the machine config is a single v1alpha1 document with `machine.network` and `machine.registries` sections (the shape shown in [Node files](node-files.md)). Standalone documents that older Talos already accepts, such as the cozystack preset's `ExtensionServiceConfig`, follow it. - **Talos >= v1.12** — multi-document format with separate typed documents instead of the deprecated monolithic network and registry fields. -- **Talos >= v1.14** — a second round of the same move, this time for the Kubernetes settings: `machine.kubelet`, `machine.nodeLabels` and the control-plane components each get a document of their own, and a config carrying both shapes is rejected. The shipped charts still write the v1alpha1 fields, which is why the presets pin below this. +- **Talos >= v1.14** — a second round of the same move, this time for the Kubernetes settings: `machine.kubelet`, `machine.nodeLabels` and the control-plane components each get a document of their own, and a config carrying both shapes is rejected. The charts emit those documents, described below. + +## Kubernetes documents from v1.14 + +On a v1.14 or newer contract the charts write the Kubernetes settings as typed documents, and only `cluster.etcd` stays in v1alpha1, since it has no typed counterpart: + +| Setting | Document | +| --- | --- | +| `clusterName`, `endpoint` | `KubeClusterConfig` | +| `advertisedSubnets` for the kubelet node IP | `KubeNodeConfig` (`nodeIP.validSubnets`) | +| `extraKubeletExtraArgs` | `KubeletConfig` (`config`) | +| `podSubnets`, `serviceSubnets`, `clusterDomain` (cozystack) | `KubeNetworkConfig` | +| `extraApiServerArgs`, `certSANs` | `KubeAPIServerConfig` (`extraArgs`, `certExtraSANs`) | +| `extraControllerManagerArgs` | `KubeControllerManagerConfig` | +| `extraSchedulerArgs` | `KubeSchedulerConfig` | + +These documents carry only what the chart sets. The config Talos generates already contains every one of them, with the component images, the control-plane node label and taint, and the other defaults, and talm merges the chart's documents into those rather than replacing them, the same way a `talosctl` patch would. A delete directive works the same way: the cozystack preset removes the flannel and discovery-service documents, the control-plane taint and the external-load-balancer exclusion label, and sets `KubeProxyConfig` to `enabled: false`, which is what it turned off on earlier contracts. Every other document kind the chart emits (`HostnameConfig`, `LinkConfig` and so on) still replaces the generated one of the same name. A default render keeps the chart's documents as written in the node file; apply merges them again. + +kube-apiserver no longer takes authentication or authorization flags on this contract. The cozystack preset's `oidcIssuerUrl` becomes a JWT authenticator in `KubeAuthenticationConfig` with the same audience (`kubernetes`), claims (`preferred_username`, `groups`) and username prefix (`#`) the flags defaulted to, so existing RBAC bindings keep matching. An `extraApiServerArgs` key starting with `oidc-`, `anonymous-auth`, `authentication-config`, `authorization-config`, `authorization-mode` or `authorization-webhook-` fails the render; put the equivalent `KubeAuthenticationConfig` or `KubeAuthorizerConfig` document in a per-node patch instead. ## Documents emitted in multi-doc mode diff --git a/docs/getting-started/init.md b/docs/getting-started/init.md index 42cfe5d3..bf6ffc94 100644 --- a/docs/getting-started/init.md +++ b/docs/getting-started/init.md @@ -32,7 +32,7 @@ To set the Kubernetes control-plane URL at init time, pass `--cluster-endpoint`: talm init -p cozystack -N myawesomecluster --cluster-endpoint https://vip.example.test:6443 ``` -`--cluster-endpoint` writes the URL into `values.yaml::endpoint`, which the chart renders into `cluster.controlPlane.endpoint` of every node's MachineConfig (the URL kubelet and kube-proxy dial). +`--cluster-endpoint` writes the URL into `values.yaml::endpoint`, which the chart renders into every node's config as the URL kubelet and kube-proxy dial: `endpoint` of the `KubeClusterConfig` document on Talos v1.14 and newer (the preset default), `cluster.controlPlane.endpoint` below that. ## `--endpoints` vs `--cluster-endpoint` diff --git a/docs/manual-test-plan.md b/docs/manual-test-plan.md index 0ced2721..f2e74f67 100644 --- a/docs/manual-test-plan.md +++ b/docs/manual-test-plan.md @@ -17,7 +17,7 @@ The apply-time safety gate matrix (Phase 1 / 2A / 2B / 2C, per-case triggers + e The commands in this plan assume `talm` is invokable by bare name. Adjust paths accordingly if you keep the binary elsewhere. -2. Have a reachable Talos cluster (3 controlplane nodes recommended so you can exercise reset / etcd-member-removal without losing quorum). A small OCI / cloud / bare-metal v1.12.x stand is enough. +2. Have a reachable Talos cluster (3 controlplane nodes recommended so you can exercise reset / etcd-member-removal without losing quorum). A small OCI / cloud / bare-metal stand is enough. Both presets pin `talosVersion: "v1.14"`, whose config a node on an older Talos cannot load: run the plan on a v1.14.x stand, or set `talosVersion` (and `kubernetesVersion`) in the project's `Chart.yaml` to what the stand runs right after A1. 3. Work through the sections below in order. Each section lists a command, the expected outcome, and the failure modes to watch for. **Regression anchors** at the bottom of some sections call out specific behaviours to assert against — formulated as forward-looking checks the operator runs, not as a retrospective of past findings. @@ -50,7 +50,7 @@ Watch for: ### A1a. Version pins on a project created before them -Run inside the A1 project (`--preset cozystack`, which pins `talosVersion: "v1.12"`). +Run inside the A1 project (`--preset cozystack`, which pins `talosVersion: "v1.14"` and `kubernetesVersion: "v1.36.5"`). ```bash sed -i'' -e 's|talosVersion: .*|talosVersion: ""|' -e 's|kubernetesVersion: .*|kubernetesVersion: ""|' Chart.yaml @@ -59,7 +59,7 @@ talm template --offline --full -t templates/controlplane.yaml Expected: the render stops with `templateOptions.kubernetesVersion is not set` and a hint naming the key. -Restore `talosVersion: "v1.12"` alone and re-run: the render succeeds and warns on stderr that no component images are pinned. Restore the preset's `kubernetesVersion: "v1.34.3"` too and re-run: no warning, and the output carries `kubelet`, `kube-apiserver`, `kube-controller-manager`, `kube-proxy` and `kube-scheduler` images on that version. +Restore `talosVersion: "v1.14"` alone and re-run: the render still stops with `templateOptions.kubernetesVersion is not set`, because from v1.14 the Kubernetes documents require an image. Set `talosVersion: "v1.13"` with `kubernetesVersion` still empty and re-run: the render succeeds and warns on stderr that no component images are pinned. Restore both preset pins and re-run: no warning, and the output carries `kubelet`, `kube-apiserver`, `kube-controller-manager`, `kube-proxy` and `kube-scheduler` images on `v1.36.5`. `--full` matters here. A default render is a diff against the same bundle, so the images cancel out on both sides and never appear in a node file either way. @@ -330,7 +330,7 @@ talm init --update --preset --force talm template -f nodes/node0.yaml --offline 2>&1 >/dev/null | grep '^WARN:.*preset' && echo "FAIL: preset drift not cleared" || echo "OK: preset drift cleared" ``` -`init --update` rewrites `.talm-preset.lock` to the current baseline, so the warning clears even if you declined individual preset-file diffs. Expected: `OK: preset drift cleared`. Note: `--force` auto-accepts every preset-file diff, overwriting the operator edit made earlier in this scenario — the "edits are preserved via the interactive diff" promise holds only without `--force`; this test stand is disposable, a real project should run the clear step interactively. +`init --update` rewrites `.talm-preset.lock` to the current baseline, so the warning clears even if you declined individual preset-file diffs. Expected: `OK: preset drift cleared`. Note: `--force` accepts every preset file without asking, overwriting the operator edit made earlier in this scenario. Without `--force`, `init --update` asks `Overwrite? [y/N]` per file and shows no diff, so a real project declines its edited files and merges them by hand; this test stand is disposable. Regression anchors: `TestPresetLock_RoundTrip_NoDrift`, `TestCheckPresetDrift_StaleBaseline_Drift`, `TestCheckPresetDrift_OperatorEditedTemplates_NoDrift` (the false-positive guard), `TestCheckPresetDrift_NoLock_NoBaselineSentinel`, and `TestEvaluatePresetDrift` pin every branch above. @@ -379,11 +379,11 @@ Pin both schema renders dropping kernel-managed scopes (`host` / `link` / `nowhe ```bash # Render the legacy v1.11 path first — explicit talosVersion pin: -sed -i.bak 's/^talosVersion: "v1.12"/talosVersion: "v1.11"/' Chart.yaml +sed -i.bak 's/talosVersion: .*/talosVersion: "v1.11"/' Chart.yaml talm template -f nodes/node0.yaml | grep -E "address: 169\.254" && echo "FAIL: link-scoped leaked into v1.11" || echo "OK v1.11" mv Chart.yaml.bak Chart.yaml -# Then render the v1.12 path (default): +# Then render the preset's own pin (v1.12 or later): talm template -f nodes/node0.yaml | grep -E "address: 169\.254" && echo "FAIL: link-scoped leaked into v1.12" || echo "OK v1.12" ``` @@ -400,7 +400,8 @@ The cozystack preset ships hardcoded defaults for kernel modules, sysctls, kubel extraKernelModules: - name: nf_conntrack extraKubeletExtraArgs: - feature-gates: "NodeSwap=true" + featureGates: + NodeSwap: true extraSysctls: net.core.somaxconn: "65535" extraMachineFiles: @@ -413,7 +414,7 @@ extraMachineFiles: ```bash talm template -f nodes/node0.yaml > /tmp/render.yaml yq '.machine.kernel.modules' /tmp/render.yaml -yq '.machine.kubelet.extraConfig' /tmp/render.yaml +yq 'select(.kind == "KubeletConfig") | .config' /tmp/render.yaml yq '.machine.sysctls' /tmp/render.yaml yq '.machine.files[].path' /tmp/render.yaml ``` @@ -421,7 +422,7 @@ yq '.machine.files[].path' /tmp/render.yaml Expected: - `.machine.kernel.modules` lists the built-in seven (`openvswitch`, `drbd` with `usermode_helper=disabled`, `drbd_transport_tcp`, `zfs`, `spl`, `vfio_pci`, `vfio_iommu_type1`) AND `nf_conntrack` — append, never override. -- `.machine.kubelet.extraConfig` carries the built-ins (`cpuManagerPolicy: static`, `maxPods: 512`) AND `feature-gates: NodeSwap=true`. Operator keys MUST NOT collide with built-ins; a collision fails the render at template time. +- The `KubeletConfig` document's `config` (`.machine.kubelet.extraConfig` on a project pinned below v1.14) carries the built-ins (`cpuManagerPolicy: static`, `maxPods: 512`) AND `featureGates: {NodeSwap: true}`. Operator keys MUST NOT collide with built-ins; a collision fails the render at template time. - `.machine.sysctls` carries the built-in `gc_thresh1/2/3`, the always-on DRBD tuning (`net.ipv4.tcp_orphan_retries`, `net.ipv4.tcp_fin_timeout`, `net.core.netdev_max_backlog`, `net.core.netdev_budget`, `net.core.netdev_budget_usecs`), `vm.nr_hugepages` (when set), AND `net.core.somaxconn`. Same rejection-on-collision rule (the DRBD keys are preset-owned too). See B9 for the keepalive triplet and etcd quota. - `.machine.files[].path` lists `/etc/cri/conf.d/20-customization.part`, `/etc/lvm/lvm.conf`, AND `/etc/example/operator.conf`. @@ -450,14 +451,14 @@ for k in kernel sysctls files; do echo "FAIL: $k block leaked into default generic render" fi done -if [ "$(yq '.machine.kubelet.extraConfig' /tmp/render.yaml)" = "null" ]; then - echo "OK: no extraConfig" +if [ -z "$(yq 'select(.kind == "KubeletConfig")' /tmp/render.yaml)" ]; then + echo "OK: no KubeletConfig" else - echo "FAIL: extraConfig leaked" + echo "FAIL: KubeletConfig leaked" fi ``` -Expected: all four prints "OK". Set any single `extra*` key non-empty, re-render — only the matching block appears; the other three stay absent. The `yq ... = "null"` form is exact-match: `yq` emits the literal string `null` on a single line for an absent path, so a leaking empty block (`modules: []`, `extraConfig: {}`) shows up as a non-`null` value and trips the FAIL branch. +Expected: all four prints "OK". Set any single `extra*` key non-empty, re-render — only the matching block appears; the other three stay absent. The `yq ... = "null"` form is exact-match: `yq` emits the literal string `null` on a single line for an absent path, so a leaking empty block (`modules: []`) shows up as a non-`null` value and trips the FAIL branch. Regression anchor: contract tests `TestContract_Machine_Extra*_Generic_NonEmptyEmitsBlock` pin the on-state for each block; `TestContract_Machine_NoCozystackOpinionsOnGeneric` pins the off-state at default values. A regression that emits an empty `modules: []` / `sysctls: {}` / `files: []` block in the default render would fail the latter. @@ -541,10 +542,10 @@ Regression anchor: `TestContract_Machine_TimeServers_RenderWhenSet` / `TestContr ```bash # values.yaml: extraApiServerArgs: { max-requests-inflight: "2000" } -talm template -f nodes/controlplane-0.yaml | yq '.cluster.apiServer.extraArgs' +talm template -f nodes/controlplane-0.yaml | yq 'select(.kind == "KubeAPIServerConfig") | .extraArgs' ``` -Expected: includes `max-requests-inflight: "2000"`. On cozystack the block appears even without `oidcIssuerUrl` (it is hoisted out of the OIDC guard). A key colliding with a preset built-in — controllerManager `bind-address`/`allocate-node-cidrs`/`cluster-cidr`, scheduler `bind-address`, or an apiServer `oidc-*` arg while `oidcIssuerUrl` is set — fails the render with a hinted message. Generic exposes the same three knobs as a plain passthrough with no built-ins to collide with. +Expected: includes `max-requests-inflight: "2000"`. On cozystack it appears whenever `extraApiServerArgs` is set, with or without `oidcIssuerUrl`; with neither, a v1.14 render carries no `extraArgs` in `KubeAPIServerConfig` (OIDC goes to `KubeAuthenticationConfig`), while a render below v1.14 still emits the `apiServer` block. A key colliding with a preset built-in — controllerManager `bind-address`/`allocate-node-cidrs`/`cluster-cidr`, scheduler `bind-address`, or an apiServer `oidc-*` arg while `oidcIssuerUrl` is set — fails the render with a hinted message. Generic exposes the same three knobs as a plain passthrough with no built-ins to collide with. Regression anchor: `TestContract_Cluster_ExtraControlPlaneArgs_Merge_Cozystack`, `TestContract_Cluster_ExtraApiServerArgs_WithoutOIDC_Cozystack`, `TestContract_Cluster_ExtraControllerManagerArgs_Collision_Cozystack`, `TestContract_Cluster_ExtraApiServerArgs_OIDCCollision_Cozystack`, `TestContract_Cluster_ExtraControlPlaneArgs_Generic`. @@ -718,6 +719,41 @@ Numeric and CIDR fields are checked before they reach the node, where the decode Regression anchor: `TestContract_NetworkMultidoc_ExtraLinks_BondTuning_Cozystack`, `TestContract_NetworkMultidoc_ExtraLinks_MtuAndRoutes_Cozystack`, `TestContract_NetworkMultidoc_ExtraLinks_VlanMtuAndRoutes_Cozystack`, `TestContract_NetworkMultidoc_ExtraLinks_RouteNoGateway_Fails_Cozystack`, `TestContract_NetworkMultidoc_ExtraLinks_CollidesWithPreserved_Fails_Cozystack`, `TestContract_NetworkMultidoc_ExtraLinks_MtuOnly_Fails_Cozystack`, `TestContract_NetworkMultidoc_ExtraLinks_AddressIsVIP_Fails_Cozystack`, `TestContract_NetworkMultidoc_ExtraLinksBondSlaveGetsNoOwnDocument`, `TestContract_NetworkMultidoc_ExtraLinksBondSlaveWithAddresses_Fails`, `TestContract_NetworkMultidoc_ExtraLinksBondSlaveAddressesMovedOntoBond`, `TestContract_NetworkMultidoc_ExtraLinksBondSlaveDefaultRouteNotMoved_Fails`, `TestContract_NetworkMultidoc_ExtraLinksBondSlaveScopedRouteIsNotDefault_Fails`, `TestContract_NetworkMultidoc_ExtraLinksBondSlaveStaticRouteNotMoved_Fails`, `TestContract_NetworkMultidoc_ExtraLinksAddressBadPrefix_Fails`, `TestContract_NetworkMultidoc_ExtraLinksBadMTU_Fails`, `TestContract_NetworkMultidoc_ExtraLinksFractionalVlanID_Fails`. +### B11. Kubernetes documents on the v1.14 contract + +Render both presets offline at their default pins (`talosVersion: "v1.14"`): + +```bash +for p in generic cozystack; do + mkdir -p /tmp/talm-v114/$p && cd /tmp/talm-v114/$p + talm init --preset $p --name v114 --cluster-endpoint https://192.0.2.10:6443 + sed -i'' -e 's|^advertisedSubnets: .*|advertisedSubnets: [192.0.2.0/24]|' values.yaml + echo '# talm: nodes=["192.0.2.10"], endpoints=["192.0.2.10"], templates=["templates/controlplane.yaml"]' > nodes/cp.yaml + talm template --offline --file nodes/cp.yaml > /tmp/talm-v114/$p.yaml + talm template --offline --full --file nodes/cp.yaml > /tmp/talm-v114/$p-full.yaml +done +``` + +Expected for both: + +- The render succeeds. `yq '.machine.kubelet, .cluster.clusterName, .cluster.network, .cluster.apiServer' /tmp/talm-v114/$p.yaml` prints only `null`; `.cluster.etcd.advertisedSubnets` is `[192.0.2.0/24]`. +- The node file carries `KubeClusterConfig` (`clusterName: "v114"`, the endpoint), `KubeNodeConfig` (`nodeIP.validSubnets`) and `KubeNetworkConfig`, each with only the fields the chart sets — no `image:` lines. +- The `--full` render has exactly one document per `Kube*` kind and name (`KubeAuthorizerConfig` comes twice, as `node` and `rbac`), and the component documents carry `image:` on the pinned `kubernetesVersion`. + +Expected for cozystack only, on the node file: `KubeNodeConfig` deletes the `node.kubernetes.io/exclude-from-external-load-balancers` label and the `node-role.kubernetes.io/control-plane` taint, and `KubeFlannelCNIConfig` and `DiscoveryServiceConfig` (`name: default`) are delete selectors. In the `--full` render none of those four appear, `KubeProxyConfig` has `enabled: false`, and `KubeletConfig.config` holds `cpuManagerPolicy: static` and `maxPods: 512`. + +OIDC on cozystack: set `oidcIssuerUrl: https://issuer.example.com` and re-render. Expected: no `oidc-` key anywhere; a `KubeAuthenticationConfig` document holds a `jwt` entry with `issuer.url: "https://issuer.example.com"`, `audiences: [kubernetes]`, username claim `preferred_username` with prefix `"https://issuer.example.com#"`, groups claim `groups` with prefix `""`, and the `anonymous` block allowing `/livez`, `/readyz` and `/healthz`. + +Refused flags, either preset: set `extraApiServerArgs: { oidc-issuer-url: x }` and re-render. Expected: the render fails naming `extraApiServerArgs.oidc-issuer-url` and pointing at `KubeAuthenticationConfig` or a pin at v1.13 or lower. The same key renders on a project pinned at v1.13. + +Node files from an earlier contract, on a node running Talos v1.14: in a project pinned at `v1.13`, run `talm template -I --file nodes/.yaml`, then set `talosVersion: "v1.14"` and `kubernetesVersion` to what the cluster runs, and run `talm apply --dry-run -f nodes/.yaml`. Expected: the apply stops before contacting the node's apply API with `rendered config mixes v1alpha1 fields with the documents that superseded them`, naming the node file, and a hint to re-render with `talm template -I`. `talm template --file nodes/.yaml` without `-I` succeeds at this point, since it does not read the node file's body. Run `talm template -I --file nodes/.yaml`: the node file is rewritten in the v1.14 shape and the dry-run passes. + +On a node running Talos v1.14 (any preset): `talm apply --dry-run -f nodes/.yaml`, then apply. Expected: the drift preview lists `Kube*` documents rather than `cluster.*` fields, the node accepts the config, and `talm get kubeletconfig` / `talm get staticpods` show the kubelet and control-plane args from values. On cozystack, `kubectl get ds --namespace kube-system` lists neither `kube-flannel` nor `kube-proxy`, and control-plane nodes carry no `node-role.kubernetes.io/control-plane:NoSchedule` taint. + +Partial edits of replace-tagged typed fields, on a node running Talos v1.14. A list-valued flag needs a rendered list to edit, and `extraApiServerArgs` takes scalars only, so first, on generic, set `extraApiServerArgs: { max-requests-inflight: "400" }` in `values.yaml`, add `feature-gates: [A=true]`, indented two spaces, on the line after the `extraApiServerArgs` include under `kind: KubeAPIServerConfig` in the project's `templates/_helpers.tpl`, and re-render the node file with `talm template -I`. Then, in a v1.14 node file, add a second entry to `podSubnets` of the `KubeNetworkConfig` document (keep the rendered one); on cozystack with `oidcIssuerUrl` set, add a second audience to the `KubeAuthenticationConfig` jwt entry; and in the generic node file change the flag to `feature-gates: [A=true, B=true]`. Run `talm apply --dry-run -f nodes/.yaml`. Expected: the config sent keeps the rendered pod subnet next to the added one, and `KubeAuthenticationConfig` still carries the `anonymous` block with `/livez`, `/readyz` and `/healthz` next to both audiences, and the flag keeps both entries. + +Regression anchor: `TestContract_ShippedChartsRenderValidConfig` loads both presets through the engine on v1.12, v1.13 and v1.14, at defaults and with every Kubernetes knob set, and validates the full render and the applied node file in machinery; `TestContract_ApiServerAuthFlagsRefusedFromV114` pins the refused flags; `TestApplyTemplatesPerNode_RefusesNodeFileWithSupersededFields` pins the apply-time check on an old node file; `TestContract_ChartKubeDocumentMergesIntoBundle`, `TestContract_ChartDeleteSelectorRemovesBundleDocument` and `TestContract_NonFullRenderKeepsChartKubeDocumentsVerbatim` pin the merge; the `KubeNetworkConfig` and `KubeAuthenticationConfig` cases of `TestMergeFileAsPatch`, `TestPruneIdenticalKeys_ReplacePathsAreKeyedByKind` and `TestPruneIdenticalKeys_KeepsReplaceTaggedNetworkAndCRIFields` (the same rule for `BGPInstanceConfig` and `CRIBaseRuntimeSpecConfig` body documents) pin the partial edits. + ## C. Apply (auth path) This section is the smoke-test for the apply pipe itself; the per-gate matrix lives in **Section C-safety** below. @@ -793,6 +829,8 @@ Expected: the drift preview line for `machine.token` reads `machine.token: ***re Regression anchor: rotating any field in the allowlist (`cluster.{secret,token,aescbcEncryptionSecret,secretboxEncryptionSecret}`, `cluster.{ca,aggregatorCA,serviceAccount,etcd.ca}.key`, `cluster.acceptedCAs[].key`, `machine.{token,ca.key}`, `machine.acceptedCAs[].key`) MUST redact. A regression that silently leaks a secret value into stderr is a security-class bug — verify the substring with `grep -F` against the captured output. +On a project pinned at `talosVersion: "v1.14"` the same material lives in typed documents. Regenerate `secrets.yaml` (keep a copy of the original) and run `talm apply --dry-run -f nodes/node0.yaml`. Expected: the `~ KubeAPIServerCAConfig`, `~ KubeAggregatorCAConfig`, `~ KubeServiceAccountConfig`, `~ KubeEtcdEncryptionConfig` and `~ DiscoveryIdentityConfig` entries show `issuingCA.key`, `issuer.privateKey`, `config.resources` and `clusterSecret` as `***redacted (len=N)***`, and no key from either `secrets.yaml` appears anywhere in stderr, the `Config diff:` block included. Restore the original `secrets.yaml` before applying anything. + ### C5a. Drift preview redacts user secret values (encrypted value files) With an encrypted value file in scope (B2a) injecting a secret into a NON-allowlisted field (e.g. a `machine.pods[]` env, or `machine.registries.config..auth.password`): @@ -819,7 +857,9 @@ talm apply --dry-run --show-secrets-in-drift -f nodes/node0.yaml 2>&1 | grep -cF Expected: in the `Config diff:` block, a user secret renders as `value: ***` and a bootstrap `*.key` / `token` / encryption-secret renders as `key: ***` (etc.) by default. `--show-secrets-in-drift` prints the block verbatim (the same flag that governs the structured drift preview governs this surface too). A leak here is a security-class bug: `talm` prints the server-computed diff verbatim, so an unredacted `ModeDetails` would expose CA private keys and user secrets in CI logs. -Regression anchor: the `Config diff:` shows secrets as context lines too (an unchanged `key:` adjacent to a change hunk), not only on `+`/`-` lines — value-based masking covers both. Public material adjacent to a secret slice (an `acceptedCAs[].crt`) is redacted alongside its key by design (the slice is masked whole); `--show-secrets-in-drift` restores it. +Both sides, on a project pinned at `talosVersion: "v1.14"`: keep a copy of `secrets.yaml`, regenerate it so the rendered secrets differ from the ones the node runs, and run `talm apply --dry-run -f nodes/node0.yaml 2>&1`. Expected: none of the `-` lines (the node's current keys) and none of the `+` lines (the regenerated ones) shows key material. The CA and service-account keys are multi-line PEM there, so every line between `-----BEGIN` and `-----END` in the `issuingCA.key` and `issuer.privateKey` blocks reads `***`, while the armor lines stay. If talm cannot read the node's config (a COSI timeout, an RPC error), the dry run prints `warning: the node's side of the dry-run diff is not masked` and the apply itself is unaffected. Restore the original `secrets.yaml` before applying anything. + +Regression anchor: the `Config diff:` shows secrets as context lines too (an unchanged `key:` adjacent to a change hunk), not only on `+`/`-` lines — value-based masking covers both. `TestModeDetails_PrintedDiffMasksSecretsOnBothSides` runs a real Talos config diff between two configs with different secrets through the print path, on v1.13 and v1.14. Public material adjacent to a secret slice (an `acceptedCAs[].crt`) is redacted alongside its key by design (the slice is masked whole); `--show-secrets-in-drift` restores it. ### C6. Drift preview shows secrets with explicit opt-in @@ -1025,7 +1065,7 @@ The Phase 1 walker validates the syntactic shape of net-addr fields in three v1a | Identical desired/on-node | First-run apply after the same render | `0 addition, 0 removal, 0 update, N unchanged.` | | Removed doc | Apply config that drops a previously-emitted doc (e.g. dropping a LinkConfig that was on-node) | `- LinkConfig{name: …}` line | | Added doc | Apply config that adds a fresh doc | `+ LinkConfig{name: …}` line | -| Updated leaf | Change one nested field (e.g. `clusterDomain`) | `~ MachineConfig` plus `cluster.network.dnsDomain: cozy.local -> cozy.example` | +| Updated leaf | Change one nested field (e.g. `clusterDomain`) | `~ KubeNetworkConfig` plus `dnsDomain: cozy.local -> cozy.example` (below v1.14: `~ MachineConfig` plus `cluster.network.dnsDomain: …`) | | Identical inputs include Equal entries | Verified via Diff API; OpEqual entries returned, FilterChanged drops them | — | | Distinguish absent vs null | YAML `extraField: null` added to one side | FieldChange.HasOld=false / HasNew=true; formatter renders `(absent) -> ` | | Stable ordering | Re-run on same inputs | Identical output bytes | @@ -1048,7 +1088,7 @@ The Phase 1 walker validates the syntactic shape of net-addr fields in three v1a #### Secret-bearing field redaction -The drift preview redacts allowlisted paths by default. The opt-out is operator-explicit: `--show-secrets-in-drift`. Allowlist lives in `secretFieldPaths` (`pkg/commands/preflight_apply_safety_redact.go`). +The drift preview redacts allowlisted paths by default. The opt-out is operator-explicit: `--show-secrets-in-drift`. Allowlist lives in `secretFieldPaths` and, per typed document kind, `secretTypedFieldPaths` (`pkg/commands/preflight_apply_safety_redact.go`); `TestDriftPreview_MasksEverySecretTalosRedacts` and `TestModeDetails_CollectsEverySecretTalosRedacts` check both against what Talos itself redacts. | Case | Trigger | Expected | | --- | --- | --- | @@ -1067,12 +1107,12 @@ The drift preview redacts allowlisted paths by default. The opt-out is operator- | Case | Trigger | Expected | | --- | --- | --- | -| Scalar field change | Change `clusterDomain` in values | `cluster.network.dnsDomain: cozy.local -> cozy.example` inline | -| Map field change | Add a key to `machine.nodeLabels` | YAML flow mapping `{role: control-plane, tier: primary}`, NOT `map[role:control-plane tier:primary]` | -| Multi-element slice add-or-remove | Add a SAN to `cluster.apiServer.certSANs` | `cluster.apiServer.certSANs: added [192.0.2.6]` — set-diff form, NOT a full-slice dump | -| Duplicate-cleanup on slice | Remove a duplicate entry from `certSANs` | `cluster.apiServer.certSANs: removed [127.0.0.1]` — multiset semantics surface a single removal | -| Both add and remove | Replace one SAN with another | `cluster.apiServer.certSANs: removed [old.example], added [new.example]` | -| Reorder-only change | Same elements in a different order | `cluster.apiServer.certSANs: reordered (3 element(s))` — explicit signal, NOT silent OpUpdate | +| Scalar field change | Change `clusterDomain` in values | `dnsDomain: cozy.local -> cozy.example` inline under `~ KubeNetworkConfig` | +| Map field change | Add a key to `labels` in a `KubeNodeConfig` body document | YAML flow mapping `{role: control-plane, tier: primary}`, NOT `map[role:control-plane tier:primary]` | +| Multi-element slice add-or-remove | Add a SAN via `certSANs` in values | `certExtraSANs: added [192.0.2.6]` — set-diff form, NOT a full-slice dump | +| Duplicate-cleanup on slice | Remove a duplicate entry from `certSANs` | `certExtraSANs: removed [127.0.0.1]` — multiset semantics surface a single removal | +| Both add and remove | Replace one SAN with another | `certExtraSANs: removed [old.example], added [new.example]` | +| Reorder-only change | Same elements in a different order | `certExtraSANs: reordered (3 element(s))` — explicit signal, NOT silent OpUpdate | | Slice appearing from absent | Field added that wasn't there before | `(absent) -> [a, b, c]` — flow-style list, NOT `[a b c]` | ### Phase 2B — post-apply state verification @@ -1767,8 +1807,9 @@ Each command should return promptly (sub-second) for read-only paths. ### K1. Preflight version-mismatch warning ```bash -# Bump talosVersion in Chart.yaml to one minor ahead of running. -sed -i 's|talosVersion: "v1.12"|talosVersion: "v1.13"|' Chart.yaml +# Section K needs a stand one minor behind the version it upgrades to; this step assumes v1.12.x. +# Pin talosVersion one minor ahead of the node's running Talos. +sed -i 's|talosVersion: .*|talosVersion: "v1.13"|' Chart.yaml talm apply --dry-run -f nodes/node0.yaml | head -5 ``` @@ -1781,7 +1822,7 @@ Expected: `warning: pre-flight: configured talosVersion=v1.13 is newer than the Run an intentionally-bad cross-vendor upgrade and expect a hint-bearing blocker: ```bash -sed -i 's|cozystack/cozystack/talos:v1.12.6|siderolabs/installer:v1.13.0|' values.yaml +sed -i 's|^image: .*|image: "ghcr.io/siderolabs/installer:v1.13.0"|' values.yaml talm upgrade -f nodes/node0.yaml ``` diff --git a/docs/operations/upgrading.md b/docs/operations/upgrading.md index 88150210..5a73393d 100644 --- a/docs/operations/upgrading.md +++ b/docs/operations/upgrading.md @@ -24,7 +24,7 @@ Re-sync the vendored library with `talm init --update`: talm init --update --preset ``` -This refreshes `charts/talm/` (always) and offers to update the preset templates (interactively, since you may have edited them). Your `values.yaml`, secrets, and node files are left untouched. +This refreshes `charts/talm/` (always) and offers to update the preset files — the templates, `Chart.yaml` and `values.yaml` — one at a time, interactively, since you may have edited them. Secrets and node files are left untouched. To catch drift automatically, a release build compares the vendored `charts/talm/` against its own built-in copy on every config-loading command. The comparison is by **content**, not version number — re-vendoring after a binary bump that did not change the library is a no-op and raises no warning. When the content genuinely differs, talm prints a non-fatal warning to stderr (stdout and the exit code are unchanged): @@ -50,11 +50,11 @@ A project pinned at or below `v1.13` with an empty `kubernetesVersion` renders w Node files never carried those images — a default render diffs against the same bundle, so they cancel out — but the config an apply sends did, because an earlier talm substituted its own built-in version for an empty key. In the drift preview this shows up as five `-` lines for the kubelet and control-plane images, and after the apply those components follow the node's own Talos release. If that release is older than the talm that last wrote the config, it is a downgrade. If it is newer, the same mechanism quietly upgrades the cluster's Kubernetes — the more common direction after a Talos bump. Read the preview before applying, or pin the key. -Pin both keys to what the cluster actually runs, and keep `talosVersion` at or below `v1.13` until the charts emit the typed Kubernetes documents. [Talos versions and output format](../configuration/talos-versions.md) explains what each key selects and why an unpinned value stopped being safe. +Pin both keys to what the cluster actually runs: `talosVersion` no newer than the Talos on the nodes, `kubernetesVersion` the release the control plane runs. [Talos versions and output format](../configuration/talos-versions.md) explains what each key selects and why an unpinned value stopped being safe. -The generic preset gained its pins in this release, so its embedded hash changed: an existing generic-preset project will report preset drift until it is re-synced. (The cozystack preset already carried both pins, so projects on it are unaffected.) That is the mechanism described below, and `talm init --update --preset generic` re-syncs both at once — but read this before running it. +Both presets now pin `talosVersion: "v1.14"`, generic with `kubernetesVersion: "v1.37.0"` and cozystack with `kubernetesVersion: "v1.36.5"` and a Talos v1.14.2 install image, so their embedded hashes changed: an existing project that pins a baseline in `.talm-preset.lock` will report preset drift until it is re-synced. That is the mechanism described below, and `talm init --update --preset ` re-syncs both at once — but read this before running it. -It rewrites `Chart.yaml` from the preset, keeping only `name` and `version`, so the preset's pins replace whatever the project used. The preset pins `kubernetesVersion: "v1.36.2"`, which is what a project running with an empty key was already rendering on the Talos v1.13 line — accepting the re-sync leaves the control plane where it was. That holds only while the two track each other; the moment the preset's pin and the version the project actually runs diverge, accepting the preset's value moves the control plane, and a move **down** is not something Kubernetes supports. Set the key to the version your cluster runs rather than keeping whatever the re-sync wrote, and copy any other settings back too. The command also refuses to run non-interactively without `--force`. +It rewrites `Chart.yaml` from the preset, keeping only the project's `name` (`version` becomes the talm binary's own), so the preset's pins replace whatever the project used. A v1.14 contract is only valid for nodes already running Talos v1.14 or newer, and a different `kubernetesVersion` moves the control plane on the next apply, while a move **down** is not something Kubernetes supports. Set both keys back to what your cluster runs rather than keeping whatever the re-sync wrote, and copy any other settings back too. Once the nodes run Talos v1.14 and the project pins it, re-render the node files with `talm template -I`, after saving any hand-written body below the modeline since `-I` rewrites it: files rendered on an earlier contract carry v1alpha1 Kubernetes fields the v1.14 render refuses. The same run offers the preset's `values.yaml` too, and on cozystack its `image` now points at the Talos v1.14.2 build: decline that change, or restore your own `image` and values afterwards, unless the nodes are meant to move to it. Run without a terminal and without `--force`, it stops at the first preset file that differs from the project's copy instead of asking. ## Preset drift @@ -68,7 +68,7 @@ presetHash: A release build then compares the binary's *current* preset hash against that pinned baseline — never against your edited `templates/` — so operator customizations are never reported as drift. When a newer binary ships changed preset defaults, the baseline no longer matches and talm warns: ```text -WARN: project's cozystack preset differs from the copy built into talm ; run `talm init --update --preset cozystack` to pull the new preset defaults (your templates/ edits are preserved via the interactive diff) +WARN: project's cozystack preset differs from the copy built into talm ; run `talm init --update --preset cozystack` to pull the new preset defaults; it asks before it overwrites each preset file whole, so decline the files you have edited and merge their changes by hand ``` -`talm init --update --preset ` shows you an interactive diff of the new preset against your `templates/`, lets you merge what you want, and advances the baseline — which clears the warning even if you decline individual diffs to keep your customizations. `--strict-charts` / `strictCharts: true` escalate this to a hard error exactly as for the library. Projects with no `.talm-preset.lock` (generated before preset pinning) stay silent — there is no baseline to compare — unless strict mode is on, which treats a missing baseline as a blocker. Commit `.talm-preset.lock` so the baseline is shared across your team. +`talm init --update --preset ` asks `File differs from template. Overwrite? [y/N]` for each preset file that differs, shows no diff, and replaces the file whole when you accept. Decline the files you have edited, compare them with the preset by hand (the preset is in `charts//` of the talm source at the binary's version), and merge what you want. The run advances the baseline either way, which clears the warning even if you decline every file. `--force` accepts every file without asking. `--strict-charts` / `strictCharts: true` escalate this to a hard error exactly as for the library. Projects with no `.talm-preset.lock` (generated before preset pinning) stay silent — there is no baseline to compare — unless strict mode is on, which treats a missing baseline as a blocker. Commit `.talm-preset.lock` so the baseline is shared across your team. diff --git a/docs/reference/apply.md b/docs/reference/apply.md index a56acf80..3324dd13 100644 --- a/docs/reference/apply.md +++ b/docs/reference/apply.md @@ -47,7 +47,7 @@ talm apply [flags] --set-json stringArray set JSON values on the command line (can specify multiple or separate values with commas: key1=jsonval1,key2=jsonval2) --set-literal stringArray set a literal STRING value on the command line --set-string stringArray set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2). Nothing is type-converted, so the value reaches the template exactly as typed. - --show-secrets-in-drift show secret-bearing field values verbatim in drift preview / post-apply verify output (default: redacted). Covers both the Talos bootstrap allowlist (cluster.token, cluster.ca.key, machine.token, Wireguard private keys, etc.) and values from encrypted value files (*.encrypted.yaml). Counterpart on template is --show-secrets, which governs the same values in template's stdout render. + --show-secrets-in-drift show secret-bearing field values verbatim in drift preview / post-apply verify output (default: redacted). Covers the fields Talos treats as secret (cluster.token, cluster.ca.key, machine.token, Wireguard private keys, the CA, service-account and etcd encryption keys in their own documents from Talos v1.14, registry credentials, etc.) and values from encrypted value files (*.encrypted.yaml). Counterpart on template is --show-secrets, which governs the same values in template's stdout render. --skip-drift-preview skip the pre-apply diff of on-node vs rendered MachineConfig --skip-post-apply-verify skip the post-apply structural verification of on-node vs sent MachineConfig (default skip until the Talos-mutated field allowlist lands) (default true) --skip-resource-validation skip the pre-apply check that declared host resources (links, disks) exist on the target node diff --git a/pkg/commands/apply.go b/pkg/commands/apply.go index c52ef875..b1e9cc15 100644 --- a/pkg/commands/apply.go +++ b/pkg/commands/apply.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "io" + "maps" "os" "path/filepath" "slices" @@ -486,6 +487,8 @@ func buildApplyClosure() applyFunc { return err } + nodeValues := nodeSecretValues(cosiCtx, cosiMachineConfigReader(c, applyCmdFlags.insecure), os.Stderr) + resp, err := c.ApplyConfiguration(ctx, &machineapi.ApplyConfigurationRequest{ Data: data, Mode: applyCmdFlags.Mode.Mode, @@ -496,7 +499,7 @@ func buildApplyClosure() applyFunc { return errors.Wrap(annotateApplyConfigError(err), "applying new configuration") } - if err := emitApplyResults(resp, data, true); err != nil { + if err := emitApplyResults(resp, data, nodeValues, true); err != nil { return err } @@ -587,6 +590,8 @@ func applyOneFileDirectPatchMode(configFile, withSecretsPath string) error { read := cosiVersionReader(c) + nodeValues := map[string]struct{}{} + for _, node := range targetNodes { nodeCtx := client.WithNode(ctx, node) preflightCheckTalosVersion(nodeCtx, read, applyCmdFlags.talosVersion, os.Stderr) @@ -594,6 +599,8 @@ func applyOneFileDirectPatchMode(configFile, withSecretsPath string) error { if err := runPreApplyGates(nodeCtx, c, result, node, os.Stderr, false); err != nil { return err } + + maps.Copy(nodeValues, nodeSecretValues(nodeCtx, cosiMachineConfigReader(c, applyCmdFlags.insecure), os.Stderr)) } resp, err := c.ApplyConfiguration(ctx, &machineapi.ApplyConfigurationRequest{ @@ -614,7 +621,7 @@ func applyOneFileDirectPatchMode(configFile, withSecretsPath string) error { return errors.Wrap(annotateApplyConfigError(err), "applying new configuration") } - if err := emitApplyResults(resp, result, false); err != nil { + if err := emitApplyResults(resp, result, nodeValues, false); err != nil { return err } @@ -1061,6 +1068,23 @@ func renderMergeAndApply(ctx context.Context, c *client.Client, opts engine.Opti } } + // The render was checked on its own; a node file written on an older + // contract can still bring back the v1alpha1 fields it superseded. The + // engine's hints are about the render, which passed, so they are dropped. + if err := engine.CheckSupersededFields(merged, opts.TalosVersion); err != nil { + files := strings.Join(append([]string{configFile}, sidePatches...), ", ") + + //nolint:wrapcheck // cockroachdb/errors.WithHint at boundary. + return errors.WithHintf( + errors.Newf("laying %s over the rendered config: %s", files, err.Error()), + "one of these files sets a v1alpha1 field next to the document that replaced it. A node file "+ + "rendered on an earlier Talos contract does: save its hand-written body, re-render it with "+ + "`talm template -I --file %s` and put the body back. "+ + "Otherwise move the reported field out of v1alpha1 into that document.", + configFile, + ) + } + return apply(ctx, c, merged) } @@ -1249,7 +1273,7 @@ func init() { applyCmd.Flags().BoolVar(&applyCmdFlags.skipResourceValidation, "skip-resource-validation", false, "skip the pre-apply check that declared host resources (links, disks) exist on the target node") applyCmd.Flags().BoolVar(&applyCmdFlags.skipDriftPreview, "skip-drift-preview", false, "skip the pre-apply diff of on-node vs rendered MachineConfig") applyCmd.Flags().BoolVar(&applyCmdFlags.skipPostApplyVerify, "skip-post-apply-verify", true, "skip the post-apply structural verification of on-node vs sent MachineConfig (default skip until the Talos-mutated field allowlist lands)") - applyCmd.Flags().BoolVar(&applyCmdFlags.showSecretsInDrift, "show-secrets-in-drift", false, "show secret-bearing field values verbatim in drift preview / post-apply verify output (default: redacted). Covers both the Talos bootstrap allowlist (cluster.token, cluster.ca.key, machine.token, Wireguard private keys, etc.) and values from encrypted value files (*.encrypted.yaml). Counterpart on template is --show-secrets, which governs the same values in template's stdout render.") + applyCmd.Flags().BoolVar(&applyCmdFlags.showSecretsInDrift, "show-secrets-in-drift", false, "show secret-bearing field values verbatim in drift preview / post-apply verify output (default: redacted). Covers the fields Talos treats as secret (cluster.token, cluster.ca.key, machine.token, Wireguard private keys, the CA, service-account and etcd encryption keys in their own documents from Talos v1.14, registry credentials, etc.) and values from encrypted value files (*.encrypted.yaml). Counterpart on template is --show-secrets, which governs the same values in template's stdout render.") helpers.AddModeFlags(&applyCmdFlags.Mode, applyCmd) // Shell completion for `talm apply` flags. `--file` returns the diff --git a/pkg/commands/apply_test.go b/pkg/commands/apply_test.go index b854e6be..499c966d 100644 --- a/pkg/commands/apply_test.go +++ b/pkg/commands/apply_test.go @@ -392,6 +392,75 @@ func TestApplyTemplatesPerNode_LoopsOncePerNodeWithSingleNodeContext(t *testing. } } +// A node file rendered on a contract that kept the Kubernetes settings in +// v1alpha1 still carries them after the project moves to v1.14. Laid over the +// v1.14 render, it produces the mix Talos refuses; the apply has to stop with +// talm's hint instead of sending it to the node. +func TestApplyTemplatesPerNode_RefusesNodeFileWithSupersededFields(t *testing.T) { + dir := t.TempDir() + configFile := filepath.Join(dir, "node.yaml") + + body := "# talm: nodes=[\"a\"], templates=[\"templates/worker.yaml\"]\nmachine:\n kubelet:\n extraArgs:\n rotate-server-certificates: \"true\"\n" + if err := os.WriteFile(configFile, []byte(body), 0o644); err != nil { + t.Fatalf("write configFile: %v", err) + } + + render := func(context.Context, *client.Client, engine.Options) ([]byte, error) { + return []byte("version: v1alpha1\nmachine:\n type: worker\n---\napiVersion: v1alpha1\nkind: KubeletConfig\nimage: ghcr.io/siderolabs/kubelet:v1.37.0\n"), nil + } + apply := func(context.Context, *client.Client, []byte) error { + t.Error("a config mixing v1alpha1 kubelet with KubeletConfig was sent to the node") + + return nil + } + + err := applyTemplatesPerNode(engine.Options{TalosVersion: "v1.14"}, configFile, nil, []string{testNodeAddrA}, fakeAuthOpenClient(context.Background()), render, apply) + if err == nil { + t.Fatal("expected the apply to stop on the superseded kubelet field") + } + + if !strings.Contains(err.Error(), "mixes v1alpha1 fields") { + t.Errorf("error does not report the conflict: %v", err) + } + + // The templates rendered cleanly; the node file is what needs re-rendering. + hints := strings.Join(errors.GetAllHints(err), "\n") + if !strings.Contains(hints, "talm template -I") || strings.Contains(hints, "init --update") { + t.Errorf("hint does not point at re-rendering the node file:\n%s", hints) + } +} + +// A node file already in the v1.14 shape restates the typed documents and +// carries no v1alpha1 Kubernetes field, so the check lets the apply through. +func TestApplyTemplatesPerNode_AcceptsNodeFileInTypedShape(t *testing.T) { + dir := t.TempDir() + configFile := filepath.Join(dir, "node.yaml") + + body := "# talm: nodes=[\"a\"], templates=[\"templates/worker.yaml\"]\nmachine:\n type: worker\n---\napiVersion: v1alpha1\nkind: KubeletConfig\nextraArgs:\n rotate-server-certificates: \"true\"\n" + if err := os.WriteFile(configFile, []byte(body), 0o644); err != nil { + t.Fatalf("write configFile: %v", err) + } + + render := func(context.Context, *client.Client, engine.Options) ([]byte, error) { + return []byte("version: v1alpha1\nmachine:\n type: worker\n---\napiVersion: v1alpha1\nkind: KubeletConfig\nimage: ghcr.io/siderolabs/kubelet:v1.37.0\n"), nil + } + + applied := false + apply := func(context.Context, *client.Client, []byte) error { + applied = true + + return nil + } + + if err := applyTemplatesPerNode(engine.Options{TalosVersion: "v1.14"}, configFile, nil, []string{testNodeAddrA}, fakeAuthOpenClient(context.Background()), render, apply); err != nil { + t.Fatalf("applyTemplatesPerNode: %v", err) + } + + if !applied { + t.Error("a node file in the typed shape was not applied") + } +} + // TestApplyTemplatesPerNode_NeverBatchesNodes guarantees no future tweak // can revert applyTemplatesPerNode to batching all nodes into a single // render call. Phrased against this helper directly, not against diff --git a/pkg/commands/chart_drift.go b/pkg/commands/chart_drift.go index 40cad387..647878d6 100644 --- a/pkg/commands/chart_drift.go +++ b/pkg/commands/chart_drift.go @@ -310,7 +310,8 @@ func CheckPresetDrift(rootDir, binaryVersion string) (bool, string, error) { return true, fmt.Sprintf( "project's %s preset differs from the copy built into talm %s; "+ - "run `talm init --update --preset %s` to pull the new preset defaults (your templates/ edits are preserved via the interactive diff)", + "run `talm init --update --preset %s` to pull the new preset defaults; it asks before it overwrites each "+ + "preset file whole, so decline the files you have edited and merge their changes by hand", lock.Preset, binaryVersion, lock.Preset, ), nil } diff --git a/pkg/commands/chart_drift_test.go b/pkg/commands/chart_drift_test.go index bed047a2..be0b1bc6 100644 --- a/pkg/commands/chart_drift_test.go +++ b/pkg/commands/chart_drift_test.go @@ -304,6 +304,14 @@ func TestCheckPresetDrift_StaleBaseline_Drift(t *testing.T) { if !strings.Contains(msg, "talm init --update --preset cozystack") { t.Errorf("drift message must point at `talm init --update --preset cozystack`; got %q", msg) } + // init --update asks to overwrite each preset file whole and shows no + // diff, so the message must not promise that edits survive a merge. + if strings.Contains(msg, "interactive diff") || strings.Contains(msg, "preserved") { + t.Errorf("drift message promises a merge init --update does not do; got %q", msg) + } + if !strings.Contains(msg, "overwrite") { + t.Errorf("drift message must say init --update offers to overwrite files; got %q", msg) + } } // TestCheckPresetDrift_NoLock_NoBaselineSentinel pins the missing-baseline diff --git a/pkg/commands/contract_drift_typed_secret_redaction_test.go b/pkg/commands/contract_drift_typed_secret_redaction_test.go new file mode 100644 index 00000000..b3fb4b3e --- /dev/null +++ b/pkg/commands/contract_drift_typed_secret_redaction_test.go @@ -0,0 +1,432 @@ +package commands + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" + "testing" + + "github.com/cockroachdb/errors" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/config/configdiff" + "github.com/siderolabs/talos/pkg/machinery/config/configloader" + "github.com/siderolabs/talos/pkg/machinery/config/encoder" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "gopkg.in/yaml.v3" + + "github.com/cozystack/talm/pkg/applycheck" + "github.com/cozystack/talm/pkg/engine" +) + +const talosRedactionMarker = "TALM-TEST-REDACTED" + +// typedSecretDocs carries one secret in every typed document kind whose +// Redact method masks something, beyond the ones the generator emits. +const typedSecretDocs = `--- +apiVersion: v1alpha1 +kind: RegistryAuthConfig +name: registry.example.com +username: operator +password: registry-password-value +--- +apiVersion: v1alpha1 +kind: RegistryTLSConfig +name: tls.example.com +clientIdentity: + cert: registry-client-crt-value + key: registry-client-key-value +--- +apiVersion: v1alpha1 +kind: HCloudVIPConfig +name: hcloud-vip +link: eth0 +apiToken: hcloud-token-value +--- +apiVersion: v1alpha1 +kind: UserVolumeConfig +name: data +provisioning: + diskSelector: + match: "!system_disk" +encryption: + provider: luks2 + keys: + - slot: 0 + static: + passphrase: volume-passphrase-value +` + +// v114ControlPlaneWithSecrets renders a v1.14 control-plane config the way +// talm does, with its own freshly generated secrets, plus typedSecretDocs. +func v114ControlPlaneWithSecrets(t *testing.T) []byte { + t.Helper() + + return append(controlPlaneWithSecrets(t, "v1.14"), []byte(typedSecretDocs)...) +} + +// controlPlaneWithSecrets renders a control-plane config on the given +// contract with its own freshly generated secrets. +func controlPlaneWithSecrets(t *testing.T, talosVersion string) []byte { + t.Helper() + + configBundle, err := engine.InitializeConfigBundle(engine.Options{TalosVersion: talosVersion, KubernetesVersion: "v1.34.3"}) + if err != nil { + t.Fatalf("InitializeConfigBundle: %v", err) + } + + out, err := configBundle.Serialize(encoder.CommentsDisabled, machine.TypeControlPlane) + if err != nil { + t.Fatalf("Serialize: %v", err) + } + + return out +} + +// talosSecretLeaves returns every string leaf Talos itself masks in cfgBytes, +// as kind -> dotted path -> value, by diffing the config against its own +// RedactSecrets copy. +func talosSecretLeaves(t *testing.T, cfgBytes []byte) map[string]map[string]string { + t.Helper() + + cfg, err := configloader.NewFromBytes(cfgBytes) + if err != nil { + t.Fatalf("loading config: %v", err) + } + + redactedBytes, err := cfg.RedactSecrets(talosRedactionMarker).EncodeBytes(encoder.WithComments(encoder.CommentsDisabled)) + if err != nil { + t.Fatalf("encoding redacted config: %v", err) + } + + originalBytes, err := cfg.EncodeBytes(encoder.WithComments(encoder.CommentsDisabled)) + if err != nil { + t.Fatalf("encoding config: %v", err) + } + + original, redacted := decodeAll(t, originalBytes), decodeAll(t, redactedBytes) + if len(original) != len(redacted) { + t.Fatalf("redaction changed the document count: %d -> %d", len(original), len(redacted)) + } + + out := map[string]map[string]string{} + + for i := range original { + kind, _ := original[i]["kind"].(string) + if kind == "" { + kind = "MachineConfig" + } + + collectChangedLeaves(original[i], redacted[i], "", func(path, value string) { + if out[kind] == nil { + out[kind] = map[string]string{} + } + + out[kind][path] = value + }) + } + + return out +} + +func decodeAll(t *testing.T, data []byte) []map[string]any { + t.Helper() + + var docs []map[string]any + + dec := yaml.NewDecoder(bytes.NewReader(data)) + + for { + var doc map[string]any + + err := dec.Decode(&doc) + if errors.Is(err, io.EOF) { + return docs + } + + if err != nil { + t.Fatalf("decoding config: %v", err) + } + + if doc != nil { + docs = append(docs, doc) + } + } +} + +func collectChangedLeaves(original, redacted any, path string, add func(path, value string)) { + switch o := original.(type) { + case map[string]any: + r, _ := redacted.(map[string]any) + for key, value := range o { + child := key + if path != "" { + child = path + "." + key + } + + collectChangedLeaves(value, r[key], child, add) + } + case []any: + r, _ := redacted.([]any) + for i, value := range o { + var rv any + if i < len(r) { + rv = r[i] + } + + collectChangedLeaves(value, rv, fmt.Sprintf("%s[%d]", path, i), add) + } + case string: + if r, _ := redacted.(string); r != o { + add(path, o) + } + } +} + +// Every field Talos itself treats as secret has to be masked in the drift +// preview, whatever document carries it. From contract v1.14 the CA keys, +// the service-account key, the etcd encryption keys and the discovery +// secret live in typed documents, and a path list written for v1alpha1 +// printed them in the clear. +func TestDriftPreview_MasksEverySecretTalosRedacts(t *testing.T) { + current := v114ControlPlaneWithSecrets(t) + desired := v114ControlPlaneWithSecrets(t) + + changes, err := applycheck.Diff(current, desired) + if err != nil { + t.Fatalf("Diff: %v", err) + } + + buf := &bytes.Buffer{} + printDriftPreview(buf, "drift:", changes, secretRedactor{}) + preview := buf.String() + + for _, cfg := range [][]byte{current, desired} { + for kind, leaves := range talosSecretLeaves(t, cfg) { + for path, value := range leaves { + if strings.Contains(preview, value) { + t.Errorf("%s %s printed in the clear by the drift preview", kind, path) + } + } + } + } + + // The fixture must actually differ in its generated secrets, or the + // preview would have nothing to leak and the check above proves nothing. + if !strings.Contains(preview, "KubeServiceAccountConfig") { + t.Fatalf("expected the two configs to differ in their service-account keys:\n%s", preview) + } +} + +// The dry-run diff the node returns is opaque text, so talm masks it by the +// secret values the rendered config carries; those have to include every +// secret Talos redacts, typed documents included. +func TestModeDetails_CollectsEverySecretTalosRedacts(t *testing.T) { + cfg := v114ControlPlaneWithSecrets(t) + + collected, err := collectConfigSecretValues(cfg) + if err != nil { + t.Fatalf("collectConfigSecretValues: %v", err) + } + + for kind, leaves := range talosSecretLeaves(t, cfg) { + for path, value := range leaves { + if _, ok := collected[value]; !ok { + t.Errorf("%s %s is not collected for dry-run diff redaction", kind, path) + } + } + } +} + +// The etcd encryption config is a whole Kubernetes EncryptionConfiguration, +// and only its `secret` leaves are secret. Collecting the rest (provider key +// names, the `secrets` resource) would mask those words everywhere in the +// dry-run diff, as Talos itself does not. +func TestModeDetails_EtcdEncryptionCollectsOnlySecretLeaves(t *testing.T) { + values, err := collectConfigSecretValues(v114ControlPlaneWithSecrets(t)) + if err != nil { + t.Fatalf("collectConfigSecretValues: %v", err) + } + + for _, plain := range []string{"secrets", "key2"} { + if _, ok := values[plain]; ok { + t.Errorf("%q is collected as a secret value", plain) + } + } + + printed := redactValuesInText("+ - hostPath: /var/lib/secrets\n+ name: key2-volume\n", values) + if strings.Contains(printed, modeDetailsRedactionSentinel) { + t.Errorf("plain diff lines were masked:\n%s", printed) + } +} + +// The dry-run diff the node returns shows both its own config and the one +// being applied. Every secret Talos redacts must be masked on both sides, as +// printed: from v1.14 the CA and service-account keys are multi-line PEM, +// which the diff splits into one prefixed line per line of the key. +func TestModeDetails_PrintedDiffMasksSecretsOnBothSides(t *testing.T) { + for _, talosVersion := range []string{"v1.13", "v1.14"} { + t.Run(talosVersion, func(t *testing.T) { + onNode := controlPlaneWithSecrets(t, talosVersion) + rendered := controlPlaneWithSecrets(t, talosVersion) + + diff := nodeDiff(t, onNode, rendered) + + nodeValues, err := collectConfigSecretValues(onNode) + if err != nil { + t.Fatalf("collectConfigSecretValues: %v", err) + } + + values, err := modeDetailsSecretValues(rendered, nodeValues) + if err != nil { + t.Fatalf("modeDetailsSecretValues: %v", err) + } + + buf := &bytes.Buffer{} + printApplyResultsRedacted(&machineapi.ApplyConfigurationResponse{ + Messages: []*machineapi.ApplyConfiguration{{ModeDetails: diff}}, + }, values, buf) + printed := buf.String() + + if !strings.Contains(printed, "@@") { + t.Fatalf("the two configs produced no diff to check:\n%s", printed) + } + + for side, cfg := range map[string][]byte{"node": onNode, "rendered": rendered} { + for kind, leaves := range talosSecretLeaves(t, cfg) { + for path, value := range leaves { + if line, leaked := leakedSecretLine(printed, value); leaked { + t.Errorf("%s side: %s %s printed in the dry-run diff (%q)", side, kind, path, line) + } + } + } + } + }) + } +} + +// The node's secrets are collected before the apply, so a node config talm +// cannot read or decode costs the masking of that side, never the apply. A +// dry run says so instead of leaking silently; a maintenance connection has +// no config to read and stays quiet. +func TestNodeSecretValues(t *testing.T) { + origShow, origDryRun := applyCmdFlags.showSecretsInDrift, applyCmdFlags.dryRun + t.Cleanup(func() { applyCmdFlags.showSecretsInDrift, applyCmdFlags.dryRun = origShow, origDryRun }) + + applyCmdFlags.dryRun = true + + onNode := controlPlaneWithSecrets(t, "v1.14") + + reader := func(data []byte, ok bool, err error) machineConfigReader { + return func(context.Context) ([]byte, bool, error) { return data, ok, err } + } + + for _, tc := range []struct { + name string + show bool + read machineConfigReader + wantValues bool + wantWarning bool + }{ + {name: "read", read: reader(onNode, true, nil), wantValues: true}, + {name: "read error", read: reader(nil, false, errors.New("rpc timeout")), wantWarning: true}, + {name: "maintenance connection", read: reader(nil, false, nil)}, + {name: "undecodable config", read: reader([]byte("machine: [unterminated"), true, nil), wantWarning: true}, + {name: "show secrets", show: true, read: func(context.Context) ([]byte, bool, error) { + t.Error("--show-secrets-in-drift must not read the node's config") + + return nil, false, nil + }}, + } { + t.Run(tc.name, func(t *testing.T) { + applyCmdFlags.showSecretsInDrift = tc.show + + buf := &bytes.Buffer{} + values := nodeSecretValues(context.Background(), tc.read, buf) + + if got := len(values) > 0; got != tc.wantValues { + t.Errorf("collected secrets = %v, want %v", got, tc.wantValues) + } + + if got := strings.Contains(buf.String(), "not masked"); got != tc.wantWarning { + t.Errorf("warning printed = %v, want %v: %q", got, tc.wantWarning, buf.String()) + } + }) + } + + // Talos puts the config diff into the response of a dry run only, so a + // real apply has nothing of the node's to mask and skips the read. + t.Run("no read outside a dry run", func(t *testing.T) { + applyCmdFlags.showSecretsInDrift, applyCmdFlags.dryRun = false, false + + buf := &bytes.Buffer{} + values := nodeSecretValues(context.Background(), func(context.Context) ([]byte, bool, error) { + t.Error("a real apply must not read the node's config for redaction") + + return onNode, true, nil + }, buf) + + if values != nil || buf.Len() != 0 { + t.Errorf("a real apply collected %d values and printed %q", len(values), buf.String()) + } + }) +} + +func nodeDiff(t *testing.T, oldBytes, newBytes []byte) string { + t.Helper() + + oldCfg, err := configloader.NewFromBytes(oldBytes) + if err != nil { + t.Fatalf("loading node config: %v", err) + } + + newCfg, err := configloader.NewFromBytes(newBytes) + if err != nil { + t.Fatalf("loading rendered config: %v", err) + } + + diff, err := configdiff.DiffConfigs(oldCfg, newCfg) + if err != nil { + t.Fatalf("DiffConfigs: %v", err) + } + + return diff +} + +// leakedSecretLine reports a line of value that shows up in printed. A +// multi-line value is checked line by line, as the diff prints it; the PEM +// armor lines are not secret. +func leakedSecretLine(printed, value string) (string, bool) { + for _, line := range strings.Split(value, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "-----") { + continue + } + + for _, out := range strings.Split(printed, "\n") { + if strings.TrimSpace(strings.TrimLeft(out, "+- ")) == line || (len(line) >= 16 && strings.Contains(out, line)) { + return line, true + } + } + } + + return "", false +} + +// The same field name is secret on one kind and harmless on another: +// KubeEtcdEncryptionConfig.config holds the encryption keys, +// KubeletConfig.config holds kubelet settings an operator needs to see. +func TestSecretRedactor_TypedPathsAreKeyedByKind(t *testing.T) { + if !(secretRedactor{}).forKind("KubeEtcdEncryptionConfig").redactPath("config.resources") { + t.Error("KubeEtcdEncryptionConfig config.resources is not redacted") + } + + if (secretRedactor{}).forKind("KubeletConfig").redactPath("config.maxPods") { + t.Error("KubeletConfig config.maxPods is redacted") + } + + if (secretRedactor{show: true}).forKind("KubeEtcdEncryptionConfig").redactPath("config.resources") { + t.Error("--show-secrets-in-drift does not reveal typed secrets") + } +} diff --git a/pkg/commands/contract_modedetails_redaction_test.go b/pkg/commands/contract_modedetails_redaction_test.go index 3367477a..ef3d2df0 100644 --- a/pkg/commands/contract_modedetails_redaction_test.go +++ b/pkg/commands/contract_modedetails_redaction_test.go @@ -134,6 +134,32 @@ func TestContract_RedactValuesInText(t *testing.T) { } } +// A multi-line secret, such as a PEM key in a typed document, reaches the diff +// as one prefixed line per line of the key and is masked line by line. A +// base64 line can itself start with `+`, so only one diff marker is dropped; +// the armor lines are not secret and stay. +func TestContract_RedactValuesInText_MultiLineSecret(t *testing.T) { + const key = "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIAp+Ue3pzd6MDqVdJK5WfuOx4abcdefghijklmnopqrstuv\n+k0sbpQwkE50w/5mixhKjerqWCXabcdefghijklmnopqrstuvwxyz01\n-----END EC PRIVATE KEY-----\n" + + text := "@@ -1,6 +1,6 @@\n issuingCA:\n- key: |\n+ key: |\n+ -----BEGIN EC PRIVATE KEY-----\n+ MHcCAQEEIAp+Ue3pzd6MDqVdJK5WfuOx4abcdefghijklmnopqrstuv\n+ +k0sbpQwkE50w/5mixhKjerqWCXabcdefghijklmnopqrstuvwxyz01\n+ -----END EC PRIVATE KEY-----\n" + + got := redactValuesInText(text, secretSetOf(key)) + + for _, leaked := range []string{"MHcCAQEEIAp", "k0sbpQwkE50w"} { + if strings.Contains(got, leaked) { + t.Errorf("a line of the key survived redaction:\n%s", got) + } + } + + if strings.Count(got, "+ "+modeDetailsRedactionSentinel) != 2 { + t.Errorf("each key line must become the sentinel behind its marker and indentation:\n%s", got) + } + + if !strings.Contains(got, "-----BEGIN EC PRIVATE KEY-----") { + t.Errorf("armor lines are not secret and must stay:\n%s", got) + } +} + // TestContract_PrintApplyResultsRedacted pins the end-to-end print path: a // ModeDetails diff carrying a secret is masked when the value is in scope and // printed verbatim when the value set is empty (show-secrets). diff --git a/pkg/commands/preflight_apply_safety.go b/pkg/commands/preflight_apply_safety.go index eb844911..f210b7b4 100644 --- a/pkg/commands/preflight_apply_safety.go +++ b/pkg/commands/preflight_apply_safety.go @@ -313,7 +313,7 @@ func printDriftPreview(w io.Writer, header string, changes []applycheck.Change, for j := range change.Fields { f := &change.Fields[j] - _, _ = fmt.Fprintf(w, " %s\n", formatFieldChangeLine(f, redactor)) + _, _ = fmt.Fprintf(w, " %s\n", formatFieldChangeLine(f, redactor.forKind(change.ID.Kind))) } } diff --git a/pkg/commands/preflight_apply_safety_modedetails.go b/pkg/commands/preflight_apply_safety_modedetails.go index bc9b62ab..49f664ef 100644 --- a/pkg/commands/preflight_apply_safety_modedetails.go +++ b/pkg/commands/preflight_apply_safety_modedetails.go @@ -16,8 +16,10 @@ package commands import ( "bytes" + "context" "fmt" "io" + "maps" "os" "sort" "strings" @@ -55,12 +57,13 @@ var secretLeafNames = map[string]struct{}{ // collectConfigSecretValues walks the rendered (multi-doc) MachineConfig and // returns the set of secret string VALUES it carries: every leaf whose dotted -// path is on the Talos bootstrap allowlist (isSecretPath) plus every Wireguard -// key leaf (secretLeafNames). It exists to redact the server-computed dry-run -// diff (ModeDetails), which talm prints verbatim and which would otherwise leak -// ca.key / token / encryption-secret material by value — the structured drift -// preview redacts these by PATH, but ModeDetails is opaque diff text, so the -// only handle talm has on it is the value itself. +// path is on the Talos bootstrap allowlist or a typed document's secret field +// (isSecretFieldOf) plus every Wireguard key leaf (secretLeafNames). It exists +// to redact the server-computed dry-run diff (ModeDetails), which talm prints +// verbatim and which would otherwise leak ca.key / token / encryption-secret +// material by value — the structured drift preview redacts these by PATH, +// but ModeDetails is opaque diff text, so the only handle talm has on it is +// the value itself. // // Over-collection is bounded and harmless: an acceptedCAs entry contributes its // public crt alongside the key, so a cert rotation renders as the sentinel in @@ -87,7 +90,8 @@ func collectConfigSecretValues(rendered []byte) (map[string]struct{}, error) { return nil, errors.Wrap(err, "decoding rendered config for dry-run diff redaction") } - walkConfigSecretValues(doc, "", out) + kind, _ := doc["kind"].(string) + walkConfigSecretValues(doc, kind, "", out) } return out, nil @@ -98,7 +102,7 @@ func collectConfigSecretValues(rendered []byte) (map[string]struct{}, error) { // name. When a node matches, the whole subtree's string leaves are taken // (collectStringLeaves) so a slice-valued secret path like cluster.acceptedCAs // is covered without descending further. -func walkConfigSecretValues(node any, path string, out map[string]struct{}) { +func walkConfigSecretValues(node any, kind, path string, out map[string]struct{}) { switch typed := node.(type) { case map[string]any: for key, val := range typed { @@ -113,17 +117,53 @@ func walkConfigSecretValues(node any, path string, out map[string]struct{}) { continue } - if isSecretPath(child) { - collectStringLeaves(val, out) + if isSecretFieldOf(kind, child) { + if leafKey, ok := secretTypedLeafKeys[kind]; ok { + collectNamedStringLeaves(val, leafKey, out) + } else { + collectStringLeaves(val, out) + } continue } - walkConfigSecretValues(val, child, out) + walkConfigSecretValues(val, kind, child, out) } case []any: for i, item := range typed { - walkConfigSecretValues(item, fmt.Sprintf("%s[%d]", path, i), out) + walkConfigSecretValues(item, kind, fmt.Sprintf("%s[%d]", path, i), out) + } + } +} + +// secretTypedLeafKeys narrows the values collected under a kind's secret +// field to the leaves with this key. The etcd encryption config is a whole +// EncryptionConfiguration whose provider key names and resource names are +// plain words; masking them by value would cut them out of every diff line. +// Talos redacts only its `secret` leaves. +// +//nolint:gochecknoglobals // static map of secret leaf keys per typed document. +var secretTypedLeafKeys = map[string]string{ + kindKubeEtcdEncryptionConfig: "secret", +} + +// collectNamedStringLeaves adds every string leaf under value whose map key +// is leafKey. +func collectNamedStringLeaves(value any, leafKey string, out map[string]struct{}) { + switch typed := value.(type) { + case map[string]any: + for key, item := range typed { + if s, ok := item.(string); ok && key == leafKey && s != "" { + out[s] = struct{}{} + + continue + } + + collectNamedStringLeaves(item, leafKey, out) + } + case []any: + for _, item := range typed { + collectNamedStringLeaves(item, leafKey, out) } } } @@ -133,18 +173,34 @@ func walkConfigSecretValues(node any, path string, out map[string]struct{}) { // substring of another, the longer match is consumed first and no fragment of // the longer value survives. An empty value set returns text unchanged, which // is exactly the --show-secrets-in-drift behaviour (no values collected). +// +// A multi-line value, such as a PEM key, never appears whole: the diff prints +// each of its lines on a line of its own, behind a `+`/`-` marker and the +// indentation. Its lines are matched one diff line at a time instead. func redactValuesInText(text string, values map[string]struct{}) string { if len(values) == 0 { return text } ordered := make([]string, 0, len(values)) + secretLines := map[string]struct{}{} + for value := range values { - if value != "" { - ordered = append(ordered, value) + if value == "" { + continue + } + + if strings.Contains(value, "\n") { + addSecretLines(secretLines, value) + + continue } + + ordered = append(ordered, value) } + text = redactSecretLines(text, secretLines) + sort.Slice(ordered, func(i, j int) bool { if len(ordered[i]) != len(ordered[j]) { return len(ordered[i]) > len(ordered[j]) @@ -160,6 +216,46 @@ func redactValuesInText(text string, values map[string]struct{}) string { return text } +// addSecretLines adds the lines of a multi-line secret to set. PEM armor +// lines name the key type and are not secret. +func addSecretLines(set map[string]struct{}, value string) { + for line := range strings.SplitSeq(value, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "-----") { + continue + } + + set[line] = struct{}{} + } +} + +// redactSecretLines masks every diff line whose content, after the `+`, `-` +// or space marker and the indentation, is a line of a multi-line secret. +// Exactly one marker character is dropped: base64 lines can start with `+`. +func redactSecretLines(text string, secretLines map[string]struct{}) string { + if len(secretLines) == 0 { + return text + } + + lines := strings.Split(text, "\n") + for i, line := range lines { + body := line + if body != "" && strings.ContainsRune("+- ", rune(body[0])) { + body = body[1:] + } + + content := strings.TrimSpace(body) + if _, secret := secretLines[content]; !secret { + continue + } + + indent := len(body) - len(strings.TrimLeft(body, " \t")) + lines[i] = line[:len(line)-len(body)+indent] + modeDetailsRedactionSentinel + } + + return strings.Join(lines, "\n") +} + // printApplyResultsRedacted mirrors talosctl's helpers.PrintApplyResults // (warnings to stderr, ModeDetails to w) but runs ModeDetails through // redactValuesInText first. ModeDetails is the server-returned dry-run / @@ -178,6 +274,51 @@ func printApplyResultsRedacted(resp *machineapi.ApplyConfigurationResponse, valu } } +// nodeSecretValues reads the config the node runs and collects its secret +// values, for masking the node's side of the dry-run diff. It runs before the +// apply, so a node config talm cannot read or decode costs only that masking. +// A maintenance connection has no config to read and contributes nothing; +// any other failure on a dry run is reported to w, so the unmasked side is +// not a silent leak. +func nodeSecretValues(ctx context.Context, read machineConfigReader, w io.Writer) map[string]struct{} { + // Talos puts the config diff into the response of a dry run only. + if applyCmdFlags.showSecretsInDrift || !applyCmdFlags.dryRun { + return nil + } + + current, ok, err := read(ctx) + if err == nil && !ok { + return nil + } + + var values map[string]struct{} + if err == nil { + values, err = collectConfigSecretValues(current) + } + + if err != nil { + _, _ = fmt.Fprintf(w, "warning: the node's side of the dry-run diff is not masked, its config could not be read: %v\n", err) + + return nil + } + + return values +} + +// modeDetailsSecretValues collects the secret values the node's dry-run diff +// can show: the diff prints the node's own config next to the rendered one, +// so with different secrets on the two sides both sets have to be masked. +func modeDetailsSecretValues(rendered []byte, nodeValues map[string]struct{}) (map[string]struct{}, error) { + values, err := collectConfigSecretValues(rendered) + if err != nil { + return nil, err + } + + maps.Copy(values, nodeValues) + + return values, nil +} + // emitApplyResults is the redacting replacement for the two // helpers.PrintApplyResults call sites on the apply paths. It collects the // secret value set the dry-run diff must hide — Talos bootstrap material from @@ -189,14 +330,18 @@ func printApplyResultsRedacted(resp *machineapi.ApplyConfigurationResponse, valu // path feeds value files into the applied config, so only it can leak a user // secret through ModeDetails; the direct-patch path renders none, so collecting // them there would be pure overhead and would wrongly require a talm.key. -func emitApplyResults(resp *machineapi.ApplyConfigurationResponse, rendered []byte, rendersUserValues bool) error { +// +// nodeValues are the secrets of the config the node runs before the apply, +// collected by nodeSecretValues: the diff prints that config next to the +// rendered one, so its secrets are masked too. +func emitApplyResults(resp *machineapi.ApplyConfigurationResponse, rendered []byte, nodeValues map[string]struct{}, rendersUserValues bool) error { if applyCmdFlags.showSecretsInDrift { printApplyResultsRedacted(resp, nil, os.Stderr) return nil } - values, err := collectConfigSecretValues(rendered) + values, err := modeDetailsSecretValues(rendered, nodeValues) if err != nil { return err } diff --git a/pkg/commands/preflight_apply_safety_redact.go b/pkg/commands/preflight_apply_safety_redact.go index 8fa36551..7f08ddd4 100644 --- a/pkg/commands/preflight_apply_safety_redact.go +++ b/pkg/commands/preflight_apply_safety_redact.go @@ -18,6 +18,7 @@ import ( "fmt" "regexp" "slices" + "strings" ) // secretFieldPaths is the operator-visible allowlist of paths the @@ -81,6 +82,59 @@ var secretFieldPaths = []string{ "peers", } +// secretTypedFieldPaths lists, per typed document kind, the fields its +// Talos Redact method masks. A field name alone cannot carry this: +// `config` is the encryption keys on KubeEtcdEncryptionConfig and plain +// kubelet settings on KubeletConfig. An entry covers the field and +// everything under it, because the differ reports a changed slice by its +// parent path and the dry-run diff walk descends into it. +// +// From contract v1.14 the generator puts the Kubernetes CA keys, the +// service-account key, the etcd encryption keys and the discovery secret +// in these documents instead of v1alpha1. TestDriftPreview_MasksEverySecretTalosRedacts +// checks the table against what Talos redacts in a generated config. +// +//nolint:gochecknoglobals // static allowlist of secret fields per typed document. +var secretTypedFieldPaths = map[string][]string{ + "DiscoveryIdentityConfig": {"clusterSecret"}, + "KubeAPIServerCAConfig": {secretFieldIssuingCAKey}, + "KubeAggregatorCAConfig": {secretFieldIssuingCAKey}, + "KubeServiceAccountConfig": {"issuer.privateKey"}, + kindKubeEtcdEncryptionConfig: {secretFieldEtcdEncryptionConfig}, + "RegistryAuthConfig": {"password", "auth", "identityToken"}, + "RegistryTLSConfig": {"clientIdentity.key"}, + "HCloudVIPConfig": {"apiToken"}, + "SideroLinkConfig": {"apiUrl", "uniqueToken"}, + "VolumeConfig": {secretFieldVolumeEncryptionKeys}, + "UserVolumeConfig": {secretFieldVolumeEncryptionKeys}, + "RawVolumeConfig": {secretFieldVolumeEncryptionKeys}, + "SwapVolumeConfig": {secretFieldVolumeEncryptionKeys}, +} + +const ( + secretFieldIssuingCAKey = "issuingCA.key" + secretFieldVolumeEncryptionKeys = "encryption.keys" + secretFieldEtcdEncryptionConfig = "config" + kindKubeEtcdEncryptionConfig = "KubeEtcdEncryptionConfig" +) + +// isSecretFieldOf reports whether path is secret in a document of the +// given kind: on the kind-independent allowlist, or at or under one of +// the kind's own secret fields. +func isSecretFieldOf(kind, path string) bool { + if isSecretPath(path) { + return true + } + + for _, field := range secretTypedFieldPaths[kind] { + if path == field || strings.HasPrefix(path, field+".") || strings.HasPrefix(path, field+"[") { + return true + } + } + + return false +} + // arrayIndexPattern matches `[N]` segments (one or more digits) so // isSecretPath can normalise paths like `cluster.acceptedCAs[2].key` // down to `cluster.acceptedCAs[].key` before comparing against the @@ -119,12 +173,22 @@ func isSecretPath(path string) bool { type secretRedactor struct { show bool userSecrets map[string]struct{} + // kind is the document the paths belong to, for the secret fields + // that are secret only on a particular typed document. + kind string +} + +// forKind returns the redactor for the fields of one document kind. +func (r secretRedactor) forKind(kind string) secretRedactor { + r.kind = kind + + return r } // redactPath reports whether the whole field line must be masked because its // path is an allowlisted Talos bootstrap secret. func (r secretRedactor) redactPath(path string) bool { - return !r.show && isSecretPath(path) + return !r.show && isSecretFieldOf(r.kind, path) } // redactSide reports whether one side of a field change must be masked because diff --git a/pkg/engine/contract_multidoc_render_test.go b/pkg/engine/contract_multidoc_render_test.go index 51e7daad..02f3d697 100644 --- a/pkg/engine/contract_multidoc_render_test.go +++ b/pkg/engine/contract_multidoc_render_test.go @@ -286,3 +286,159 @@ func TestContract_BundleSerializesV1Alpha1First(t *testing.T) { }) } } + +// From contract v1.14 the bundle generates every Kubernetes document with the +// values only the generator knows: component images pinned to +// kubernetesVersion, the control-plane node label and taint, the kubelet +// seccomp default. A chart writes only what it sets, so its Kubernetes +// documents merge into the bundle's instead of replacing them. +func TestContract_ChartKubeDocumentMergesIntoBundle(t *testing.T) { + patch := `machine: + type: controlplane +--- +apiVersion: v1alpha1 +kind: KubeClusterConfig +clusterName: prod +endpoint: https://192.0.2.10:6443 +--- +apiVersion: v1alpha1 +kind: KubeAPIServerConfig +extraArgs: + oidc-issuer-url: https://issuer.example.com +` + + out, err := applyPatchesAndRenderConfig( + Options{KubernetesVersion: "v1.34.3", TalosVersion: "v1.14", Full: true}, + []string{patch}) + if err != nil { + t.Fatalf("applyPatchesAndRenderConfig: %v", err) + } + + rendered := string(out) + + for _, kind := range []string{"KubeAPIServerConfig", "KubeClusterConfig"} { + if got := strings.Count(rendered, "kind: "+kind+"\n"); got != 1 { + t.Errorf("render carries %d %s documents, want 1\n--- rendered ---\n%s", got, kind, rendered) + } + } + + for _, want := range []string{ + "registry.k8s.io/kube-apiserver:v1.34.3", + "oidc-issuer-url: https://issuer.example.com", + "clusterName: prod", + // The bundle is regenerated from the chart's endpoint, so documents + // derived from it agree with KubeClusterConfig. + "issuerURL: https://192.0.2.10:6443", + } { + if !strings.Contains(rendered, want) { + t.Errorf("render is missing %q\n--- rendered ---\n%s", want, rendered) + } + } + + if _, err := configloader.NewFromBytes(out); err != nil { + t.Errorf("Talos refuses the rendered config: %v", err) + } +} + +// A delete selector only means something to the patcher. Appended verbatim to a +// full render it would leave the bundle's document in place and add a document +// Talos cannot load. +func TestContract_ChartDeleteSelectorRemovesBundleDocument(t *testing.T) { + patch := `machine: + type: controlplane +--- +apiVersion: v1alpha1 +kind: KubeClusterConfig +clusterName: prod +endpoint: https://192.0.2.10:6443 +--- +apiVersion: v1alpha1 +kind: KubeFlannelCNIConfig +$patch: delete +--- +apiVersion: v1alpha1 +kind: DiscoveryServiceConfig +name: default +$patch: delete +` + + out, err := applyPatchesAndRenderConfig( + Options{KubernetesVersion: "v1.34.3", TalosVersion: "v1.14", Full: true}, + []string{patch}) + if err != nil { + t.Fatalf("applyPatchesAndRenderConfig: %v", err) + } + + rendered := string(out) + + for _, unwanted := range []string{"kind: KubeFlannelCNIConfig", "kind: DiscoveryServiceConfig", "$patch"} { + if strings.Contains(rendered, unwanted) { + t.Errorf("render still carries %q\n--- rendered ---\n%s", unwanted, rendered) + } + } + + if _, err := configloader.NewFromBytes(out); err != nil { + t.Errorf("Talos refuses the rendered config: %v", err) + } +} + +// A non-full render is the node file: the chart's Kubernetes documents and +// selectors stay as written, and apply merges them into the bundle again. +func TestContract_NonFullRenderKeepsChartKubeDocumentsVerbatim(t *testing.T) { + patch := `machine: + type: controlplane +--- +apiVersion: v1alpha1 +kind: KubeClusterConfig +clusterName: prod +endpoint: https://192.0.2.10:6443 +--- +apiVersion: v1alpha1 +kind: KubeAPIServerConfig +extraArgs: + oidc-issuer-url: https://issuer.example.com +--- +apiVersion: v1alpha1 +kind: KubeFlannelCNIConfig +$patch: delete +` + + opts := Options{KubernetesVersion: "v1.34.3", TalosVersion: "v1.14"} + + out, err := applyPatchesAndRenderConfig(opts, []string{patch}) + if err != nil { + t.Fatalf("applyPatchesAndRenderConfig: %v", err) + } + + rendered := string(out) + + if !strings.Contains(rendered, "kind: KubeAPIServerConfig\nextraArgs:\n oidc-issuer-url: https://issuer.example.com") { + t.Errorf("the chart's KubeAPIServerConfig did not survive verbatim\n--- rendered ---\n%s", rendered) + } + + if strings.Contains(rendered, "registry.k8s.io/kube-apiserver") { + t.Errorf("the node file picked up the bundle's image\n--- rendered ---\n%s", rendered) + } + + if !strings.Contains(rendered, "kind: KubeFlannelCNIConfig\n$patch: delete") { + t.Errorf("the delete selector was dropped from the node file\n--- rendered ---\n%s", rendered) + } + + configBundle, machineType, err := FullConfigProcess(opts, []string{rendered}) + if err != nil { + t.Fatalf("FullConfigProcess on the node file: %v", err) + } + + applied, err := SerializeConfiguration(configBundle, machineType, opts.TalosVersion, opts.KubernetesVersion) + if err != nil { + t.Fatalf("SerializeConfiguration: %v", err) + } + + if strings.Contains(string(applied), "kind: KubeFlannelCNIConfig") { + t.Errorf("applying the node file kept flannel\n--- applied ---\n%s", applied) + } + + if !strings.Contains(string(applied), "oidc-issuer-url: https://issuer.example.com") { + t.Errorf("applying the node file lost the chart's apiserver args\n--- applied ---\n%s", applied) + } +} diff --git a/pkg/engine/contract_shipped_charts_validate_test.go b/pkg/engine/contract_shipped_charts_validate_test.go new file mode 100644 index 00000000..4ec615d3 --- /dev/null +++ b/pkg/engine/contract_shipped_charts_validate_test.go @@ -0,0 +1,312 @@ +package engine + +import ( + "bytes" + "io" + "maps" + "os" + "path" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/cockroachdb/errors" + "gopkg.in/yaml.v3" + "helm.sh/helm/v4/pkg/chart/common" + "helm.sh/helm/v4/pkg/chart/v2/loader" + + helmEngine "github.com/cozystack/talm/pkg/engine/helm" +) + +// The shipped charts must render a config the node accepts on every contract +// they support, through the whole engine: the chart output is a patch on the +// generated bundle, and only the merged result tells whether Talos takes it. +// From v1.14 that means the Kubernetes settings travel in typed documents and +// none of the v1alpha1 fields they replaced survive. +func TestContract_ShippedChartsRenderValidConfig(t *testing.T) { + charts := []struct { + name string + path string + }{ + {"cozystack", cozystackChartPath}, + {"generic", genericChartPath}, + } + + // Every knob that lands in a Kubernetes document, so each branch of the + // typed rendering goes through machinery validation at least once. + knobs := map[string]any{ + "certSANs": []any{"api.example.com"}, + "extraKubeletExtraArgs": map[string]any{"imageGCHighThresholdPercent": 80}, + "extraApiServerArgs": map[string]any{"max-requests-inflight": "2000"}, + "extraControllerManagerArgs": map[string]any{"concurrent-gc-syncs": "30"}, + "extraSchedulerArgs": map[string]any{"kube-api-qps": "100"}, + "oidcIssuerUrl": "https://issuer.example.com", + } + + for _, chart := range charts { + for _, machineType := range []string{"controlplane", "worker"} { + for _, talosVersion := range []string{"v1.12", "v1.13", "v1.14"} { + for _, values := range []string{"defaults", "knobs"} { + overrides := map[string]any{} + if values == "knobs" { + overrides = knobs + } + + t.Run(chart.name+"/"+machineType+"/"+talosVersion+"/"+values, func(t *testing.T) { + rendered := renderShippedChart(t, chart.path, "templates/"+machineType+".yaml", talosVersion, overrides) + full, applied := renderNodeFileAndApply(t, talosVersion, rendered) + + // The node file restates what the render produced; laying + // it back over the render must change nothing, or every + // apply grows the lists that merge by appending. + if !reflect.DeepEqual(decodeDocuments(t, applied), decodeDocuments(t, full)) { + t.Errorf("applying the node file changes the rendered config\n--- full ---\n%s\n--- applied ---\n%s", full, applied) + } + + for label, out := range map[string][]byte{"full render": full, "applied node file": applied} { + if err := validateRendered(t, out); err != nil { + t.Errorf("%s does not validate: %v", label, err) + } + + if values == "knobs" && machineType == "controlplane" { + for _, want := range []string{"api.example.com", "max-requests-inflight", "concurrent-gc-syncs", "kube-api-qps", "imageGCHighThresholdPercent"} { + if !strings.Contains(string(out), want) { + t.Errorf("%s lost %q", label, want) + } + } + } + + if talosVersion != "v1.14" { + continue + } + + assertNoSupersededV1Alpha1Fields(t, label, out) + + if chart.name == "cozystack" && machineType == "controlplane" { + assertCozystackControlPlaneV114(t, label, string(out)) + + if values == "knobs" { + assertCozystackOIDCAuthenticatorV114(t, label, string(out)) + } + } + } + }) + } + } + } + } +} + +// From v1.14 Talos refuses kube-apiserver authentication and authorization +// flags in KubeAPIServerConfig. The node would reject the apply, so the render +// refuses them first and names the document that replaced them. +func TestContract_ApiServerAuthFlagsRefusedFromV114(t *testing.T) { + for _, chartPath := range []string{cozystackChartPath, genericChartPath} { + for _, flag := range []string{"oidc-issuer-url", "anonymous-auth", "authentication-config", "authorization-config", "authorization-mode", "authorization-webhook-version"} { + t.Run(filepath.Base(chartPath)+"/"+flag, func(t *testing.T) { + err := renderExpectingError(t, chartPath, "v1.14", helmEngineEmptyLookup, map[string]any{ + "endpoint": testEndpoint, + "advertisedSubnets": []any{testAdvertisedSubnet}, + "extraApiServerArgs": map[string]any{flag: "x"}, + }) + if err == nil { + t.Fatalf("extraApiServerArgs.%s rendered on v1.14", flag) + } + + if !strings.Contains(err.Error(), "extraApiServerArgs."+flag) { + t.Errorf("error does not name the flag: %v", err) + } + }) + } + + // Talos denies a fixed list, not the whole authorization- family. + t.Run(filepath.Base(chartPath)+"/authorization-policy-file allowed", func(t *testing.T) { + renderShippedChart(t, chartPath, "templates/controlplane.yaml", "v1.14", map[string]any{ + "extraApiServerArgs": map[string]any{"authorization-policy-file": "/etc/policy.jsonl"}, + }) + }) + } +} + +// The OIDC flags the cozystack preset used to put on the kube-apiserver +// command line, as the JWT authenticator Talos takes from v1.14. A username +// claim other than email gets the "#" prefix the flags defaulted to, +// so existing RBAC bindings keep matching. +func assertCozystackOIDCAuthenticatorV114(t *testing.T, label, out string) { + t.Helper() + + if strings.Contains(out, "oidc-") { + t.Errorf("%s: OIDC is still passed as kube-apiserver flags", label) + } + + for _, want := range []string{ + "kind: KubeAuthenticationConfig", + "url: https://issuer.example.com", + "claim: preferred_username", + "prefix: https://issuer.example.com#", + "claim: groups", + "- path: /healthz", + } { + if !strings.Contains(out, want) { + t.Errorf("%s: OIDC authenticator is missing %q", label, want) + } + } +} + +// On generic, certSANs alone is enough to emit KubeAPIServerConfig: the SANs +// have to reach the kube-apiserver certificate whether or not any extra +// apiserver flag is set. +func TestContract_GenericCertSANsWithoutApiServerArgsV114(t *testing.T) { + rendered := renderShippedChart(t, genericChartPath, "templates/controlplane.yaml", "v1.14", map[string]any{ + "certSANs": []any{"api.example.com"}, + }) + + if !strings.Contains(rendered, "kind: KubeAPIServerConfig\ncertExtraSANs:\n - api.example.com") { + t.Errorf("certSANs did not reach KubeAPIServerConfig without extraApiServerArgs:\n%s", rendered) + } + + full, _ := renderNodeFileAndApply(t, "v1.14", rendered) + if !strings.Contains(string(full), "- api.example.com") { + t.Errorf("the full render lost the SAN:\n%s", full) + } +} + +func renderShippedChart(t *testing.T, chartPath, templateFile, talosVersion string, overrides map[string]any) string { + t.Helper() + + origLookup := helmEngine.LookupFunc + t.Cleanup(func() { helmEngine.LookupFunc = origLookup }) + helmEngine.LookupFunc = simpleNicLookup() + + chrt, err := loader.LoadDir(chartPath) + if err != nil { + t.Fatalf("load chart: %v", err) + } + + values := cloneValues(chrt.Values) + values["endpoint"] = testEndpoint + values["advertisedSubnets"] = []any{testAdvertisedSubnet} + maps.Copy(values, overrides) + + out, err := (&helmEngine.Engine{}).Render(chrt, common.Values{"Values": values, "TalosVersion": talosVersion}) + if err != nil { + t.Fatalf("render: %v", err) + } + + return out[path.Join(chrt.Name(), templateFile)] +} + +// renderNodeFileAndApply returns the full render and what `talm apply` sends +// for the node file a non-full render writes: the templates rendered in full +// with the node file laid over them. +func renderNodeFileAndApply(t *testing.T, talosVersion, rendered string) ([]byte, []byte) { + t.Helper() + + opts := Options{KubernetesVersion: "v1.34.3", TalosVersion: talosVersion} + + nodeFile, err := applyPatchesAndRenderConfig(opts, []string{rendered}) + if err != nil { + t.Fatalf("non-full render: %v", err) + } + + opts.Full = true + + full, err := applyPatchesAndRenderConfig(opts, []string{rendered}) + if err != nil { + t.Fatalf("full render: %v", err) + } + + nodeFilePath := filepath.Join(t.TempDir(), "node.yaml") + if err := os.WriteFile(nodeFilePath, nodeFile, 0o600); err != nil { + t.Fatalf("writing node file: %v", err) + } + + applied, err := MergeFileAsPatch(full, nodeFilePath) + if err != nil { + t.Fatalf("merging the node file: %v", err) + } + + return full, applied +} + +func decodeDocuments(t *testing.T, data []byte) []map[string]any { + t.Helper() + + var docs []map[string]any + + dec := yaml.NewDecoder(bytes.NewReader(data)) + + for { + var doc map[string]any + + err := dec.Decode(&doc) + if errors.Is(err, io.EOF) { + return docs + } + + if err != nil { + t.Fatalf("decoding rendered config: %v", err) + } + + if doc != nil { + docs = append(docs, doc) + } + } +} + +func assertNoSupersededV1Alpha1Fields(t *testing.T, label string, out []byte) { + t.Helper() + + var v1alpha1 struct { + Machine map[string]any `yaml:"machine"` + Cluster map[string]any `yaml:"cluster"` + } + + if err := yaml.Unmarshal(out, &v1alpha1); err != nil { + t.Fatalf("%s: decoding the v1alpha1 document: %v", label, err) + } + + for _, key := range []string{"kubelet", "nodeLabels", "nodeTaints"} { + if _, ok := v1alpha1.Machine[key]; ok { + t.Errorf("%s: v1alpha1 machine.%s is set on the v1.14 contract", label, key) + } + } + + for _, key := range []string{"clusterName", "controlPlane", "network", "apiServer", "controllerManager", "scheduler", "proxy", "discovery", "allowSchedulingOnControlPlanes"} { + if _, ok := v1alpha1.Cluster[key]; ok { + t.Errorf("%s: v1alpha1 cluster.%s is set on the v1.14 contract", label, key) + } + } +} + +// What the cozystack preset turns off or strips on a control plane: Cozystack +// ships its own CNI and kube-proxy replacement, schedules workloads on control +// planes, and lets external load balancers reach them. +func assertCozystackControlPlaneV114(t *testing.T, label, out string) { + t.Helper() + + for _, unwanted := range []string{ + "kind: KubeFlannelCNIConfig", + "kind: DiscoveryServiceConfig", + "node.kubernetes.io/exclude-from-external-load-balancers", + "node-role.kubernetes.io/control-plane: NoSchedule", + "$patch", + } { + if strings.Contains(out, unwanted) { + t.Errorf("%s: cozystack control plane still carries %q", label, unwanted) + } + } + + for _, want := range []string{ + "kind: KubeProxyConfig\nenabled: false", + "cpuManagerPolicy: static", + "bind-address: 0.0.0.0", + "quota-backend-bytes:", + "registry.k8s.io/kube-apiserver:v1.34.3", + } { + if !strings.Contains(out, want) { + t.Errorf("%s: cozystack control plane is missing %q", label, want) + } + } +} diff --git a/pkg/engine/contract_validate_render_test.go b/pkg/engine/contract_validate_render_test.go index 92c3ef10..84b1829e 100644 --- a/pkg/engine/contract_validate_render_test.go +++ b/pkg/engine/contract_validate_render_test.go @@ -480,27 +480,43 @@ kind: HostnameConfig hostname: node0 ` + // The way out depends on why the render mixed the shapes. Unpinned, the + // charts render v1alpha1 whatever the contract, so only a pin helps. + // Pinned above the boundary, the project's templates predate the typed + // documents and need a re-sync. Re-rendering a node file fixes neither: the + // render never reads its body. for _, tc := range []struct { name string talosVersion string wantPinLower bool + wantInHint string }{ - {name: "above the boundary", talosVersion: "v1.14", wantPinLower: true}, + {name: "unpinned", talosVersion: "", wantPinLower: true, wantInHint: "Pin it in Chart.yaml"}, + {name: "above the boundary", talosVersion: "v1.14", wantPinLower: true, wantInHint: "talm init --update"}, {name: "at the boundary", talosVersion: "v1.13", wantPinLower: false}, {name: "below the boundary", talosVersion: "v1.12", wantPinLower: false}, } { t.Run(tc.name, func(t *testing.T) { - err := checkSupersededFields([]byte(rendered), tc.talosVersion) + err := CheckSupersededFields([]byte(rendered), tc.talosVersion) if err == nil { t.Fatalf("talosVersion=%s: expected a conflict", tc.talosVersion) } hints := strings.Join(errors.GetAllHints(err), "\n") - if got := strings.Contains(hints, "Pin it to"); got != tc.wantPinLower { + if got := strings.Contains(hints, "v1.13 or lower"); got != tc.wantPinLower { t.Errorf("talosVersion=%s: hint offers a lower pin = %v, want %v:\n%s", tc.talosVersion, got, tc.wantPinLower, hints) } + + if tc.wantInHint != "" && !strings.Contains(hints, tc.wantInHint) { + t.Errorf("talosVersion=%s: hint does not say %q:\n%s", tc.talosVersion, tc.wantInHint, hints) + } + + if strings.Contains(hints, "template -I") { + t.Errorf("talosVersion=%s: hint sends the operator to re-render a node file the render never reads:\n%s", + tc.talosVersion, hints) + } }) } } diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 62f621ea..8e0de7a7 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -12,6 +12,7 @@ import ( "path/filepath" "reflect" "regexp" + "slices" "strings" "unsafe" @@ -292,7 +293,7 @@ func SerializeConfiguration(configBundle *bundle.Bundle, machineType machine.Typ return nil, err } - if err := checkSupersededFields(serialized, talosVersion); err != nil { + if err := CheckSupersededFields(serialized, talosVersion); err != nil { return nil, err } @@ -823,8 +824,8 @@ func isPatchDeleteDirective(n *yaml.Node) bool { k, v := n.Content[0], n.Content[1] - return k.Kind == yaml.ScalarNode && k.Value == "$patch" && - v.Kind == yaml.ScalarNode && v.Value == "delete" + return k.Kind == yaml.ScalarNode && k.Value == patchDirectiveKey && + v.Kind == yaml.ScalarNode && v.Value == patchDirectiveDelete } // pruneBodyIdentitiesAgainstRendered removes from body every key whose value @@ -955,14 +956,20 @@ func pruneBodyIdentitiesAgainstRendered(body, rendered []byte) ([]byte, bool, er // body is the zero value, in which case rendered survives. Each entry // mirrors a struct field tagged `merge:"replace"` in the upstream // machinery types (collected from -// pkg/machinery/config/types/v1alpha1/v1alpha1_types.go and -// pkg/machinery/config/types/network/rule_config.go): +// pkg/machinery/config/types/v1alpha1/v1alpha1_types.go and the typed +// documents under pkg/machinery/config/types/{network,k8s,cri}): // // - cluster/network/podSubnets — v1alpha1 `PodSubnet []string ... merge:"replace"` // - cluster/network/serviceSubnets — v1alpha1 `ServiceSubnet []string ... merge:"replace"` // - cluster/apiServer/auditPolicy — v1alpha1 `AuditPolicyConfig Unstructured ... merge:"replace"` -// - ingress — typed NetworkRuleConfig `Ingress IngressConfig ... merge:"replace"` -// - portSelector/ports — typed NetworkRuleConfig `Ports PortRanges ... merge:"replace"` +// - NetworkRuleConfig/ingress — `Ingress IngressConfig ... merge:"replace"` +// - NetworkRuleConfig/portSelector/ports — `Ports PortRanges ... merge:"replace"` +// - BGPInstanceConfig/advertise, BGPInstanceConfig/importRoutes, +// BGPInstanceConfig/neighbors +// - CRIBaseRuntimeSpecConfig/overrides +// - KubeNetworkConfig/podSubnets, KubeNetworkConfig/serviceSubnets +// - KubeAuthenticationConfig/configuration, KubeAuditPolicyConfig/configuration, +// KubeCredentialProviderConfig/configuration, KubeStaticPodConfig/pod // // At these paths the prune must NOT subtract rendered-side entries from // body's primitive list, recurse into body's map, or descend into body's @@ -973,13 +980,13 @@ func pruneBodyIdentitiesAgainstRendered(body, rendered []byte) ([]byte, bool, er // original; a partial-edit on auditPolicy that adds a rule loses the // rendered apiVersion/kind/other rules. // -// Paths are walked relative to the document root that owns the field, -// so the typed NetworkRuleConfig entries appear without an apiVersion/ -// kind prefix — pruneBodyIdentitiesAgainstRendered pairs body and -// rendered docs by identity tuple before calling pruneIdenticalKeys, so -// each walk sees only its own document's keys. The bare paths above do -// not collide with anything in the v1alpha1 root (no `ingress` or -// `portSelector` keys exist there), so a flat lookup is sufficient. +// Paths are walked relative to the document that owns the field: +// pruneBodyIdentitiesAgainstRendered pairs body and rendered docs by +// identity tuple before calling pruneIdenticalKeys, so each walk sees +// only its own document's keys. A typed document's paths start with its +// kind, because the same field is replace-tagged on one kind and merged +// on another (KubeAdmissionControlConfig's `configuration` merges); the +// v1alpha1 document has no kind, so its paths start at the root. // // The deep-equal short-circuit at the top of pruneIdenticalKeysAt is // still safe for replace paths: when body byte-equals rendered, deleting @@ -992,8 +999,88 @@ var replaceSemanticPaths = map[string]struct{}{ "cluster/network/podSubnets": {}, "cluster/network/serviceSubnets": {}, "cluster/apiServer/auditPolicy": {}, - "ingress": {}, - "portSelector/ports": {}, + + "NetworkRuleConfig/ingress": {}, + "NetworkRuleConfig/portSelector/ports": {}, + "BGPInstanceConfig/advertise": {}, + "BGPInstanceConfig/importRoutes": {}, + "BGPInstanceConfig/neighbors": {}, + "CRIBaseRuntimeSpecConfig/overrides": {}, + + "KubeNetworkConfig/podSubnets": {}, + "KubeNetworkConfig/serviceSubnets": {}, + "KubeAuthenticationConfig/configuration": {}, + "KubeAuditPolicyConfig/configuration": {}, + "KubeCredentialProviderConfig/configuration": {}, + "KubeStaticPodConfig/pod": {}, +} + +// keyReplaceMapPaths lists the maps Talos merges key by key, replacing the +// whole value of every key the patch sets: the component extraArgs, typed +// meta.Args, whose Merge is a maps.Copy. At these paths the prune drops a +// key only when body restates it exactly; a key whose value differs goes +// to the patcher whole, or a list-valued flag edited in the node file +// would be cut to the operator's delta and replace the rendered entries. +// +//nolint:gochecknoglobals // immutable lookup table consulted by pruneIdenticalKeysAt. +var keyReplaceMapPaths = map[string]struct{}{ + "machine/kubelet/extraArgs": {}, + "cluster/apiServer/extraArgs": {}, + "cluster/controllerManager/extraArgs": {}, + "cluster/proxy/extraArgs": {}, + "cluster/scheduler/extraArgs": {}, + "cluster/etcd/extraArgs": {}, + + "KubeletConfig/extraArgs": {}, + "KubeAPIServerConfig/extraArgs": {}, + "KubeControllerManagerConfig/extraArgs": {}, + "KubeSchedulerConfig/extraArgs": {}, + "KubeProxyConfig/extraArgs": {}, +} + +// pruneAtReplacePath handles body[key] when Talos replaces rather than merges +// at childPath, and reports whether it did. Under `merge:"replace"` the value +// is left whole: any prune leaks rendered-side entries out of the final +// config (see replaceSemanticPaths). Under a per-key replace only exactly +// restated keys go (see keyReplaceMapPaths). +func pruneAtReplacePath(body map[string]any, key string, bodyV, renderedV any, childPath string) bool { + if _, replace := replaceSemanticPaths[childPath]; replace { + return true + } + + if _, perKey := keyReplaceMapPaths[childPath]; perKey { + pruneEqualMapKeys(body, key, bodyV, renderedV) + + return true + } + + return false +} + +// pruneEqualMapKeys drops from body[key] the entries rendered carries with +// the same value, and body[key] itself once nothing is left in it. +func pruneEqualMapKeys(body map[string]any, key string, bodyV, renderedV any) { + bodySub, ok := bodyV.(map[string]any) + if !ok { + return + } + + renderedSub, ok := renderedV.(map[string]any) + if !ok { + return + } + + before := len(bodySub) + + for k, v := range bodySub { + if rv, exists := renderedSub[k]; exists && reflect.DeepEqual(v, rv) { + delete(bodySub, k) + } + } + + if before > 0 && len(bodySub) == 0 { + delete(body, key) + } } // objectArrayMergeKeys lists the identity fields Talos's upstream @@ -1084,7 +1171,10 @@ var objectArrayMergeKeys = map[string][]string{ // threads the YAML path so the object-array descent can look up the // identity key for the current location. func pruneIdenticalKeys(body, rendered map[string]any) { - pruneIdenticalKeysAt(body, rendered, "") + // A typed document's paths are rooted at its kind: the same field name + // can be replace-tagged on one kind and merged on another. + kind, _ := body[k8sKeyKind].(string) + pruneIdenticalKeysAt(body, rendered, kind) } // pruneIdenticalKeysAt is pruneIdenticalKeys's recursive workhorse. @@ -1111,10 +1201,7 @@ func pruneIdenticalKeysAt(body, rendered map[string]any, yamlPath string) { } childPath := joinYAMLPath(yamlPath, key) - if _, replace := replaceSemanticPaths[childPath]; replace { - // Upstream `merge:"replace"` overwrites rendered with body - // verbatim. Any prune at this path leaks rendered-side - // entries out of the final config — see replaceSemanticPaths. + if pruneAtReplacePath(body, key, bodyV, renderedV, childPath) { continue } @@ -1469,6 +1556,10 @@ const ( // k8sKeyKind is the standard Kubernetes/COSI document key // used as part of the (apiVersion, kind, name) identity tuple. k8sKeyKind = "kind" + // patchDirectiveKey and patchDirectiveDelete spell a strategic-merge + // delete directive, `$patch: delete`. + patchDirectiveKey = "$patch" + patchDirectiveDelete = "delete" // k8sAPIVersionV1 is the value of apiVersion for the synthetic // List envelope newLookupFunction emits when wrapping multi-item // COSI lookup results. @@ -1899,6 +1990,39 @@ func extractExtraDocuments(patches []string) ([]string, []string, error) { return talosPatches, extraDocs, nil } +// splitMergedDocuments picks out the chart documents that patch the bundle +// instead of replacing a document of the same identity: every Kube* kind and +// delete selectors. From contract v1.14 the bundle generates the Kubernetes +// documents with values only the generator knows (component images, the +// control-plane label and taint), so a chart writes just what it sets. The +// KubeSpan kinds match the prefix too; the generator emits none of them, so +// for those a merge is the same as a replace. A delete selector is +// meaningless anywhere but in the patcher. Every other kind keeps +// replace semantics: the bundle's HostnameConfig carries `auto: stable`, which +// Talos refuses next to the chart's hostname. +func splitMergedDocuments(extraDocs []string) ([]string, []string, error) { + var merged, replacing []string + + for _, doc := range extraDocs { + var parsed map[string]any + + if err := yaml.Unmarshal([]byte(doc), &parsed); err != nil { + return nil, nil, errors.Wrap(err, "unmarshaling extra document") + } + + kind, _ := parsed[k8sKeyKind].(string) + selector, _ := parsed[patchDirectiveKey].(string) + + if strings.HasPrefix(kind, "Kube") || selector == patchDirectiveDelete { + merged = append(merged, doc) + } else { + replacing = append(replacing, doc) + } + } + + return merged, replacing, nil +} + // applyPatchesAndRenderConfig assembles the final Talos config bytes // for the non-Full template path: split out extra documents, run two // bundle-rebuild passes (TypeUnknown then resolved machine type), @@ -1912,6 +2036,11 @@ func applyPatchesAndRenderConfig(opts Options, configPatches []string) ([]byte, return nil, err } + mergedDocs, replacingDocs, err := splitMergedDocuments(extraDocs) + if err != nil { + return nil, err + } + // Generate options for the configuration based on the provided flags genOptions := []generate.Option{} @@ -1949,7 +2078,7 @@ func applyPatchesAndRenderConfig(opts Options, configPatches []string) ([]byte, return nil, errors.Wrap(err, "creating initial config bundle") } - patches, err := configpatcher.LoadPatches(talosPatches) + patches, err := configpatcher.LoadPatches(append(slices.Clone(talosPatches), mergedDocs...)) if err != nil { if opts.Debug { debugPhase(opts, configPatches, "", "", machine.TypeUnknown) @@ -2061,18 +2190,21 @@ func applyPatchesAndRenderConfig(opts Options, configPatches []string) ([]byte, // // A non-full render returns a patch rather than a config, so what it returns // is deliberately not what gets checked here. - assembled, err := assembleTargetDocuments(configFull, nil, extraDocs) + assembled, err := assembleTargetDocuments(configFull, nil, replacingDocs) if err != nil { return nil, err } - if err := checkSupersededFields(assembled.Bytes(), opts.TalosVersion); err != nil { + if err := CheckSupersededFields(assembled.Bytes(), opts.TalosVersion); err != nil { return nil, err } + // The merged documents are already inside configFull. The node file keeps + // them as the chart wrote them, and apply merges them again. var target []byte if opts.Full { target = configFull + extraDocs = replacingDocs } else { target, err = yamltools.DiffYAMLs(configOrigin, configFull) if err != nil { @@ -2088,7 +2220,7 @@ func applyPatchesAndRenderConfig(opts Options, configPatches []string) ([]byte, return buf.Bytes(), nil } -// checkSupersededFields fails the render when the config sets a v1alpha1 field +// CheckSupersededFields fails the render when the config sets a v1alpha1 field // that the targeted contract has moved into a document of its own. Talos rejects // such a config outright, so catching it here turns an apply-time rejection into // a render-time error naming the way out. @@ -2101,7 +2233,7 @@ func applyPatchesAndRenderConfig(opts Options, configPatches []string) ([]byte, // // Only that class is reported: a render is legitimately incomplete in other ways // (no install disk yet, no endpoint), and those are the operator's to fill in. -func checkSupersededFields(rendered []byte, talosVersion string) error { +func CheckSupersededFields(rendered []byte, talosVersion string) error { cfg, err := configloader.NewFromBytes(rendered) if err != nil { // Not this check's job to decide the config is unloadable; the render @@ -2137,16 +2269,20 @@ func checkSupersededFields(rendered []byte, talosVersion string) error { return nil } + return withSupersededFieldsHint(errors.Wrap( + errors.Join(conflicts...), + "rendered config mixes v1alpha1 fields with the documents that superseded them", + ), talosVersion) +} + +// withSupersededFieldsHint attaches the way out of a v1alpha1/typed-document +// mix, which depends on the contract and on whether the version is pinned. +func withSupersededFieldsHint(conflict error, talosVersion string) error { pinned := talosVersion if pinned == "" { pinned = "unset, so the render targets the Talos version talm was built from" } - conflict := errors.Wrap( - errors.Join(conflicts...), - "rendered config mixes v1alpha1 fields with the documents that superseded them", - ) - // Above the boundary the contract is what moved the settings, so lowering it // resolves the conflict. At or below it the contract is innocent: something // in the charts or the patches wrote both shapes, and telling the operator @@ -2163,13 +2299,33 @@ func checkSupersededFields(rendered []byte, talosVersion string) error { ) } + // The charts pick the shape from the pinned version, so unpinned they write + // v1alpha1 on every contract and only a pin helps. + if talosVersion == "" { + //nolint:wrapcheck // cockroachdb/errors.WithHint at boundary. + return errors.WithHintf( + conflict, + "templateOptions.talosVersion is %s; above %s Talos keeps these settings in documents of their own, "+ + "and the charts emit them only for a pinned version. Pin it in Chart.yaml to the Talos version "+ + "the nodes run, or to %s or lower to keep the v1alpha1 shape.", + pinned, + config.TalosVersion1_13, + config.TalosVersion1_13, + ) + } + //nolint:wrapcheck // cockroachdb/errors.WithHint at boundary. return errors.WithHintf( conflict, - "templateOptions.talosVersion is %s. Pin it to %s or lower in Chart.yaml; above that "+ - "contract Talos keeps these settings in documents of their own, which the charts do not emit.", + "templateOptions.talosVersion is %s; above %s Talos keeps these settings in documents of their own. "+ + "Templates vendored by an earlier talm still write them in v1alpha1: re-sync them with "+ + "`talm init --update --preset `, accepting the templates but reviewing the Chart.yaml and "+ + "values.yaml changes it also offers (they move the version pins and the install image), "+ + "and move the reported fields out of v1alpha1 in your own "+ + "templates and patches. Pin it to %s or lower in Chart.yaml to keep the v1alpha1 shape.", pinned, config.TalosVersion1_13, + config.TalosVersion1_13, ) } diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go index c6631f63..63910c14 100644 --- a/pkg/engine/engine_test.go +++ b/pkg/engine/engine_test.go @@ -23,6 +23,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "regexp" "strconv" "strings" @@ -1205,15 +1206,25 @@ func TestObjectArrayMergeKeysMatchesUpstreamMergerSurface(t *testing.T) { // - cluster/network/podSubnets — v1alpha1 PodSubnet // - cluster/network/serviceSubnets — v1alpha1 ServiceSubnet // - cluster/apiServer/auditPolicy — v1alpha1 AuditPolicyConfig -// - ingress — typed NetworkRuleConfig Ingress -// - portSelector/ports — typed NetworkRuleConfig Ports +// - NetworkRuleConfig/ingress, NetworkRuleConfig/portSelector/ports +// - the replace-tagged fields of the typed Kubernetes documents func TestReplaceSemanticPathsMatchesUpstreamReplaceTags(t *testing.T) { expected := map[string]struct{}{ - "cluster/network/podSubnets": {}, - "cluster/network/serviceSubnets": {}, - "cluster/apiServer/auditPolicy": {}, - "ingress": {}, - "portSelector/ports": {}, + "cluster/network/podSubnets": {}, + "cluster/network/serviceSubnets": {}, + "cluster/apiServer/auditPolicy": {}, + "NetworkRuleConfig/ingress": {}, + "NetworkRuleConfig/portSelector/ports": {}, + "BGPInstanceConfig/advertise": {}, + "BGPInstanceConfig/importRoutes": {}, + "BGPInstanceConfig/neighbors": {}, + "CRIBaseRuntimeSpecConfig/overrides": {}, + "KubeNetworkConfig/podSubnets": {}, + "KubeNetworkConfig/serviceSubnets": {}, + "KubeAuthenticationConfig/configuration": {}, + "KubeAuditPolicyConfig/configuration": {}, + "KubeCredentialProviderConfig/configuration": {}, + "KubeStaticPodConfig/pod": {}, } if len(replaceSemanticPaths) != len(expected) { t.Fatalf("table size drift: have %d entries, want %d (re-grep upstream for `merge:\"replace\"` and reconcile)\nhave: %#v\nwant: %#v", @@ -1226,6 +1237,59 @@ func TestReplaceSemanticPathsMatchesUpstreamReplaceTags(t *testing.T) { } } +// The same field is replace-tagged on one kind and merged on another, so +// the prune has to tell them apart by kind: KubeAuthenticationConfig's +// configuration reaches the patcher whole, KubeAdmissionControlConfig's is +// reduced to the operator's delta. +func TestPruneIdenticalKeys_ReplacePathsAreKeyedByKind(t *testing.T) { + doc := func(kind string, keys ...string) map[string]any { + configuration := map[string]any{} + for _, k := range keys { + configuration[k] = k + } + + return map[string]any{k8sKeyAPIVersion: "v1alpha1", k8sKeyKind: kind, "configuration": configuration} + } + + for kind, wantKept := range map[string]int{"KubeAuthenticationConfig": 2, "KubeAdmissionControlConfig": 1} { + body := doc(kind, "rendered", "added") + pruneIdenticalKeys(body, doc(kind, "rendered")) + + configuration, _ := body["configuration"].(map[string]any) + if len(configuration) != wantKept { + t.Errorf("%s: configuration keeps %d keys after the prune, want %d: %v", kind, len(configuration), wantKept, configuration) + } + } +} + +// A partial edit of a replace-tagged list or map in the BGP and CRI +// documents has to reach the patcher whole: pruned to the operator's delta, +// the replace would drop what the render carries. +func TestPruneIdenticalKeys_KeepsReplaceTaggedNetworkAndCRIFields(t *testing.T) { + for _, tc := range []struct { + kind, field string + rendered, ok any + }{ + {"BGPInstanceConfig", "advertise", []any{"dummy0"}, []any{"dummy0", "dummy1"}}, + {"BGPInstanceConfig", "importRoutes", []any{map[string]any{"instance": "a"}}, []any{map[string]any{"instance": "a"}, map[string]any{"instance": "b"}}}, + {"BGPInstanceConfig", "neighbors", []any{map[string]any{"address": "192.0.2.1"}}, []any{map[string]any{"address": "192.0.2.1"}, map[string]any{"address": "192.0.2.2"}}}, + {"CRIBaseRuntimeSpecConfig", "overrides", map[string]any{"process": "a"}, map[string]any{"process": "a", "hooks": "b"}}, + } { + t.Run(tc.kind+"/"+tc.field, func(t *testing.T) { + doc := func(v any) map[string]any { + return map[string]any{k8sKeyAPIVersion: "v1alpha1", k8sKeyKind: tc.kind, "name": "fabric", tc.field: v} + } + + body := doc(deepClone(tc.ok)) + pruneIdenticalKeys(body, doc(tc.rendered)) + + if !reflect.DeepEqual(body[tc.field], tc.ok) { + t.Errorf("%s was pruned to %v, want it whole: %v", tc.field, body[tc.field], tc.ok) + } + }) + } +} + // TestJoinYAMLPath pins joinYAMLPath's contract: the document root // (empty parent) joins without a leading separator so the path lookup // in objectArrayMergeKeys matches the documented form ("machine/..."); diff --git a/pkg/engine/golden_test.go b/pkg/engine/golden_test.go index af6afe86..78b2f6bc 100644 --- a/pkg/engine/golden_test.go +++ b/pkg/engine/golden_test.go @@ -63,10 +63,16 @@ func TestGoldenRender(t *testing.T) { {"generic-worker-multidoc", genericChartPath, "templates/worker.yaml", "v1.12"}, {"generic-controlplane-legacy", genericChartPath, "templates/controlplane.yaml", "v1.11"}, {"generic-worker-legacy", genericChartPath, "templates/worker.yaml", "v1.11"}, - // v1.13 is what charts/generic pins, so it is the contract that actually - // reaches users of that preset. + // v1.13 is the last contract that keeps the Kubernetes settings in + // v1alpha1. {"generic-controlplane-v113", genericChartPath, "templates/controlplane.yaml", "v1.13"}, {"generic-worker-v113", genericChartPath, "templates/worker.yaml", "v1.13"}, + // v1.14 moved the Kubernetes settings out of v1alpha1 into typed + // documents, so it is a schema of its own. + {"cozystack-controlplane-v114", cozystackChartPath, "templates/controlplane.yaml", "v1.14"}, + {"cozystack-worker-v114", cozystackChartPath, "templates/worker.yaml", "v1.14"}, + {"generic-controlplane-v114", genericChartPath, "templates/controlplane.yaml", "v1.14"}, + {"generic-worker-v114", genericChartPath, "templates/worker.yaml", "v1.14"}, } for _, tc := range cases { diff --git a/pkg/engine/render_test.go b/pkg/engine/render_test.go index e19487a1..3671a59a 100644 --- a/pkg/engine/render_test.go +++ b/pkg/engine/render_test.go @@ -3057,6 +3057,170 @@ ingress: } }) + // From contract v1.14 the charts write KubeNetworkConfig and, on + // cozystack with OIDC, KubeAuthenticationConfig into every node file. + // Both carry replace-tagged fields, keyed by kind because + // KubeAdmissionControlConfig has a `configuration` that merges. + const kubeDocsRendered = `version: v1alpha1 +machine: + type: controlplane +--- +apiVersion: v1alpha1 +kind: KubeNetworkConfig +podSubnets: + - 10.244.0.0/16 +serviceSubnets: + - 10.96.0.0/16 +--- +apiVersion: v1alpha1 +kind: KubeAuthenticationConfig +configuration: + anonymous: + enabled: true + conditions: + - path: /livez + jwt: + - issuer: + url: https://issuer.example.com + audiences: + - kubernetes +` + + mergeKubeDocsBody := func(t *testing.T, body string) string { + t.Helper() + + nodeFile := filepath.Join(t.TempDir(), "node0.yaml") + if err := os.WriteFile(nodeFile, []byte(body), 0o644); err != nil { + t.Fatalf("write node file: %v", err) + } + + merged, err := MergeFileAsPatch([]byte(kubeDocsRendered), nodeFile) + if err != nil { + t.Fatalf("MergeFileAsPatch: %v", err) + } + + return string(merged) + } + + t.Run("KubeNetworkConfig podSubnets partial edit preserves rendered subnet", func(t *testing.T) { + out := mergeKubeDocsBody(t, `# talm: nodes=["10.0.0.1"] +machine: + type: controlplane +--- +apiVersion: v1alpha1 +kind: KubeNetworkConfig +podSubnets: + - 10.244.0.0/16 + - fd00:10:244::/56 +serviceSubnets: + - 10.96.0.0/16 +`) + for _, want := range []string{"10.244.0.0/16", "fd00:10:244::/56"} { + if !strings.Contains(out, want) { + t.Errorf("podSubnet %s missing after a partial edit:\n%s", want, out) + } + } + }) + + t.Run("KubeAuthenticationConfig partial edit preserves the rest of configuration", func(t *testing.T) { + out := mergeKubeDocsBody(t, `# talm: nodes=["10.0.0.1"] +machine: + type: controlplane +--- +apiVersion: v1alpha1 +kind: KubeAuthenticationConfig +configuration: + anonymous: + enabled: true + conditions: + - path: /livez + jwt: + - issuer: + url: https://issuer.example.com + audiences: + - kubernetes + - dashboard +`) + for _, want := range []string{"anonymous:", "path: /livez", "- dashboard", "- kubernetes"} { + if !strings.Contains(out, want) { + t.Errorf("%q missing after a partial edit of a replace-tagged configuration:\n%s", want, out) + } + } + }) + + // Component extraArgs merge per key, replacing the whole value of a key + // the body sets. A list-valued flag edited in the node file has to reach + // the patcher whole, or the replace drops the rendered entries. + for _, tc := range []struct{ name, rendered, body string }{ + { + name: "v1alpha1 apiServer", + rendered: `machine: + type: controlplane +cluster: + apiServer: + extraArgs: + feature-gates: + - A=true + max-requests-inflight: "400" +`, + body: `# talm: nodes=["10.0.0.1"] +machine: + type: controlplane +cluster: + apiServer: + extraArgs: + feature-gates: + - A=true + - B=true + max-requests-inflight: "400" +`, + }, + { + name: "KubeAPIServerConfig", + rendered: `machine: + type: controlplane +--- +apiVersion: v1alpha1 +kind: KubeAPIServerConfig +extraArgs: + feature-gates: + - A=true + max-requests-inflight: "400" +`, + body: `# talm: nodes=["10.0.0.1"] +machine: + type: controlplane +--- +apiVersion: v1alpha1 +kind: KubeAPIServerConfig +extraArgs: + feature-gates: + - A=true + - B=true + max-requests-inflight: "400" +`, + }, + } { + t.Run(tc.name+" extraArgs list partial edit keeps rendered values", func(t *testing.T) { + nodeFile := filepath.Join(t.TempDir(), "node0.yaml") + if err := os.WriteFile(nodeFile, []byte(tc.body), 0o644); err != nil { + t.Fatalf("write node file: %v", err) + } + + merged, err := MergeFileAsPatch([]byte(tc.rendered), nodeFile) + if err != nil { + t.Fatalf("MergeFileAsPatch: %v", err) + } + + out := string(merged) + for _, want := range []string{"A=true", "B=true", "max-requests-inflight"} { + if !strings.Contains(out, want) { + t.Errorf("%s missing after a partial edit of a list-valued flag:\n%s", want, out) + } + } + }) + } + t.Run("auditPolicy partial edit preserves rendered map keys", func(t *testing.T) { // cluster.apiServer.auditPolicy is an Unstructured map tagged // `merge:"replace"`: upstream overwrites the entire map with diff --git a/pkg/engine/testdata/golden/cozystack-controlplane-legacy.golden.yaml b/pkg/engine/testdata/golden/cozystack-controlplane-legacy.golden.yaml index 83d3c8ae..80a7e6ca 100644 --- a/pkg/engine/testdata/golden/cozystack-controlplane-legacy.golden.yaml +++ b/pkg/engine/testdata/golden/cozystack-controlplane-legacy.golden.yaml @@ -54,7 +54,7 @@ machine: global_filter = [ "r|^/dev/drbd.*|", "r|^/dev/dm-.*|", "r|^/dev/zd.*|", "r|^/dev/loop.*|" ] } install: - image: ghcr.io/cozystack/cozystack/talos:v1.12.6 + image: ghcr.io/cozystack/cozystack/talos:v1.14.2 # -- Discovered disks: disk: "/dev/sda" diff --git a/pkg/engine/testdata/golden/cozystack-controlplane-multidoc.golden.yaml b/pkg/engine/testdata/golden/cozystack-controlplane-multidoc.golden.yaml index 37c9fe4d..8a96af4c 100644 --- a/pkg/engine/testdata/golden/cozystack-controlplane-multidoc.golden.yaml +++ b/pkg/engine/testdata/golden/cozystack-controlplane-multidoc.golden.yaml @@ -54,7 +54,7 @@ machine: global_filter = [ "r|^/dev/drbd.*|", "r|^/dev/dm-.*|", "r|^/dev/zd.*|", "r|^/dev/loop.*|" ] } install: - image: ghcr.io/cozystack/cozystack/talos:v1.12.6 + image: ghcr.io/cozystack/cozystack/talos:v1.14.2 # -- Discovered disks: disk: "/dev/sda" diff --git a/pkg/engine/testdata/golden/cozystack-controlplane-v114.golden.yaml b/pkg/engine/testdata/golden/cozystack-controlplane-v114.golden.yaml new file mode 100644 index 00000000..3133874a --- /dev/null +++ b/pkg/engine/testdata/golden/cozystack-controlplane-v114.golden.yaml @@ -0,0 +1,153 @@ + +machine: + type: controlplane + sysctls: + net.ipv4.neigh.default.gc_thresh1: "4096" + net.ipv4.neigh.default.gc_thresh2: "8192" + net.ipv4.neigh.default.gc_thresh3: "16384" + net.ipv4.tcp_orphan_retries: "3" + net.ipv4.tcp_fin_timeout: "30" + net.core.netdev_max_backlog: "5000" + net.core.netdev_budget: "600" + net.core.netdev_budget_usecs: "8000" + kernel: + modules: + - name: openvswitch + - name: drbd + parameters: + - usermode_helper=disabled + - name: drbd_transport_tcp + - name: zfs + - name: spl + - name: vfio_pci + - name: vfio_iommu_type1 + certSANs: + - 127.0.0.1 + files: + - content: | + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + device_ownership_from_security_context = true + [plugins."io.containerd.cri.v1.runtime"] + device_ownership_from_security_context = true + path: /etc/cri/conf.d/20-customization.part + op: create + - op: overwrite + path: /etc/lvm/lvm.conf + permissions: 0o644 + content: | + backup { + backup = 0 + archive = 0 + } + devices { + global_filter = [ "r|^/dev/drbd.*|", "r|^/dev/dm-.*|", "r|^/dev/zd.*|", "r|^/dev/loop.*|" ] + } + install: + image: ghcr.io/cozystack/cozystack/talos:v1.14.2 + + # -- Discovered disks: + disk: "/dev/sda" +cluster: + etcd: + advertisedSubnets: + - 192.168.1.0/24 + extraArgs: + quota-backend-bytes: "8589934592" +--- +apiVersion: v1alpha1 +kind: KubeClusterConfig +clusterName: "cozystack" +endpoint: "https://talm-test.invalid:6443" +--- +apiVersion: v1alpha1 +kind: KubeNodeConfig +nodeIP: + validSubnets: + - 192.168.1.0/24 +labels: + node.kubernetes.io/exclude-from-external-load-balancers: + $patch: delete +taints: + node-role.kubernetes.io/control-plane: + $patch: delete +--- +apiVersion: v1alpha1 +kind: KubeletConfig +config: + cpuManagerPolicy: static + maxPods: 512 +--- +apiVersion: v1alpha1 +kind: KubeNetworkConfig +dnsDomain: "cozy.local" +podSubnets: + - 10.244.0.0/16 +serviceSubnets: + - 10.96.0.0/16 +--- +apiVersion: v1alpha1 +kind: KubeFlannelCNIConfig +$patch: delete +--- +apiVersion: v1alpha1 +kind: DiscoveryServiceConfig +name: default +$patch: delete +--- +apiVersion: v1alpha1 +kind: KubeProxyConfig +enabled: false +--- +apiVersion: v1alpha1 +kind: KubeControllerManagerConfig +extraArgs: + bind-address: 0.0.0.0 + allocate-node-cidrs: true + cluster-cidr: "10.244.0.0/16" +--- +apiVersion: v1alpha1 +kind: KubeSchedulerConfig +extraArgs: + bind-address: 0.0.0.0 +--- +apiVersion: v1alpha1 +kind: KubeAPIServerConfig +certExtraSANs: + - 127.0.0.1 +--- +apiVersion: v1alpha1 +kind: RegistryMirrorConfig +name: docker.io +endpoints: + - url: https://mirror.gcr.io +# -- Discovered interfaces: +# eth0: +# hardwareAddr:aa:bb:cc:00:00:01 +# busPath: pci-0000:00:1f.0 +# driver: +# vendor: +# product: ) +--- +apiVersion: v1alpha1 +kind: HostnameConfig +hostname: "talos-23c56" +--- +apiVersion: v1alpha1 +kind: ResolverConfig +nameservers: + - address: "8.8.8.8" +--- +apiVersion: v1alpha1 +kind: LinkConfig +name: eth0 +addresses: + - address: 192.168.201.10/24 +routes: + - gateway: 192.168.201.1 +--- +apiVersion: v1alpha1 +kind: ExtensionServiceConfig +name: zfs-service +environment: + - ZFS_EXPORT_TIMEOUT=0 diff --git a/pkg/engine/testdata/golden/cozystack-worker-legacy.golden.yaml b/pkg/engine/testdata/golden/cozystack-worker-legacy.golden.yaml index fe358a97..81e0a92d 100644 --- a/pkg/engine/testdata/golden/cozystack-worker-legacy.golden.yaml +++ b/pkg/engine/testdata/golden/cozystack-worker-legacy.golden.yaml @@ -51,7 +51,7 @@ machine: global_filter = [ "r|^/dev/drbd.*|", "r|^/dev/dm-.*|", "r|^/dev/zd.*|", "r|^/dev/loop.*|" ] } install: - image: ghcr.io/cozystack/cozystack/talos:v1.12.6 + image: ghcr.io/cozystack/cozystack/talos:v1.14.2 # -- Discovered disks: disk: "/dev/sda" diff --git a/pkg/engine/testdata/golden/cozystack-worker-multidoc.golden.yaml b/pkg/engine/testdata/golden/cozystack-worker-multidoc.golden.yaml index 56caccc3..fc0fd82b 100644 --- a/pkg/engine/testdata/golden/cozystack-worker-multidoc.golden.yaml +++ b/pkg/engine/testdata/golden/cozystack-worker-multidoc.golden.yaml @@ -51,7 +51,7 @@ machine: global_filter = [ "r|^/dev/drbd.*|", "r|^/dev/dm-.*|", "r|^/dev/zd.*|", "r|^/dev/loop.*|" ] } install: - image: ghcr.io/cozystack/cozystack/talos:v1.12.6 + image: ghcr.io/cozystack/cozystack/talos:v1.14.2 # -- Discovered disks: disk: "/dev/sda" diff --git a/pkg/engine/testdata/golden/cozystack-worker-v114.golden.yaml b/pkg/engine/testdata/golden/cozystack-worker-v114.golden.yaml new file mode 100644 index 00000000..1cf4e174 --- /dev/null +++ b/pkg/engine/testdata/golden/cozystack-worker-v114.golden.yaml @@ -0,0 +1,111 @@ + +machine: + type: worker + sysctls: + net.ipv4.neigh.default.gc_thresh1: "4096" + net.ipv4.neigh.default.gc_thresh2: "8192" + net.ipv4.neigh.default.gc_thresh3: "16384" + net.ipv4.tcp_orphan_retries: "3" + net.ipv4.tcp_fin_timeout: "30" + net.core.netdev_max_backlog: "5000" + net.core.netdev_budget: "600" + net.core.netdev_budget_usecs: "8000" + kernel: + modules: + - name: openvswitch + - name: drbd + parameters: + - usermode_helper=disabled + - name: drbd_transport_tcp + - name: zfs + - name: spl + - name: vfio_pci + - name: vfio_iommu_type1 + certSANs: + - 127.0.0.1 + files: + - content: | + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + device_ownership_from_security_context = true + [plugins."io.containerd.cri.v1.runtime"] + device_ownership_from_security_context = true + path: /etc/cri/conf.d/20-customization.part + op: create + - op: overwrite + path: /etc/lvm/lvm.conf + permissions: 0o644 + content: | + backup { + backup = 0 + archive = 0 + } + devices { + global_filter = [ "r|^/dev/drbd.*|", "r|^/dev/dm-.*|", "r|^/dev/zd.*|", "r|^/dev/loop.*|" ] + } + install: + image: ghcr.io/cozystack/cozystack/talos:v1.14.2 + + # -- Discovered disks: + disk: "/dev/sda" +--- +apiVersion: v1alpha1 +kind: KubeClusterConfig +clusterName: "cozystack" +endpoint: "https://talm-test.invalid:6443" +--- +apiVersion: v1alpha1 +kind: KubeNodeConfig +nodeIP: + validSubnets: + - 192.168.1.0/24 +--- +apiVersion: v1alpha1 +kind: KubeletConfig +config: + cpuManagerPolicy: static + maxPods: 512 +--- +apiVersion: v1alpha1 +kind: KubeNetworkConfig +dnsDomain: "cozy.local" +podSubnets: + - 10.244.0.0/16 +serviceSubnets: + - 10.96.0.0/16 +--- +apiVersion: v1alpha1 +kind: RegistryMirrorConfig +name: docker.io +endpoints: + - url: https://mirror.gcr.io +# -- Discovered interfaces: +# eth0: +# hardwareAddr:aa:bb:cc:00:00:01 +# busPath: pci-0000:00:1f.0 +# driver: +# vendor: +# product: ) +--- +apiVersion: v1alpha1 +kind: HostnameConfig +hostname: "talos-23c56" +--- +apiVersion: v1alpha1 +kind: ResolverConfig +nameservers: + - address: "8.8.8.8" +--- +apiVersion: v1alpha1 +kind: LinkConfig +name: eth0 +addresses: + - address: 192.168.201.10/24 +routes: + - gateway: 192.168.201.1 +--- +apiVersion: v1alpha1 +kind: ExtensionServiceConfig +name: zfs-service +environment: + - ZFS_EXPORT_TIMEOUT=0 diff --git a/pkg/engine/testdata/golden/generic-controlplane-v114.golden.yaml b/pkg/engine/testdata/golden/generic-controlplane-v114.golden.yaml new file mode 100644 index 00000000..373348c8 --- /dev/null +++ b/pkg/engine/testdata/golden/generic-controlplane-v114.golden.yaml @@ -0,0 +1,54 @@ + +machine: + type: controlplane + install: + + # -- Discovered disks: + disk: "/dev/sda" + +cluster: + etcd: + advertisedSubnets: + - 192.168.1.0/24 +--- +apiVersion: v1alpha1 +kind: KubeClusterConfig +clusterName: "generic" +endpoint: "https://talm-test.invalid:6443" +--- +apiVersion: v1alpha1 +kind: KubeNodeConfig +nodeIP: + validSubnets: + - 192.168.1.0/24 +--- +apiVersion: v1alpha1 +kind: KubeNetworkConfig +podSubnets: + - 10.244.0.0/16 +serviceSubnets: + - 10.96.0.0/16 +# -- Discovered interfaces: +# eth0: +# hardwareAddr:aa:bb:cc:00:00:01 +# busPath: pci-0000:00:1f.0 +# driver: +# vendor: +# product: ) +--- +apiVersion: v1alpha1 +kind: HostnameConfig +hostname: "talos-23c56" +--- +apiVersion: v1alpha1 +kind: ResolverConfig +nameservers: + - address: "8.8.8.8" +--- +apiVersion: v1alpha1 +kind: LinkConfig +name: eth0 +addresses: + - address: 192.168.201.10/24 +routes: + - gateway: 192.168.201.1 diff --git a/pkg/engine/testdata/golden/generic-worker-v114.golden.yaml b/pkg/engine/testdata/golden/generic-worker-v114.golden.yaml new file mode 100644 index 00000000..8b0393d2 --- /dev/null +++ b/pkg/engine/testdata/golden/generic-worker-v114.golden.yaml @@ -0,0 +1,49 @@ + +machine: + type: worker + install: + + # -- Discovered disks: + disk: "/dev/sda" +--- +apiVersion: v1alpha1 +kind: KubeClusterConfig +clusterName: "generic" +endpoint: "https://talm-test.invalid:6443" +--- +apiVersion: v1alpha1 +kind: KubeNodeConfig +nodeIP: + validSubnets: + - 192.168.1.0/24 +--- +apiVersion: v1alpha1 +kind: KubeNetworkConfig +podSubnets: + - 10.244.0.0/16 +serviceSubnets: + - 10.96.0.0/16 +# -- Discovered interfaces: +# eth0: +# hardwareAddr:aa:bb:cc:00:00:01 +# busPath: pci-0000:00:1f.0 +# driver: +# vendor: +# product: ) +--- +apiVersion: v1alpha1 +kind: HostnameConfig +hostname: "talos-23c56" +--- +apiVersion: v1alpha1 +kind: ResolverConfig +nameservers: + - address: "8.8.8.8" +--- +apiVersion: v1alpha1 +kind: LinkConfig +name: eth0 +addresses: + - address: 192.168.201.10/24 +routes: + - gateway: 192.168.201.1