From 575cb89f55360273a71000fdf385d68c1399f684 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Mon, 28 Sep 2026 17:32:31 +0000 Subject: [PATCH 01/12] Move OCCM to control plane and allow hostNetwork for it. Signed-off-by: Kurt Garloff --- providers/openstack/scs2/cluster-addon/ccm/values.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/providers/openstack/scs2/cluster-addon/ccm/values.yaml b/providers/openstack/scs2/cluster-addon/ccm/values.yaml index 3f290366..74120cba 100644 --- a/providers/openstack/scs2/cluster-addon/ccm/values.yaml +++ b/providers/openstack/scs2/cluster-addon/ccm/values.yaml @@ -3,11 +3,19 @@ openstack-cloud-controller-manager: enabled: true name: ccm-cloud-config create: true + #nodeSelector: + ## Added by KG: Move to control plane nodeSelector: + node-role.kubernetes.io/control-plane: "" tolerations: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule + ## Added by KG + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + # Allow DNS resolution before CoreDNS is up + hostNetwork: true extraVolumes: - name: clouds-yaml secret: From c0cef3d2f87dfa207b6615f1b30a7613fbb30dc5 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 29 Sep 2026 01:17:35 +0200 Subject: [PATCH 02/12] Need dnsPolicy: Default Signed-off-by: Kurt Garloff --- providers/openstack/scs2/cluster-addon/ccm/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/providers/openstack/scs2/cluster-addon/ccm/values.yaml b/providers/openstack/scs2/cluster-addon/ccm/values.yaml index 74120cba..bf47c4d2 100644 --- a/providers/openstack/scs2/cluster-addon/ccm/values.yaml +++ b/providers/openstack/scs2/cluster-addon/ccm/values.yaml @@ -16,6 +16,8 @@ openstack-cloud-controller-manager: effect: NoSchedule # Allow DNS resolution before CoreDNS is up hostNetwork: true + # dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: Default extraVolumes: - name: clouds-yaml secret: From 7604746d6b909fdb62f740e35a0c52bb485a9760 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 29 Sep 2026 07:15:35 +0000 Subject: [PATCH 03/12] Wait for kubeapi ... Signed-off-by: Kurt Garloff --- .../templates/kubeadm-control-plane-template.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml index 767eac68..02a433cb 100644 --- a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml +++ b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -75,6 +75,21 @@ spec: permissions: "0755" preKubeadmCommands: - bash /etc/kube-proxy-patch.sh + - | + # The full kubeadm join command is embedded in this same cloud-init script, + # a few lines below. Extract the VIP from it. + SELF=$(grep -l 'control-plane-endpoint' /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null | head -1) + VIP=$(grep -oP -- '--control-plane-endpoint=\K[^:]+' "$SELF") + PORT=$(grep -oP -- '--control-plane-endpoint=[^:]+:\K\d+' "$SELF") + + for i in $(seq 1 30); do + if curl -ks --max-time 5 "https://${VIP}:${PORT}/healthz"; then + echo "[pre-join] API server ${VIP}:${PORT} reachable (attempt $i)" + break + fi + echo "[pre-join] waiting… ($i/30) ${VIP}:${PORT}" + sleep 5 + done initConfiguration: nodeRegistration: kubeletExtraArgs: From 4a827e0a4143c653739d6835bf08902e8663a854 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 29 Sep 2026 12:05:45 +0000 Subject: [PATCH 04/12] Fix waiting script. Signed-off-by: Kurt Garloff --- .../kubeadm-control-plane-template.yaml | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml index 02a433cb..c88f2254 100644 --- a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml +++ b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -73,23 +73,26 @@ spec: owner: root:root path: /etc/kube-proxy-patch.sh permissions: "0755" + - content: | + #!/usr/bin/env bash + # The full kubeadm join command is embedded in this same cloud-init script, + # a few lines below. Extract the VIP from it. + VIPPORT=$(grep 'apiServerEndpoint:' /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null | head -1 | sed 's/^ *apiServerEndpoint: *//') + if test -z "$VIPPORT"; then echo "No apiServerEndpoint found, cannot wait ..."; sleep 6; fi + for i in $(seq 0 30); do + if curl -ks --max-time 5 "https://${VIPPORT}/healthz"; then + echo "[pre-join] API server ${VIPPORT} reachable (attempt $i)" + break + fi + echo "[pre-join] waiting… ($i/30) ${VIPPORT}" + sleep 5 + done + owner: root:root + permissions: "0755" + path: /etc/kube-wait-kubeapi.sh preKubeadmCommands: - bash /etc/kube-proxy-patch.sh - - | - # The full kubeadm join command is embedded in this same cloud-init script, - # a few lines below. Extract the VIP from it. - SELF=$(grep -l 'control-plane-endpoint' /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null | head -1) - VIP=$(grep -oP -- '--control-plane-endpoint=\K[^:]+' "$SELF") - PORT=$(grep -oP -- '--control-plane-endpoint=[^:]+:\K\d+' "$SELF") - - for i in $(seq 1 30); do - if curl -ks --max-time 5 "https://${VIP}:${PORT}/healthz"; then - echo "[pre-join] API server ${VIP}:${PORT} reachable (attempt $i)" - break - fi - echo "[pre-join] waiting… ($i/30) ${VIP}:${PORT}" - sleep 5 - done + - bash /etc/kube-wait-kubeapi.sh initConfiguration: nodeRegistration: kubeletExtraArgs: From 3a535371c557a23de8ea5b243b7e3c17e200dd9f Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 29 Sep 2026 12:19:39 +0000 Subject: [PATCH 05/12] Fix grep to not find itself. Signed-off-by: Kurt Garloff --- .../cluster-class/templates/kubeadm-control-plane-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml index c88f2254..9b300e4d 100644 --- a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml +++ b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -77,7 +77,7 @@ spec: #!/usr/bin/env bash # The full kubeadm join command is embedded in this same cloud-init script, # a few lines below. Extract the VIP from it. - VIPPORT=$(grep 'apiServerEndpoint:' /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null | head -1 | sed 's/^ *apiServerEndpoint: *//') + VIPPORT=$(grep '^ *apiServerEndpoint:' /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null | head -1 | sed 's/^ *apiServerEndpoint: *//') if test -z "$VIPPORT"; then echo "No apiServerEndpoint found, cannot wait ..."; sleep 6; fi for i in $(seq 0 30); do if curl -ks --max-time 5 "https://${VIPPORT}/healthz"; then From c01be08ac099431b31ee64643a2d7ea2eac0f2cb Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 29 Sep 2026 12:30:11 +0000 Subject: [PATCH 06/12] Debugging ... Signed-off-by: Kurt Garloff --- .../templates/kubeadm-control-plane-template.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml index 9b300e4d..3e0d18f0 100644 --- a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml +++ b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -78,14 +78,16 @@ spec: # The full kubeadm join command is embedded in this same cloud-init script, # a few lines below. Extract the VIP from it. VIPPORT=$(grep '^ *apiServerEndpoint:' /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null | head -1 | sed 's/^ *apiServerEndpoint: *//') + echo "VIP Port: $VIPPORT" + cat /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null if test -z "$VIPPORT"; then echo "No apiServerEndpoint found, cannot wait ..."; sleep 6; fi - for i in $(seq 0 30); do + for i in $(seq 0 10); do if curl -ks --max-time 5 "https://${VIPPORT}/healthz"; then echo "[pre-join] API server ${VIPPORT} reachable (attempt $i)" break fi echo "[pre-join] waiting… ($i/30) ${VIPPORT}" - sleep 5 + sleep 4 done owner: root:root permissions: "0755" From 656767f3508894c8a30071007a34f83e788bcde4 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 29 Sep 2026 12:33:03 +0000 Subject: [PATCH 07/12] Wait longer. Signed-off-by: Kurt Garloff --- .../cluster-class/templates/kubeadm-control-plane-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml index 3e0d18f0..d666375a 100644 --- a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml +++ b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -81,7 +81,7 @@ spec: echo "VIP Port: $VIPPORT" cat /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null if test -z "$VIPPORT"; then echo "No apiServerEndpoint found, cannot wait ..."; sleep 6; fi - for i in $(seq 0 10); do + for i in $(seq 0 15); do if curl -ks --max-time 5 "https://${VIPPORT}/healthz"; then echo "[pre-join] API server ${VIPPORT} reachable (attempt $i)" break From 07eb486ae48a336d3885ed16d6fb408c773b97eb Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 29 Sep 2026 12:57:37 +0000 Subject: [PATCH 08/12] Try controlPlaneEndpoint: Signed-off-by: Kurt Garloff --- .../templates/kubeadm-control-plane-template.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml index d666375a..57217c5d 100644 --- a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml +++ b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -78,9 +78,12 @@ spec: # The full kubeadm join command is embedded in this same cloud-init script, # a few lines below. Extract the VIP from it. VIPPORT=$(grep '^ *apiServerEndpoint:' /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null | head -1 | sed 's/^ *apiServerEndpoint: *//') + if test -z "$VIPPORT"; then + VIPPORT=$(grep '^ *controlPlaneEndpoint:' /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null | head -1 | sed 's/^ *controlPlaneEndpoint: *//') + fi echo "VIP Port: $VIPPORT" cat /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null - if test -z "$VIPPORT"; then echo "No apiServerEndpoint found, cannot wait ..."; sleep 6; fi + if test -z "$VIPPORT"; then echo "No apiServerEndpoint found, cannot wait ..."; sleep 6; exit 0; fi for i in $(seq 0 15); do if curl -ks --max-time 5 "https://${VIPPORT}/healthz"; then echo "[pre-join] API server ${VIPPORT} reachable (attempt $i)" From 9a081b59321a5be43f426d2d4b788269d9e650cd Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 29 Sep 2026 13:05:05 +0000 Subject: [PATCH 09/12] grep -h needed to avoid file name confusing us. Signed-off-by: Kurt Garloff --- .../templates/kubeadm-control-plane-template.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml index 57217c5d..21ca9962 100644 --- a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml +++ b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -77,12 +77,12 @@ spec: #!/usr/bin/env bash # The full kubeadm join command is embedded in this same cloud-init script, # a few lines below. Extract the VIP from it. - VIPPORT=$(grep '^ *apiServerEndpoint:' /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null | head -1 | sed 's/^ *apiServerEndpoint: *//') + VIPPORT=$(grep -h '^ *apiServerEndpoint:' /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null | head -1 | sed 's/^ *apiServerEndpoint: *//') if test -z "$VIPPORT"; then - VIPPORT=$(grep '^ *controlPlaneEndpoint:' /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null | head -1 | sed 's/^ *controlPlaneEndpoint: *//') + VIPPORT=$(grep -h '^ *controlPlaneEndpoint:' /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null | head -1 | sed 's/^ *controlPlaneEndpoint: *//') fi echo "VIP Port: $VIPPORT" - cat /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null + #cat /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null if test -z "$VIPPORT"; then echo "No apiServerEndpoint found, cannot wait ..."; sleep 6; exit 0; fi for i in $(seq 0 15); do if curl -ks --max-time 5 "https://${VIPPORT}/healthz"; then From 217a2bd5384e95f3408aa03fdd10f87dafb42aad Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 29 Sep 2026 13:37:29 +0000 Subject: [PATCH 10/12] Don't wait for kubeapi on init node. Signed-off-by: Kurt Garloff --- .../templates/kubeadm-control-plane-template.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml index 21ca9962..195081fb 100644 --- a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml +++ b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -82,6 +82,10 @@ spec: VIPPORT=$(grep -h '^ *controlPlaneEndpoint:' /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null | head -1 | sed 's/^ *controlPlaneEndpoint: *//') fi echo "VIP Port: $VIPPORT" + # kubeadm config is in different directory in Flatcar (/etc) and Ubuntu (/run/kubeadm). + kubeadm_file="/etc/kubeadm.yml" + if test ! -f ${kubeadm_file}; then kubeadm_file="/run/kubeadm/kubeadm.yaml" fi + if test -f ${kubeadm_file}; then echo "Init node, not waiting ..."; exit 0; fi #cat /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null if test -z "$VIPPORT"; then echo "No apiServerEndpoint found, cannot wait ..."; sleep 6; exit 0; fi for i in $(seq 0 15); do From 21b90301d9141c11778b88b42fa710ba9a21c3e6 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 29 Sep 2026 16:55:17 +0000 Subject: [PATCH 11/12] Fix syntax error. Signed-off-by: Kurt Garloff --- .../cluster-class/templates/kubeadm-control-plane-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml index 195081fb..e451cea4 100644 --- a/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml +++ b/providers/openstack/scs2/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -84,7 +84,7 @@ spec: echo "VIP Port: $VIPPORT" # kubeadm config is in different directory in Flatcar (/etc) and Ubuntu (/run/kubeadm). kubeadm_file="/etc/kubeadm.yml" - if test ! -f ${kubeadm_file}; then kubeadm_file="/run/kubeadm/kubeadm.yaml" fi + if ! test -f ${kubeadm_file}; then kubeadm_file="/run/kubeadm/kubeadm.yaml"; fi if test -f ${kubeadm_file}; then echo "Init node, not waiting ..."; exit 0; fi #cat /var/lib/cloud/instance/user-data.txt /run/cloud-init/user-data.txt 2>/dev/null if test -z "$VIPPORT"; then echo "No apiServerEndpoint found, cannot wait ..."; sleep 6; exit 0; fi From deea4306e242b2b8670254f5774bf57e6014d56d Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 29 Sep 2026 17:53:10 +0000 Subject: [PATCH 12/12] Revert "Move OCCM to control plane and allow hostNetwork for it." This reverts commit 575cb89f55360273a71000fdf385d68c1399f684. But keep the dnsPolicy: Default, as we need it. Signed-off-by: Kurt Garloff --- providers/openstack/scs2/cluster-addon/ccm/values.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/providers/openstack/scs2/cluster-addon/ccm/values.yaml b/providers/openstack/scs2/cluster-addon/ccm/values.yaml index bf47c4d2..739b63a9 100644 --- a/providers/openstack/scs2/cluster-addon/ccm/values.yaml +++ b/providers/openstack/scs2/cluster-addon/ccm/values.yaml @@ -3,17 +3,11 @@ openstack-cloud-controller-manager: enabled: true name: ccm-cloud-config create: true - #nodeSelector: - ## Added by KG: Move to control plane nodeSelector: - node-role.kubernetes.io/control-plane: "" tolerations: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule - ## Added by KG - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule # Allow DNS resolution before CoreDNS is up hostNetwork: true # dnsPolicy: ClusterFirstWithHostNet