From 943d9eec440ac3d23d91d0d8777c8ee3a69f7ad4 Mon Sep 17 00:00:00 2001 From: "riad.elhajjaji" Date: Thu, 29 Oct 2026 09:24:49 +0100 Subject: [PATCH] fix: only wait on the machines an update brings up Updating a platform waited again on every machine, the ones left alone included. On a Rocky machine where anything installed a package since it booted, `cloud-init status` then failed for the SSH user with a PermissionError on /run/cloud-init/cloud.cfg, and the update failed with "booted badly, cloud-init exited 1". The cloud-init package keeps /run/cloud-init to root through a tmpfiles rule, and the systemd package runs every tmpfiles rule again whenever a package ships one, so cloud-init can no longer read its own runtime config for anybody else (canonical/cloud-init#6680). Freshly booted machines do not show it, which is why spawns went through. Asking as root is no way out: on ARTESCA the SSH user is not allowed to run cloud-init through sudo once the product is installed. Each machine now has a wait of its own, triggered by that machine only. An update waits on the machines it brought up, which are fresh, and leaves the others alone. The waits are named after the machine, so an update of a platform spawned before this change waits once more on every machine. Refs: MK8S-519 --- __main__.py | 91 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 40 deletions(-) diff --git a/__main__.py b/__main__.py index 381efa3..97fd747 100644 --- a/__main__.py +++ b/__main__.py @@ -25,34 +25,33 @@ "rocky-8": "rocky", "rocky-9": "rocky", } -# Held until every machine answers SSH and cloud-init is done with it, so that -# a finished update means a platform that can actually be used. +# Held until a machine answers SSH and cloud-init is done with it, so that a +# finished update means a platform that can actually be used. BOOT_WAIT_TIMEOUT = 600 BOOT_WAIT_SCRIPT = """set -eu -for host in {hosts}; do - give_up=$(( $(date +%s) + {timeout} )) - until ssh -F {config} -o ConnectTimeout=10 "$host" true 2>/dev/null; do - if [ "$(date +%s)" -ge "$give_up" ]; then - echo "$host never answered ssh" >&2 - exit 1 - fi - sleep 5 - done - - # NOTE: Bounded like the wait above. `cloud-init status --wait` holds for - # as long as it takes, and a boot that never ends would otherwise hold the - # update with it. 124 is what the timeout itself exits with. - booted=0 - timeout {timeout} ssh -F {config} "$host" cloud-init status --wait || booted=$? - if [ "$booted" -eq 124 ]; then - echo "$host was still booting after {timeout}s" >&2 - exit 1 - fi - if [ "$booted" -ne 0 ]; then - echo "$host booted badly, cloud-init exited $booted" >&2 +host={host} +give_up=$(( $(date +%s) + {timeout} )) +until ssh -F {config} -o ConnectTimeout=10 "$host" true 2>/dev/null; do + if [ "$(date +%s)" -ge "$give_up" ]; then + echo "$host never answered ssh" >&2 exit 1 fi + sleep 5 done + +# NOTE: Bounded like the wait above. `cloud-init status --wait` holds for as +# long as it takes, and a boot that never ends would otherwise hold the update +# with it. 124 is what the timeout itself exits with. +booted=0 +timeout {timeout} ssh -F {config} "$host" cloud-init status --wait || booted=$? +if [ "$booted" -eq 124 ]; then + echo "$host was still booting after {timeout}s" >&2 + exit 1 +fi +if [ "$booted" -ne 0 ]; then + echo "$host booted badly, cloud-init exited $booted" >&2 + exit 1 +fi """ # What the bastion is handed, so that it reaches the nodes on its own BASTION_SETUP_SCRIPT = """set -eu @@ -580,27 +579,39 @@ def _wait_for_boot( config_file: pulumi.Output[str], hosts: list[str], instances: list[pulumi.CustomResource], -) -> pulumi_command.local.Command: - """Hold the update until every machine is done booting.""" - return pulumi_command.local.Command( - "wait-for-boot", - create=config_file.apply( - lambda config: BOOT_WAIT_SCRIPT.format( - hosts=" ".join(hosts), - config=config, - timeout=BOOT_WAIT_TIMEOUT, - ) - ), - # Wait again whenever a machine is replaced under us - triggers=[instance.id for instance in instances], - opts=pulumi.ResourceOptions(depends_on=instances), - ) +) -> list[pulumi_command.local.Command]: + """ + Hold the update until every new machine is done booting. + + NOTE: One wait per machine, so that an update only waits on the machines + it replaced. The ones left alone have been in use since, and asking + cloud-init about them again can only go wrong: once something has run the + tmpfiles rules again, as installing a package does, Rocky keeps + /run/cloud-init to root and `cloud-init status` fails for the SSH user, + which may well not be allowed to run it as root. + """ + return [ + pulumi_command.local.Command( + f"wait-for-boot-{host}", + create=config_file.apply( + lambda config, host=host: BOOT_WAIT_SCRIPT.format( + host=host, + config=config, + timeout=BOOT_WAIT_TIMEOUT, + ) + ), + # Wait again whenever this machine is replaced under us + triggers=[instance.id], + opts=pulumi.ResourceOptions(depends_on=[instance]), + ) + for host, instance in zip(hosts, instances, strict=True) + ] def _equip_bastion( config_file: pulumi.Output[str], bastion_key: pulumi_tls.PrivateKey, - booted: pulumi_command.local.Command, + booted: list[pulumi_command.local.Command], instances: list[pulumi.CustomResource], ) -> None: """ @@ -624,7 +635,7 @@ def _equip_bastion( # Handed again whenever a machine changes. The config names the # nodes, and a replaced bastion starts with neither key nor config. triggers=[instance.id for instance in instances], - opts=pulumi.ResourceOptions(depends_on=[booted]), + opts=pulumi.ResourceOptions(depends_on=booted), )