From 6d5d16d3308139d6996f6bd2d1595c37d74f6b39 Mon Sep 17 00:00:00 2001 From: Yoan Moscatelli Date: Fri, 30 Oct 2026 14:31:01 +0000 Subject: [PATCH] feat: run a user-data script on the nodes `instance_user_data` names a script every node runs once booted, to install a product for instance. It runs as a systemd unit of its own rather than inside cloud-init, so the boot wait is not held by it, and from /usr/local/libexec, where SELinux leaves it unconfined. Under the configuration directory it ran in the domain of init, which may not even run chage or write /etc/shadow. The script learns where its node stands from /etc/platform-spawner/node.env. Once the machines have booted, the nodes are handed the key of the bastion over SSH, so that one of them can drive the others. examples/user-data/artesca-install.sh installs ARTESCA from its AMI on one node or three, and QUICKSTART walks through spawning both. Co-Authored-By: Claude Opus 5.5 --- Pulumi.yaml | 4 + README.md | 1 + __main__.py | 109 ++++++++++++++++++++++++++ docs/PLATFORM.md | 28 +++++++ docs/QUICKSTART.md | 76 +++++++++++++++++- examples/user-data/artesca-install.sh | 71 +++++++++++++++++ 6 files changed, 288 insertions(+), 1 deletion(-) create mode 100755 examples/user-data/artesca-install.sh diff --git a/Pulumi.yaml b/Pulumi.yaml index 042d3e81..96244b03 100644 --- a/Pulumi.yaml +++ b/Pulumi.yaml @@ -94,6 +94,10 @@ config: type: integer description: "Root disk size for the instance in GiB (AWS only, set by the flavor on OpenStack)" default: 50 + instance_user_data: + type: string + description: "Script every node runs in the background once booted (see docs/PLATFORM.md)" + default: "" # Bastion host configuration bastion_image: diff --git a/README.md b/README.md index e7f0f642..1d6e1147 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ A platform is described by the configuration of its stack. Set a key with | instance_flavor | Flavor of the instance, a portable size or a name from the cloud catalog | string | `medium` | no | | instance_ssh_user | SSH user for the instances (defaults to the one the image is known to use) | string | `` | no | | instance_root_disk_size | Root disk size for the instance in GiB (AWS only, set by the flavor on OpenStack) | integer | `50` | no | +| instance_user_data | Script every node runs in the background once booted (see docs/PLATFORM.md) | string | `` | no | | bastion_image | Image for the bastion host | string | `rocky-9` | no | | bastion_flavor | Flavor of the bastion, a portable size or a name from the cloud catalog | string | `small` | no | | bastion_ssh_user | SSH user for the bastion (defaults to the one the image is known to use) | string | `` | no | diff --git a/__main__.py b/__main__.py index 381efa3c..5f1d871d 100644 --- a/__main__.py +++ b/__main__.py @@ -60,6 +60,22 @@ printf '%s' "$BASTION_PRIVATE_KEY" | ssh -F {config} bastion "$install" scp -F {config} {bastion_config} bastion:{remote_config} """ +# Where the user-data of the nodes lands, and the unit it runs as. +# NOTE: The script goes where local programs do. Under SELinux, systemd runs +# what it finds there unconfined, while a script left among the configuration +# runs in the domain of init, which may not even touch /etc/shadow. +USER_DATA_ENV = pathlib.PurePosixPath("/etc/platform-spawner/node.env") +USER_DATA_SCRIPT = pathlib.PurePosixPath("/usr/local/libexec/platform-spawner/user-data") +USER_DATA_UNIT = "platform-spawner-user-data" +# Relative to the home of the SSH user, which needs no privilege to write to +NODES_KEY = ".ssh/platform-spawner-nodes" +# What the nodes are handed when they run user-data, so they reach each other +NODES_SETUP_SCRIPT = """set -eu +install="install -d -m 700 ~/.ssh && cat > ~/{key} && chmod 600 ~/{key}" +for host in {hosts}; do + printf '%s' "$NODES_PRIVATE_KEY" | ssh -F {config} "$host" "$install" +done +""" # Address space of the routed planes, needed both to carve the subnets out # and to tell the bastion whose clock it is allowed to serve CONTROL_PLANE_CIDR = "172.30.100.0/24" @@ -229,6 +245,10 @@ def __main__() -> None: "user": _ssh_user(config, "bastion"), } nodes_info = {} + user_data = config.require("instance_user_data") + present_nodes = [ + index for index in range(1, instance_count + 1) if f"node-{index}" not in absent_nodes + ] for node_index in range(1, instance_count + 1): # NOTE: The interfaces of an absent node are kept. They hold its @@ -266,6 +286,13 @@ def __main__() -> None: cloud_config={ "ssh_authorized_keys": [bastion_key.public_key_openssh], **_ntp_config(servers=[bastion_cp_iface.ip, bastion_wp_iface.ip]), + **( + _user_data_config( + user_data, node_index, present_nodes, _ssh_user(config, "instance") + ) + if user_data + else {} + ), }, ) instances.append(node) @@ -294,6 +321,8 @@ def __main__() -> None: booted = _wait_for_boot(ssh_config_ready, ["bastion", *nodes_info], instances) _equip_bastion(ssh_config_ready, bastion_key, booted, instances) + if user_data: + _equip_nodes(ssh_config_ready, list(nodes_info), bastion_key, booted, instances) _clean_up_ssh_files(ssh_config_file) @@ -402,6 +431,56 @@ def _ntp_config(**settings: object) -> dict: return {"ntp": {"enabled": True, "ntp_client": NTP_CLIENT, **settings}} +def _user_data_config( + script_path: str, + node_index: int, + present_nodes: list[int], + ssh_user: str, +) -> dict: + """ + Render the cloud-init configuration running a node's user-data. + + NOTE: Run as a unit of its own rather than by cloud-init. The update waits + for cloud-init to be done with every machine, and what a script is handed + to do, installing a product for instance, easily outlasts that wait. + + The script is told where the node stands through an environment file. Its + addresses are fixed, so they are known before any machine exists. + """ + env = { + "NODE_NAME": f"node-{node_index}", + "NODE_INDEX": node_index, + "NODE_COUNT": len(present_nodes), + "NODES_CONTROL_PLANE_IPS": " ".join(f"172.30.100.{100 + i}" for i in present_nodes), + "NODES_WORKLOAD_PLANE_IPS": " ".join(f"172.30.200.{100 + i}" for i in present_nodes), + "CONTROL_PLANE_CIDR": CONTROL_PLANE_CIDR, + "WORKLOAD_PLANE_CIDR": WORKLOAD_PLANE_CIDR, + "SSH_USER": ssh_user, + "NODES_KEY": NODES_KEY, + } + return { + "write_files": [ + { + "path": str(USER_DATA_ENV), + "content": "".join(f'{name}="{value}"\n' for name, value in env.items()), + }, + { + "path": str(USER_DATA_SCRIPT), + "content": pathlib.Path(script_path).read_text(), + "permissions": "0755", + }, + ], + "runcmd": [ + [ + "systemd-run", + f"--unit={USER_DATA_UNIT}", + f"--property=EnvironmentFile={USER_DATA_ENV}", + str(USER_DATA_SCRIPT), + ], + ], + } + + def _extra_network_cidr(index: int) -> str: """ Return the address space of the `index`th extra network. @@ -628,5 +707,35 @@ def _equip_bastion( ) +def _equip_nodes( + config_file: pulumi.Output[str], + nodes: list[str], + bastion_key: pulumi_tls.PrivateKey, + booted: pulumi_command.local.Command, + instances: list[pulumi.CustomResource], +) -> None: + """ + Hand the nodes running user-data a key, so they reach each other. + + NOTE: The key of the bastion rather than one more. Every node trusts it + already, and it opens nothing but the nodes, which is all a node driving + the others needs. It lands in the home of the SSH user: a node whose + product has taken sudo away still takes it on the next update. + """ + pulumi_command.local.Command( + "nodes-ssh-key", + create=config_file.apply( + lambda config: NODES_SETUP_SCRIPT.format( + config=config, + key=NODES_KEY, + hosts=" ".join(nodes), + ) + ), + environment={"NODES_PRIVATE_KEY": bastion_key.private_key_openssh}, + triggers=[instance.id for instance in instances], + opts=pulumi.ResourceOptions(depends_on=[booted]), + ) + + if __name__ == "__main__": __main__() diff --git a/docs/PLATFORM.md b/docs/PLATFORM.md index dd965b4a..799c776d 100644 --- a/docs/PLATFORM.md +++ b/docs/PLATFORM.md @@ -68,6 +68,34 @@ platform, and is not what a platform is spawned to test. From a workflow, the same goes through the [`update` action](GITHUB_ACTION.md#updating). +### Running a script on the nodes + +`instance_user_data` names a script every node runs once it has booted, +installing a product for instance. It runs as root, in a unit of its own, +`platform-spawner-user-data`, rather than inside cloud-init: the update only +waits for the machines to boot, and the script outlasts it as long as it +needs. Follow it with: + +```bash +ssh -F ssh_config node-1 sudo journalctl -fu platform-spawner-user-data +``` + +The script learns where it stands from its environment: + +| Variable | What it holds | +|----------|---------------| +| `NODE_NAME`, `NODE_INDEX` | Which node runs it, `node-1` and `1` for instance | +| `NODE_COUNT` | How many nodes are there, absent ones left out | +| `NODES_CONTROL_PLANE_IPS`, `NODES_WORKLOAD_PLANE_IPS` | The addresses of every node, itself included | +| `CONTROL_PLANE_CIDR`, `WORKLOAD_PLANE_CIDR` | The address spaces of the planes | +| `SSH_USER` | The user the nodes are reached with | +| `NODES_KEY` | Where a key opening the other nodes lands, under the home of `SSH_USER` | + +The key arrives once every machine has booted, so a script needing the other +nodes waits for it. It is the key of the bastion, which opens the nodes and +nothing else. [`examples/user-data/artesca-install.sh`](../examples/user-data/artesca-install.sh) +installs ARTESCA that way, on one node or three. + ## Networks | Network | CIDR | What it is | diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index d68bd8fa..d34096ee 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -142,7 +142,7 @@ The ones worth knowing early: | Key | Default | What it does | |-----|---------|--------------| -| `instance_flavor` | `medium` | `small`, `medium`, `large` or `xlarge`. | +| `instance_flavor` | `medium` | `small`, `medium`, `large` or `xlarge`, or any name of the cloud catalog (`c5.4xlarge`). | | `instance_root_disk_size` | `50` | Root disk of a node, in GiB. AWS only, since the flavor decides it on OpenStack. | | `bastion_flavor` | `small` | Same sizes as `instance_flavor`. | | `authorized_cidrs` | `['__my_ip__']` | Who may reach the bastion. `__my_ip__` stands for the public address of the machine spawning. | @@ -238,6 +238,80 @@ for someone to decide, so remove it once you are sure: rm -f ssh__ ssh__.pub ``` +## Example: ARTESCA from its AMI + +One platform per architecture, both from the same AMI, here +`artesca-4.4.0-preview.32` in `eu-north-1`. The AMI is shared with us, so it is +named by its id. Every command names its stack with `-s`, which lets both +live side by side without `pulumi stack select`: + +```bash +for stack in -1node:1 -3node:3; do + name=${stack%%:*} count=${stack##*:} + pulumi stack init "$name" + pulumi -s "$name" config set provider aws + pulumi -s "$name" config set aws:region eu-north-1 + pulumi -s "$name" config set product artesca + pulumi -s "$name" config set instance_image ami-0cc90c0853c34e7c4 + pulumi -s "$name" config set instance_count "$count" + pulumi -s "$name" config set instance_flavor c5.4xlarge + pulumi -s "$name" config set instance_root_disk_size 300 + pulumi -s "$name" config set instance_ssh_user artesca-os + pulumi -s "$name" config set ssh_private_key_create true + pulumi -s "$name" config set instance_user_data examples/user-data/artesca-install.sh +done +``` + +| Key | Why | +|-----|-----| +| `instance_flavor` | `medium` is far too small for ARTESCA. `c5.4xlarge` is what the CI demo uses. | +| `instance_root_disk_size` | The `demo` profile keeps its volumes on the root disk. | +| `instance_ssh_user` | The AMI is none of the `rocky-*` images, so nothing else names its user. | +| `instance_user_data` | Installs ARTESCA once the nodes are up, see below. Leave it out for bare nodes. | + +Both spawn at once, from two shells or in the background: + +```bash +pulumi -s -1node up --yes & +pulumi -s -3node up --yes & +wait +``` + +> **Warning** +> The AMI expires the password of `artesca-os`, and an expired password turns +> SSH away without a terminal to change it, keys included. The update then +> fails with `node-1 never answered ssh`. +> [`artesca-install.sh`](../examples/user-data/artesca-install.sh) lifts the +> expiry, so a stack without `instance_user_data` needs a script that does at +> least that much. A node already spawned without one takes an interactive +> `ssh -tt`, and the initial password of the AMI, to change it by hand. + +`pulumi up` returns once the nodes have booted, and the install carries on on +its own on node-1, which drives the other nodes on three. It takes a while. +Follow it, then reach the UI once it is done: + +```bash +ssh -F ssh_config--3node node-1 sudo journalctl -fu platform-spawner-user-data +ssh -F ssh_config--3node -L 8443:172.30.100.101:8443 node-1 # then https://localhost:8443 +``` + +Installer logs are in `/var/log/artesca/` on node-1, and the configuration it +was given in `~/config.yaml`. The other nodes are one hop away, through the +bastion like any other: + +```bash +ssh -F ssh_config--1node node-1 +ssh -F ssh_config--3node node-3 +``` + +Destroy both the same way as any other: + +```bash +for name in -1node -3node; do + pulumi -s "$name" destroy --yes && pulumi stack rm -s "$name" --yes +done +``` + ## Cheat sheet ```bash diff --git a/examples/user-data/artesca-install.sh b/examples/user-data/artesca-install.sh new file mode 100755 index 00000000..2cf6f6cd --- /dev/null +++ b/examples/user-data/artesca-install.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# Install ARTESCA from an ARTESCA AMI, on one node or three. +# +# Meant for `instance_user_data`. node-1 drives the install and reaches the +# other nodes with the key platform-spawner hands it once they have booted. +# +# NOTE: AWS has no layer 2 between the nodes and checks the source of every +# packet, so there are no virtual IPs and pods speak through IP-in-IP. +set -euo pipefail + +# NOTE: The AMI expires the password of its user, and an expired password +# refuses SSH without a TTY to change it, keys included. Every node needs it +# lifted: platform-spawner waits on SSH, and node-1 drives the others with it. +# The AMI expires it from cloud-init, which started this script and may not +# have got there yet. +cloud-init status --wait >/dev/null || true +chage --lastday "$(date +%F)" "$SSH_USER" + +[ "$NODE_INDEX" = 1 ] || exit 0 + +home=$(getent passwd "$SSH_USER" | cut -d: -f6) +key=/etc/metalk8s/pki/bastion + +echo "Waiting for the nodes key" +until [ -s "$home/$NODES_KEY" ]; do sleep 10; done +install -D -m 600 "$home/$NODES_KEY" "$key" + +nodes=() +for ip in $NODES_CONTROL_PLANE_IPS; do + index=${ip##*.} + index=$((index - 100)) + [ "$index" = 1 ] && continue + + echo "Waiting for node-$index ($ip)" + until ssh -i "$key" -o StrictHostKeyChecking=no -o ConnectTimeout=10 \ + "$SSH_USER@$ip" true; do + sleep 10 + done + nodes+=(--node "{name: node-$index, host: $ip, key: $key}") +done + +if [ "$NODE_COUNT" = 1 ]; then + architecture=(--architecture single-node) +else + architecture=( + --architecture 3-nodes + --no-managed-management-vip + --no-data-vip + ) +fi + +cd "$home" +rm -f config.yaml +su "$SSH_USER" -c "$(printf '%q ' sudo ./artesca-installer config generate \ + --output config.yaml \ + "${architecture[@]}" \ + --profile demo \ + --use-case generic \ + --username scality \ + --base-domain workloadplane.scality.local \ + --local-node-name node-1 \ + --ipip-always \ + --workload-plane-cidr "$WORKLOAD_PLANE_CIDR" \ + --control-plane-cidr "$CONTROL_PLANE_CIDR" \ + --management-ip '' \ + "${nodes[@]}")" +su "$SSH_USER" -c "sudo ./artesca-installer config validate --config config.yaml" + +echo "Configuration generated, deploying" +su "$SSH_USER" -c "sudo ./artesca-installer deploy --allow-unverified-raid -c config.yaml" +echo "Deployment finished"