diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index e433620..42dc37a 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -24,10 +24,24 @@ jobs: # own venvs. ansible-lint pulls ansible-core into its venv itself. pipx install ansible-core pipx install ansible-lint - # ansible-core ships no community collections, so without this the - # roles' non-builtin FQCNs resolve locally (requirements.txt pins the - # batteries-included `ansible` package) and fail here. - ansible-galaxy collection install -r ansible/requirements.yml + # The roles' non-builtin FQCNs resolve locally because requirements.txt + # pins the batteries-included `ansible` package; here they have to be + # fetched - into one path every later step agrees on. Each tool has + # its own default view of where collections live: the runner ships + # some preinstalled, and ansible-lint runs from its own pipx venv and + # reads neither those nor ansible-core's. Without the explicit -p and + # --force, galaxy finds the runner's copies, reports "nothing to do", + # and ansible-lint still cannot resolve community.general. + # + # Exported through GITHUB_ENV rather than the job's env: block + # because that block cannot see the runner context. Singular _PATH - + # ansible-compat hard errors on the plural spelling - and outside the + # workspace, so ansible-lint never walks it as project files. + collections="$RUNNER_TEMP/ansible-collections" + echo "ANSIBLE_COLLECTIONS_PATH=$collections" >>"$GITHUB_ENV" + ansible-galaxy collection install -r ansible/requirements.yml \ + -p "$collections" --force + ansible-galaxy collection list -p "$collections" - name: copy examples run: | diff --git a/.gitignore b/.gitignore index 51a1a0e..eaf1594 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,7 @@ private/** !private/.gitkeep # scratch tmp/ +# ansible-lint's cache and its symlinked copy of the project, written into +# whichever directory holds .git - so the repo root, whatever `ansible-lint` +# is run from. +.ansible/ diff --git a/AGENTS.md b/AGENTS.md new file mode 120000 index 0000000..681311e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1 @@ +CLAUDE.md \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 9472d4e..21735fa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,8 +1,9 @@ # OpenTofu provisioning for ProxMox VE Infrastructure as code for provisioning and deprovisioning VMs on ProxMox VE. -Three layers: OpenTofu drives the hypervisor, cloud-init gives the guest its -boot-time identity, and Ansible installs software post-boot. +Three layers: a golden image carries what has to exist before first boot, +OpenTofu drives the hypervisor and PVE's native cloud-init settings, and +Ansible owns everything after boot. See README.md for setup; this file is what a working session needs that is not obvious from reading the code. @@ -11,18 +12,22 @@ obvious from reading the code. Template placeholders — fill in for your site (marked `EDIT` in the code): - ProxMox node name/endpoint: `variables.tofu` (`pve_node`, `pve_endpoint`). -- Datastores: snippets need a dir datastore with the `snippets` content type; - lvmthin cannot hold them. -- Base image: referenced directly as a disk `file_id`, so ProxMox imports and - converts it; there is no `download_file` resource. +- Datastores: `disk_datastore` holds VM disks and cloud-init drives; the + golden image needs a datastore with the **`import`** content type (not + `iso` — see the `import_from` gotcha below). +- Golden image: built by `scripts/build-image.sh` and named as the disk's + `import_from`, so ProxMox imports and converts it server-side; there is no + `download_file` resource. It exists to carry `qemu-guest-agent` — see the + gotcha below. - Secrets: `secrets.enc.json`, age via sops, key at `~/.config/sops/age/keys.txt`. Holds the API token and the state passphrase. ## Hard rules **Pre-existing VMs must never be touched.** `guards.tofu` plus lifecycle -preconditions in `modules/vm/main.tofu` enforce a VMID floor, a named protected -list, and a live check that rejects any VMID belonging to a VM not tagged +preconditions in `modules/vm-pve/main.tofu` enforce a VMID floor, a named +protected list, and a live check that rejects any VMID belonging to a VM not +tagged `opentofu`. Validations stop the floor and the list from being weakened by tfvars or `TF_VAR_*` overrides, and a postcondition fails the plan if the protected VMs are not all visible in the API listing (fail closed). @@ -41,20 +46,35 @@ the repo root. the only required variable. See `terraform.tfvars.example`. Adding a VM is adding a file to `inventory/`; deprovisioning is removing one — -by convention, `git mv` it to `inventory/destroy/` (only `inventory/*.yaml` is -scanned; subdirectories are invisible). The next apply destroys the VM, its -disk, and its snippet; moving the file back provisions a *fresh* VM. To keep a -VM and its data but power it off, set `started: false` instead. The filename -is the VM name (DNS label). Only `vm_id` is required — everything else takes -an `optional()` default from the `spec` object in `modules/vm/variables.tofu`, -which is the contract worth reading first. - -After touching anything under `cloud-init/`, run -`./scripts/check-cloud-init.sh`. `tofu validate` checks HCL and structurally -never sees the YAML that comes out of `templatefile()`. - -**Layer 2 — Ansible.** Cloud-init is decided at first boot and reachable only -by recreating the VM; everything softer — installed software and its config — +by convention, move it to `inventory/destroy/` (only `inventory/*.yaml` is +scanned; subdirectories are invisible). Plain `mv`, not `git mv`: this +template gitignores `inventory/*.yaml` and `inventory/destroy/*.yaml`, so +`git mv` fails with "not under version control" — a fork that tracks its own +specs can use `git mv`. The next apply destroys the VM and its disk; moving +the file back provisions a *fresh* VM. To keep a VM and its data but power it +off, set `started: false` instead. The filename is the VM name (DNS label). +Only `vm_id` is required — everything else takes an `optional()` default from +the `spec` object in `modules/vm-pve/variables.tofu`, which is the contract +worth reading first. + +**Layer 0 — the golden image.** `./scripts/build-image.sh [--upload]` builds +it: an upstream Ubuntu cloud image with `qemu-guest-agent` installed by +`virt-customize`, uploaded over the API. Rebuild when the upstream serial +should move; then repoint `cloud_image_file_id` at the new name. It is +uploaded as `import` content, not `iso`. Never replace an uploaded image in +place — `disk[0].import_from` is create-only *and* under `ignore_changes`, so +a same-name replacement changes what a running VM was built from with no plan +diff. Prerequisites are `libguestfs-tools`, a readable +`/boot/vmlinuz-$(uname -r)` (`dpkg-statoverride`), and membership of `kvm`; +the script's preflight names each remedy. + +**Layer 1 — PVE cloud-init.** `initialization` in `modules/vm-pve/main.tofu`: +user, keys, addresses, DNS, and `upgrade`. These are VM config over the API, so +editing one *does* diff — but cloud-init only reads the drive on first boot, so +a diff still does not reach a running guest. Rotating a key on a live VM means +ansible or a rebuild. + +**Layer 2 — Ansible.** Everything softer — installed software and its config — lives in `ansible/` and is re-applied any time with `ansible-playbook ansible/site.yaml [--limit ]` (after `source tofu.env`, which exports `ANSIBLE_CONFIG`; no plan diff, no recreation). A spec opts in @@ -65,77 +85,105 @@ which serves interactive shells only). Underscore names, each a directory under `ansible/roles/`, typos fail at plan time. The dynamic inventory (`ansible/inventory/tofu.py`) reads `tofu output -json vms`, so it needs an applied state; versions are pinned in each role's -`defaults/main.yaml`. Removing a role from the list does **not** uninstall it — -clean removal is a rebuild. Runs are idempotent (second run reports -changed=0). After touching anything under `ansible/`, run +`defaults/main.yaml`, and collections in `ansible/requirements.yml` (CI +installs from it — `ansible-core` alone ships none, so a `community.*` task +without a pin there passes locally and fails CI). The `base` role is in no +spec and cannot be opted out of: `site.yaml` applies it to every host via +`roles:`, ahead of the include loop. It carries what the cloud-init snippet +used to — the apt snapshot pin, the `APT::Periodic` zeros, the apt-daily +timers, key-only SSH, the timezone, and `spec.packages` — which means those +**are** now reachable without recreating the VM. Removing a role from the list +does **not** uninstall it; clean removal is a rebuild. Runs are idempotent +(second run reports changed=0). After touching anything under `ansible/`, run `./scripts/check-ansible.sh`. `./scripts/vm-fingerprint.sh @ fingerprints/.txt` captures a VM's environment fingerprint (packages, apt config, enabled units, users, and the ansible-installed layer-2 files under `~/.local` and `~/.bun` — excluding -per-instance noise like host keys and machine-id) into a tracked file. -Rebuild verification: capture, commit, destroy + reprovision, run -`ansible-playbook ansible/site.yaml`, capture to the same path — an empty -`git diff` proves the environment is identical. +per-instance noise like host keys and machine-id). `fingerprints/**` is +gitignored, so a capture is local unless `git add -f`d deliberately. Rebuild +verification: capture, destroy + reprovision, run `ansible-playbook +ansible/site.yaml`, capture to the same path — an empty diff proves the +environment is identical. Verified on this design: two rebuilds produced +byte-identical captures. ## Gotchas that have already cost time -- **Snippets need SSH.** The ProxMox API has no snippets endpoint, so the - provider uploads cloud-init user-data over SSH as root using - `pve_ssh_private_key_path`. Everything else goes over the API token. -- **The file resource also needs `Datastore.Allocate` on the snippet - datastore.** Before the SSH upload, the provider reads the storage config - via `GET /storage/`, which PVE gates behind the full admin privilege — - `AllocateSpace`/`AllocateTemplate` are not enough (HTTP 403 at apply). - Grant it via a role scoped to the snippet datastore, and give that role the - *complete* `Datastore.*` set: PVE ACLs on a more specific path **override** - the propagated role instead of merging with it. +- **`ciupgrade` is not root-gated, whatever the provider says.** The bpg + schema calls `initialization.upgrade` *"only allowed for `root@pam`"*; that + is stale. In `PVE/API2/Qemu.pm` (9.2.3) `ciupgrade` sits in + `$cloudinitoptions`, which needs `VM.Config.Cloudinit` **or** + `VM.Config.Network`. It is set explicitly because the provider's schema + default is Computed and PVE's own default is 1 — the opposite of + `spec.package_upgrade`'s default. Confirmed on the node, not just in + source: `qm config ` shows `ciupgrade: 0` after an apply by the token. - **An `@pve`-realm token cannot SSH anywhere.** It exists only in ProxMox's - user database — it is not a Linux account. The API identity and the SSH - identity can never be unified. -- **A cloud-config is one YAML mapping; duplicate top-level keys get silently - dropped.** That is why `base.yaml.tftpl` is the entire document and nothing - is ever appended to it — post-boot software belongs in an ansible role, not - in cloud-init. -- **Every scalar interpolated into a cloud-init template must be - `jsonencode()`d.** JSON is a YAML subset; a raw SSH key comment containing - `: ` becomes a YAML mapping (VM boots with no authorized keys), a package - containing `#` is silently truncated, and a VM named `no` becomes a boolean - hostname. `check-cloud-init.sh` has an adversarial pass that regresses this. -- **Editing cloud-init templates does not diff the VM.** The snippet - re-uploads but its ID (derived from the stable file name) is unchanged, so - the plan shows "0 to change" and running guests keep what they booted with. - Reaching an existing VM means recreating it: delete its inventory file, - apply, restore, apply. + user database — it is not a Linux account. Worth knowing before anyone + proposes reuniting the API and SSH identities to bring snippets back. +- **A disk `file_id` needs SSH; `import_from` does not.** Sourcing a disk from + `file_id` sends the provider down its "custom disk" path, which shells out + to `qm` on the node as root. That is a second SSH dependency, quite separate + from snippets, and it fails at *disk creation* with "unable to authenticate + user over SSH". `import_from` passes PVE's native import-from parameter + instead. It only accepts `images` or `import` content — never `iso` — and + the upload endpoint restricts `import` filenames to + `.ova/.qcow2/.raw/.vmdk` (`UPLOAD_IMPORT_EXT_RE_1` in `PVE/Storage.pm`). + Hence `local:import/.qcow2`: the extension is load-bearing. +- **`sshd_config.d` takes the FIRST value, apt.conf.d takes the last.** The + include glob is read in lexical order, so a *lower* number wins. The base + role's drop-in is `10-smithy.conf` for that reason; the image's own + `60-cloudimg-settings.conf` would otherwise be authoritative. +- **`cloud-init status` exits 2 on "degraded done".** Under a `set -e` that is + a silent, message-free abort — it cost a debugging session in + `vm-fingerprint.sh`. Degraded is the *steady state* here: the user-data PVE + generates uses the top-level `user:` key, deprecated in cloud-init 22.2. + Nothing in this repo emits it and nothing here can suppress it. +- **Pick the guest address by MAC, not by filtering.** The agent reports one + address list per interface, with `ipv4_addresses`, `mac_addresses` and + `network_interface_names` sharing an index. Flattening them was fine until + the docker role added a `docker0` at 172.17.0.1, which the ansible inventory + could have taken as `ansible_host`. `modules/vm-pve/outputs.tofu` selects the + entry whose MAC is `network_device[0].mac_address`; address ranges are + configurable and interface names vary by image, but that MAC cannot be + impersonated by a bridge the guest brought up itself. +- **`filename=@` must be the last `-F` in a PVE upload.** PVE parses the + multipart body in order and streams everything after the file part into the + file. A `checksum` field placed after it is never parsed *and* its bytes are + appended to the image — which `qemu-img` reads without complaint, so the + task reports OK and the corruption is silent. `scripts/build-image.sh` has + the order right and the node then verifies the checksum itself. +- **`APT::Snapshot` cannot be used while building the image.** It adds the + snapshot mirror alongside the configured one and refreshes both, and the + cloud image's root filesystem has ~366 MB free against a ~120 MB unpacked + universe index — the build dies in `dpkg --unpack`. `build-image.sh` + rewrites `ubuntu.sources` to the snapshot mirror for the duration instead. + Unrelated to a VM's own `archive_snapshot`, which still goes via apt.conf.d. - **Repeatability knobs live per-VM in the spec.** `archive_snapshot: - YYYYMMDDTHHMMSSZ` pins apt to snapshot.ubuntu.com via a `bootcmd` that - writes `APT::Snapshot` into apt.conf.d (bootcmd runs init-stage, before - apt-configure and package install; an apt.conf.d file survives cloud-init - regenerating `ubuntu.sources`). Official archive only — third-party repos - like docker are not snapshotted, which is why the docker ansible role pins - exact package versions in its defaults instead. - `package_upgrade` defaults to **false**; `package_update: true` - is explicit but not load-bearing — cloud-init refreshes indexes whenever - `packages:` is non-empty. unattended-upgrades is disabled fleet-wide (the - image ships it enabled): base's `bootcmd` zeros the `APT::Periodic` jobs and - a base runcmd disables the apt-daily timers. -- **PyYAML's `safe_load` does not error on duplicate keys**, it keeps the last - one. `scripts/check-cloud-init.sh` installs a custom loader for this reason. -- **`cloud-init schema` catches deprecations that still "work."** The check - script treats deprecation warnings as failures. + YYYYMMDDTHHMMSSZ` pins apt to snapshot.ubuntu.com; the ansible `base` role + writes `APT::Snapshot` into apt.conf.d before it installs anything, so the + pin is in force for the spec's own packages. Official archive only — + third-party repos like docker are not snapshotted, which is why the docker + role pins exact package versions in its defaults instead. `package_upgrade` + defaults to **false** and is the one first-boot-only knob left: it becomes + PVE's `ciupgrade`, so flipping it does not reach an existing VM, where + `packages` and `archive_snapshot` now do. unattended-upgrades is disabled + fleet-wide (the image ships it enabled): `base` zeros the `APT::Periodic` + jobs and masks the apt-daily timers — masked, not merely disabled, because a + postinst re-running `systemctl preset` can undo a disable. +- **The apt.conf.d filenames still say "cloudinit"** (`50cloudinit-snapshot`, + `51cloudinit-no-auto-upgrades`) and must stay that way. + `scripts/vm-fingerprint.sh` records apt configuration by filename; renaming + them would make every fingerprint captured so far incomparable. - **A child module must declare its own `required_providers`** naming - `bpg/proxmox`. Without `modules/vm/versions.tofu`, `tofu init` assumes + `bpg/proxmox`. Without `modules/vm-pve/versions.tofu`, `tofu init` assumes `hashicorp/proxmox` and fails. - **`agent { enabled = true }` makes apply block** until the guest agent reports an address. Quick with the default `package_upgrade: false`; with a - VM that sets it true, expect several minutes (timeout is 30m). -- **zsh's `echo` expands `\n`**, which corrupts expressions piped to - `tofu console`. Use a quoted heredoc. -- **`tofu console` output is double-encoded** when wrapping `jsonencode` — HCL - string quoting on top of the JSON. Two decode passes. -- **Once a state file exists, `tofu console` prints "Acquiring state lock..." - to STDOUT**, corrupting anything piped from it — hence `-lock=false` and - the result-line filter in `check-cloud-init.sh`. + VM that sets it true, expect several minutes (timeout is 30m). The agent + comes from the golden image, and nothing else can supply it — ansible needs + the address the agent reports in order to connect at all. Point + `cloud_image_file_id` at a stock cloud image (uploaded as `import`) and + apply hangs for the full 30m, then fails. - **pbkdf2 welds its salt to the key provider's block name** unless `encrypted_metadata_alias` is set — renaming a provider (or moving a passphrase between blocks) makes existing state undecryptable. Both diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d8d3f67..8ce03f9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,9 @@ or tokens. With a ProxMox node of your own: copy `terraform.tfvars.example` to `terraform.tfvars`, fill in the `EDIT` values (see README.md for the full setup, including `secrets.enc.json` — layout in `secrets.enc.json.example`), -then `source tofu.env` and use `inventory/example.yaml` as a starting spec. +then `source tofu.env` and use `inventory-example.yaml` as a starting spec. +You will also need a golden image on the node — `./scripts/build-image.sh +--upload` builds and uploads one; see README.md, "Building the golden image". Without a node, run what CI runs: @@ -39,7 +41,7 @@ Without a node, run what CI runs: export TF_VAR_state_passphrase=ci-dummy-passphrase-render-only tofu init -input=false tofu validate -./scripts/check-cloud-init.sh # needs: cloud-init, python3-yaml +ansible-galaxy collection install -r ansible/requirements.yml ansible-playbook ansible/site.yaml --syntax-check -i localhost, (cd ansible && ansible-lint) ``` @@ -47,22 +49,18 @@ ansible-playbook ansible/site.yaml --syntax-check -i localhost, The dummy passphrase satisfies the state-encryption config for read-only rendering; no state is created or read. -Notes on the two check scripts: - -- `check-cloud-init.sh` renders the real cloud-init output and validates it — - `tofu validate` alone never sees the YAML that comes out of - `templatefile()`. Run it after touching anything under `cloud-init/`. -- `check-ansible.sh` needs an applied state (it inspects the live inventory), - so it only runs against a real environment; CI covers the state-independent - parts (syntax check and lint). Run the full script if you have a node. +`check-ansible.sh` needs an applied state (it inspects the live inventory), so +it only runs against a real environment; CI covers the state-independent parts +(syntax check and lint). Run the full script if you have a node. ## Style -- Every scalar interpolated into a cloud-init template must be - `jsonencode()`d — see the comments in `cloud-init/base.yaml.tftpl` and the - adversarial pass in `check-cloud-init.sh` that regresses this. +- Any collection beyond `ansible.builtin` must be pinned in + `ansible/requirements.yml` in the same change that uses it. CI installs + `ansible-core`, which ships none, so an unpinned `community.*` task passes + locally and fails there. - Ansible role names use underscores; each is a directory under `ansible/roles/`. Pin software versions in the role's `defaults/main.yaml`. - Keep guard behavior intact: `guards.tofu` and the validations in - `modules/vm/variables.tofu` exist to protect pre-existing VMs and must not + `modules/vm-pve/variables.tofu` exist to protect pre-existing VMs and must not be weakened. diff --git a/README.md b/README.md index 04306cf..346c644 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,14 @@ the environment around the software under test stays fixed — same image, same package versions. The VMs are designed to be ephemeral. Core components for VM management: -- OpenTofu drives the ProxMox VE hypervisor (bpg/proxmox provider) -- cloud-init gives the guest its boot-time identity -- Ansible installs software post-boot — re-appliable at any time without +- A golden image carries what has to exist before first boot (the guest agent) +- OpenTofu drives the ProxMox VE hypervisor (bpg/proxmox provider) and its +native cloud-init settings — identity, keys, addressing +- Ansible owns everything after boot — re-appliable at any time without touching the VM lifecycle. Persistent backend components: -- A grafana/optel-lgtm container provides an OpenTelemetry endpoint to +- A grafana/otel-lgtm container provides an OpenTelemetry endpoint to capture and store Claude Code telemetry from agents running during testing. - A windmill-labs/windmill container is being evaluated as an orchestrator for the overall testing workflow. @@ -24,9 +25,9 @@ overall testing workflow. ## Overview - **One YAML file per VM.** Provisioning is adding a file to `inventory/`; - deprovisioning is deleting one, or moving it to the destroy subdiredtory. + deprovisioning is deleting one, or moving it to the destroy subdirectory. Only `vm_id` is required — everything else inherits a typed `optional()` - default from the `spec` contract in `modules/vm/variables.tofu`. + default from the `spec` contract in `modules/vm-pve/variables.tofu`. - **Repeatable environments.** Per-VM `archive_snapshot:` pins apt to [snapshot.ubuntu.com](https://snapshot.ubuntu.com) at a chosen instant, so package installs resolve identically forever; `package_upgrade` defaults to @@ -46,35 +47,35 @@ overall testing workflow. - **Encrypted state and plans** (pbkdf2 + AES-GCM, `enforced = true`), secrets via sops/age, and a `.gitignore` that covers the sharp edges (`crash.log` contains a TRACE-level dump including the API token, regardless of TF_LOG). -- **Real validation gates.** `tofu validate` never sees the YAML that comes - out of `templatefile()`; `scripts/check-cloud-init.sh` renders the real - user-data plus an adversarial fixture (SSH key comment containing `: `, - package containing `#`, hostname that is a YAML boolean) and runs - duplicate-key and `cloud-init schema` checks, treating deprecation warnings - as failures. `scripts/check-ansible.sh` asserts the dynamic inventory's - shape and syntax-checks (and, when installed, lints) the playbook. +- **Real validation gates.** `scripts/check-ansible.sh` asserts the dynamic + inventory's shape — every hostvar the roles rely on, every declared role + backed by a real directory — and syntax-checks and lints the playbook. + `scripts/build-image.sh` verifies the upstream image against its published + `SHA256SUMS`, asserts inside the built image that the build-time apt pin did + not leak, and has the node verify the upload's checksum. ## Layout ``` -├── versions.tofu providers.tofu encryption.tofu +├── CLAUDE.md # notes for coding agents; AGENTS.md is a symlink +├── versions.tofu +├── providers.tofu +├── encryption.tofu ├── main.tofu # sops secrets, node data source ├── guards.tofu # protected VMIDs + live foreign-VM lookup [EDIT] -├── vms.tofu # inventory/ -> module.vm, for_each +├── vms.tofu # inventory/ -> module.vm-pve, for_each ├── variables.tofu # fleet-wide defaults [EDIT] ├── outputs.tofu -├── modules/vm/ # the contract: what a VM is -├── cloud-init/ -│ ├── base.yaml.tftpl # every VM gets this - the whole document -│ └── base.runcmd.json.tftpl # commands every VM runs +├── modules/vm-pve/ # the contract: what a VM is ├── ansible/ # post-boot software │ ├── ansible.cfg -│ ├── site.yaml # one hostvar-driven play +│ ├── requirements.yml # pinned collections (CI installs these) +│ ├── site.yaml # base, then each VM's declared roles │ ├── inventory/tofu.py # dynamic inventory from tofu output -│ └── roles// # nats_server, bun, claude, docker, -│ # metafactory_arc +│ └── roles// # base (every VM), then nats_server, bun, +│ # claude, docker, metafactory_arc ├── inventory/ # one YAML file per VM [EDIT] -├── scripts/ # check-cloud-init.sh, check-ansible.sh, +├── scripts/ # build-image.sh, check-ansible.sh, │ # vm-fingerprint.sh ├── otel-lgtm/ # grafana/otel-lgtm observability stack [EDIT] │ # (docker compose; see its README.md) @@ -89,28 +90,23 @@ same system that is running opentofu and ansible, but that is not required. ## Prerequisites -- OpenTofu >= 1.10, `sops`, `age`, `ansible` (ansible-core >= 2.15; - `ansible-lint` optional), and (for the cloud-init check script) - `python3-yaml` and `cloud-init` on the workstation. +Download and install on PATH, for example ~/.local/bin - OpenTofu, https://github.com/opentofu/opentofu/releases/tag/v1.12.6 - sops, https://github.com/getsops/sops/releases/tag/v3.13.3 - age, https://github.com/FiloSottile/age/releases/tag/v1.2.1 - uv, https://github.com/astral-sh/uv/releases/tag/0.12.6 +- `libguestfs-tools` (Debian/Ubuntu package name) if you build the golden image there (see "Building the golden image"). On Arch Linux the package name is `libguestfs`. + - A ProxMox VE node (built against 9.x) with: - - a datastore that allows the `snippets` content type (`local` by default; - lvmthin cannot hold snippets), - - an Ubuntu cloud image uploaded (e.g. - `local:iso/resolute-server-cloudimg-amd64-20260720.img`), - - an API token for provisioning, and root SSH access for snippet upload - (see Credentials below). Besides the usual `VM.*`/`Datastore.*`/`SDN.Use` - privileges, the token needs **`Datastore.Allocate` on the snippet - datastore** — the provider reads the storage config (`GET /storage/`) - before uploading, and PVE gates that behind the full admin privilege. - Grant it via a role scoped to `/storage/`, and give - that role the *complete* `Datastore.*` set: PVE ACLs on a specific path - override propagated ones instead of merging. + - a datastore that allows the **`import`** content type for the golden + image (`local` by default; `iso` will not do — PVE only imports a disk + from `images` or `import` content), + - a golden image uploaded (see "Building the golden image"), and + - an API token for provisioning, with the usual + `VM.*`/`Datastore.*`/`SDN.Use` privileges. That is the only credential + this project needs — nothing here has, or wants, SSH to the node. ## Setup @@ -126,8 +122,7 @@ same system that is running opentofu and ansible, but that is not required. `secrets.enc.json` holds the ProxMox API token and the state passphrase (16 characters minimum). Once encrypted it is safe to commit. -2. **API user, roles, and token.** As root on the node (adjust the snippet - datastore path if yours is not `local`): +2. **API user, role, and token.** As root on the node: ```bash pveum user add opentofu-prov@pve --comment "OpenTofu provisioning (API token only)" @@ -135,24 +130,60 @@ same system that is running opentofu and ansible, but that is not required. pveum role add OpenTofuProv -privs "Datastore.AllocateSpace,Datastore.AllocateTemplate,Datastore.Audit,Pool.Allocate,Pool.Audit,SDN.Audit,SDN.Use,Sys.AccessNetwork,Sys.Audit,Sys.Console,Sys.Modify,VM.Allocate,VM.Audit,VM.Clone,VM.Config.CDROM,VM.Config.CPU,VM.Config.Cloudinit,VM.Config.Disk,VM.Config.HWType,VM.Config.Memory,VM.Config.Network,VM.Config.Options,VM.GuestAgent.Unrestricted,VM.Migrate,VM.PowerMgmt" pveum acl modify / -user opentofu-prov@pve -role OpenTofuProv - # Scoped role for the snippet datastore; must carry the FULL Datastore.* - # set - an ACL on a specific path overrides the propagated role, it does - # not merge with it. - pveum role add OpenTofuSnippetStore -privs "Datastore.Allocate,Datastore.AllocateSpace,Datastore.AllocateTemplate,Datastore.Audit" - pveum acl modify /storage/local -user opentofu-prov@pve -role OpenTofuSnippetStore - # privsep=0: the token inherits the user's ACLs. The secret prints ONCE - # it goes into secrets.enc.json (proxmox.api_token_secret). pveum user token add opentofu-prov@pve provisioning --privsep 0 ``` -3. **Provisioning SSH key.** The ProxMox API has no snippets endpoint, so the - provider uploads cloud-init user-data over SSH as root: + `Datastore.AllocateTemplate` is what lets the same token upload the golden + image, so no second identity is needed. - ```bash - ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519_pve -N "" - ssh-copy-id -i ~/.ssh/id_ed25519_pve.pub root@ - ``` +3. **Golden image.** Build and upload one before the first `tofu apply` — see + "Building the golden image". A stock cloud image will not do: it has no + guest agent, and apply blocks for the full 30-minute timeout waiting for an + address that never arrives. + +## Building the golden image + +Every VM's disk is imported from a golden image: an upstream Ubuntu cloud image +with `qemu-guest-agent` baked in. The agent is the one thing ansible cannot +install — `tofu apply` blocks until the agent reports an address, and the +dynamic inventory reads that same address to connect at all — so it has to be +present before first boot. + +`scripts/build-image.sh` does the whole job. On the machine that builds: + +```bash +sudo apt install libguestfs-tools +# libguestfs boots its appliance with the host kernel, which ships mode 0600 +sudo dpkg-statoverride --update --add root root 0644 /boot/vmlinuz-$(uname -r) +sudo usermod -aG kvm "$USER" # then log out and back in +``` + +The script's preflight checks each of those and names the fix if one is +missing; without `kvm` it still works, just slowly under emulation. Then: + +```bash +./scripts/build-image.sh --upload +``` + +which downloads the dated upstream serial, verifies it against that +directory's `SHA256SUMS`, installs the agent with `virt-customize`, and uploads +the result through the storage API as `import` content — the same token as +everything else, no scp. It prints the `volid` to put in +`cloud_image_file_id`. Build without `--upload` +first if you want to inspect the image (`virt-df`, `guestfish --ro`). + +Options: `--serial YYYYMMDD` picks the upstream build (default is the one this +repo currently pins), `--snapshot YYYYMMDDTHHMMSSZ` the snapshot.ubuntu.com +instant the agent is installed from, and `--outdir` where to work (default +`private/images/`, gitignored; budget ~2.5 GB). + +Rebuild when the upstream serial should move, then repoint +`cloud_image_file_id` at the new file name. **Never overwrite an uploaded +image**: the disk source is create-only, so a same-name replacement silently +changes what a running VM was built from without any plan diff. Build a new +name, repoint, and delete the old volume once nothing references it. ## Opentofu and Ansible startup @@ -168,12 +199,12 @@ Note: Edit all the copied files with your specific data. - `tofu init` # the first init installs the providers - `tofu plan` # verify the output looks like what you expect -Note: The `ssh_public_keys` in terraform.tfvars are the keys that reach the *VMs* -NOT the Proxmox VE provisioning key. Override any fleet defaults there too. +Note: The `ssh_public_keys` in terraform.tfvars are the keys that reach the +*VMs*. Override any fleet defaults there too. -## optel-lgtm startup +## otel-lgtm startup -- `cd optel-lgtm` +- `cd otel-lgtm` - `cp .env.example .env` - `docker compose up -d` @@ -201,55 +232,67 @@ NOT the Proxmox VE provisioning key. Override any fleet defaults there too. - **Add a VM:** create `inventory/.yaml` (the file name becomes the VM name and hostname — DNS label rules apply) and `tofu apply`. See - `inventory/example.yaml` for every knob including snapshot pinning and + `inventory-example.yaml` for every knob including snapshot pinning and ansible roles. -- **Remove a VM:** `git mv inventory/.yaml inventory/destroy/` and - `tofu apply` — only `inventory/*.yaml` is scanned, so the spec stays in the - repo while the VM, its disk, and its snippet are destroyed. Moving it back - provisions a fresh VM (all guest data is gone). To keep a VM but power it - off, set `started: false` instead. +- **Remove a VM:** `mv inventory/.yaml inventory/destroy/` and + `tofu apply` — only `inventory/*.yaml` is scanned, so the spec stays on disk + while the VM and its disk are destroyed. Moving it back provisions a fresh + VM (all guest data is gone). To keep a VM but power it off, set + `started: false` instead. (Plain `mv`: this template gitignores + `inventory/*.yaml`, so `git mv` fails. A fork that tracks its own specs can + use `git mv`.) - **Install software:** list roles in the spec (`ansible_roles: [docker, ...]`) and run `ansible-playbook ansible/site.yaml [--limit ]` — no plan diff, no recreation, idempotent (second run reports changed=0). Removing a role from the list does *not* uninstall it; clean removal is a rebuild. -- **After touching anything under `cloud-init/`:** run - `./scripts/check-cloud-init.sh`. **Under `ansible/`:** +- **After touching anything under `ansible/`:** run `./scripts/check-ansible.sh`. - **Prove a rebuild is identical:** `./scripts/vm-fingerprint.sh ubuntu@ fingerprints/.txt` captures the VM's environment (packages, apt config, enabled units, users, and the ansible-installed files — minus per-instance noise like host keys and - machine-id) into a tracked file. Capture, commit, destroy + reprovision, - re-run ansible, capture again to the same path: an empty `git diff` is the - proof. + machine-id). Capture, destroy + reprovision, re-run ansible, capture again + to the same path: an empty diff is the proof. `fingerprints/**` is + gitignored, so captures stay local unless you commit one deliberately. ## Fingerprints -Short version of the command sequence to destroy, recreate, and validate the fingerprint assuming the inventory entry is for a VM called ubuntu-test at an IP of 10.0.0.50. -This test assumes the first build fingerprint was saved as fingerprints/ubuntu-test-build1.txt. Rebuilding the same configuration should be an empty diff. +Short version of the command sequence to destroy, recreate, and validate the +fingerprint, assuming an inventory entry for a VM called ubuntu-test at +10.0.0.50. Capture to one path throughout and let `git diff` do the comparing; +rebuilding the same configuration should produce no diff. **Important**: Always start a session with `source tofu.env` ```sh source tofu.env -git mv inventory/ubuntu-test.yaml inventory/destroy/ +./scripts/vm-fingerprint.sh ubuntu@10.0.0.50 fingerprints/ubuntu-test.txt +cp fingerprints/ubuntu-test.txt /tmp/baseline.txt + +mv inventory/ubuntu-test.yaml inventory/destroy/ tofu apply -git mv inventory/destroy/ubuntu-test.yaml inventory/ +mv inventory/destroy/ubuntu-test.yaml inventory/ tofu apply ansible-playbook ansible/site.yaml --limit ubuntu-test -./scripts/vm-fingerprint.sh ubuntu@10.0.0.50 fingerprints/ubuntu-test-build2.txt -diff fingerprints/ubuntu-test-build1.txt fingerprints/ubuntu-test-build2.txt + +./scripts/vm-fingerprint.sh ubuntu@10.0.0.50 fingerprints/ubuntu-test.txt +diff /tmp/baseline.txt fingerprints/ubuntu-test.txt # empty = identical rebuild ``` -NOTE: The inventory/destroy directory name is intentional, so it is clear what the next "tofu apply" is expected to do. +`fingerprints/**` and `inventory/*.yaml` are both gitignored, so this uses +`cp`/`diff` and plain `mv` rather than `git add -f`/`git diff` and `git mv`. +A fork that tracks its specs and captures can use the git forms throughout. + +NOTE: The inventory/destroy directory name is intentional, so it is clear what +the next "tofu apply" is expected to do. ## Ansible Cloud-init is decided at first boot and reachable only by recreating the VM — -that is the right place for identity, network, and the apt baseline, and the -wrong place for software. Everything softer lives in `ansible/` and follows -the same declarative grammar as the rest of the repo: +that is the right place for identity and network, and the wrong place for +anything else. Everything softer lives in `ansible/` and follows the same +declarative grammar as the rest of the repo: - **Specs declare, roles implement.** `ansible_roles:` in a VM's YAML rides through the module into `tofu output -json vms`; @@ -258,6 +301,14 @@ the same declarative grammar as the rest of the repo: `site.yaml` is a single play that `include_role`s each host's declared list, so adding a role never touches it, and a typo'd role name fails at `tofu plan` time via a `fileexists()` validation in the spec contract. +- **The `base` role is implicit.** No spec lists it and none can opt out: + `site.yaml` applies it to every host via `roles:`, ahead of the include + loop. It owns the apt snapshot pin, the `APT::Periodic` zeros, the masked + apt-daily timers, key-only SSH, the timezone, and the spec's `packages:` — + the baseline that used to be cloud-init's, now re-appliable to a running VM. +- **Collections are pinned** in `ansible/requirements.yml`, and CI installs + from it. `ansible-core` ships none, so a `community.*` task without an entry + there passes locally and fails in CI. - **Role names use underscores** (`nats_server`, not `nats-server`): they double as Ansible group names, which must be valid identifiers. - **Every download is verified, every version pinned** in the role's @@ -281,22 +332,26 @@ the same declarative grammar as the rest of the repo: ## Credentials -Deliberately separate identities, one job each: +Two credentials, one job each: | Credential | Authenticates to | Used for | |---|---|---| -| API token (in `secrets.enc.json`) | ProxMox API | everything except snippet upload | -| `id_ed25519_pve` | `root@` | snippet upload only | +| API token (in `secrets.enc.json`) | ProxMox API | provisioning, and uploading the golden image | | `ssh_public_keys` (tfvars) | `@` | reaching the VMs | -The provisioning key is root on the hypervisor and exists only to upload YAML -files; keeping it out of the VMs means a compromised VM never saw the key that -owns the hypervisor. The API identity (`@pve` realm) is not a Linux account and -can never be the SSH identity — that split is structural, not a choice. +There is deliberately no hypervisor SSH credential. One used to exist, because +the ProxMox API has no snippets endpoint and the provider fell back to SSH as +root to upload cloud-init user-data. Removing the snippet removed the reason: +every call this project makes is now an API call carrying the token, and a +compromised workstation cannot reach root on the node through anything here. + +The API identity lives in the `@pve` realm, which is not a Linux account, so it +could never have been the SSH identity — that split was structural, and is now +simply absent. ## Protecting pre-existing VMs -Three plan-time layers in `guards.tofu` + `modules/vm/main.tofu` +Three plan-time layers in `guards.tofu` + `modules/vm-pve/main.tofu` (preconditions, not `check` blocks — they fail the plan rather than warn): 1. **VMID floor** — managed VMs live at or above `managed_vmid_min`. @@ -323,23 +378,28 @@ one guard OpenTofu genuinely cannot bypass. co-exist for different test setups. Baseline = manifest, delta = inventory `packages:`, delta versions = `archive_snapshot` — the whole environment is specified without booting anything. -- Ubuntu cloud images ship apt *sources* but not package *indexes*; cloud-init - refreshes indexes automatically whenever `packages:` is non-empty, so - installs work regardless of `package_update`/`package_upgrade`. +- Ubuntu cloud images ship apt *sources* but not package *indexes*; the base + role refreshes them (`update_cache: true`) before installing, so the spec's + `packages:` resolve against the pin. - **unattended-upgrades is disabled on every VM** (the image ships it - enabled). Base zeroes the `APT::Periodic` jobs via `bootcmd` and disables - the apt-daily timers at first boot — a test environment must not change - itself. Remove those lines from `base.yaml.tftpl` / - `base.runcmd.json.tftpl` if you *want* automatic security updates. -- `archive_snapshot:` writes `APT::Snapshot "";` to apt.conf.d via - `bootcmd` (init stage — before apt configures sources and installs packages; - re-applied every boot, so later manual `apt install` stays pinned). An - apt.conf.d file survives cloud-init regenerating `ubuntu.sources`. -- Changing a cloud-init template does **not** diff existing VMs (the snippet - ID is name-based and unchanged); template changes reach a VM only by - recreating it. + enabled). The base role zeroes the `APT::Periodic` jobs and *masks* the + apt-daily timers — masked rather than merely disabled, because a package + postinst re-running `systemctl preset` can undo a disable, and a test + environment must not change itself. Drop those tasks from + `ansible/roles/base/tasks/main.yaml` if you *want* automatic updates. +- `archive_snapshot:` writes `APT::Snapshot "";` to apt.conf.d from the + base role, before it installs anything, so the pin is in force for the + spec's own packages and for any later manual `apt install`. An apt.conf.d + file survives cloud-init regenerating `ubuntu.sources`. +- **What reaches a running VM, and what does not.** `packages:`, + `archive_snapshot:`, `ci_timezone` and every `ansible_roles` entry are + applied by ansible, so editing them and re-running the playbook is enough. + `package_upgrade` is the exception: it becomes PVE's `ciupgrade`, which + cloud-init reads on first boot only, so changing it reaches new VMs only. - The disk's image reference is create-only (`ignore_changes`): bumping - `cloud_image_file_id` affects new VMs, never existing ones. + `cloud_image_file_id` affects new VMs, never existing ones. That is also why + an uploaded image must never be replaced in place — same name, different + bytes, no plan diff. ## Rotating the state passphrase diff --git a/ansible/inventory/tofu.py b/ansible/inventory/tofu.py index f8ac991..98f771e 100755 --- a/ansible/inventory/tofu.py +++ b/ansible/inventory/tofu.py @@ -9,8 +9,10 @@ - one host per VM, named by inventory name (so --limit works) - group "vms" holding every reachable VM - one group per declared role, for ad-hoc targeting (ansible bun -m ...) - - hostvars: ansible_host (first agent-reported IPv4), ansible_user, - vm_id, vm_ansible_roles (what site.yaml applies) + - hostvars: ansible_host (the VM's own NIC, per the module output), + ansible_user, + vm_id, vm_ansible_roles (what site.yaml applies), and the three the base + role consumes: vm_packages, vm_archive_snapshot (may be None), vm_timezone A VM with no reported address (powered off, or agent not up yet) is skipped with a notice on stderr: an unreachable-by-design host in inventory would @@ -70,6 +72,11 @@ def build_inventory(): "ansible_user": vm["ansible_user"], "vm_id": vm["vm_id"], "vm_ansible_roles": vm.get("ansible_roles", []), + # Applied by the base role rather than at first boot, which is why + # they ride the inventory instead of the cloud-init drive. + "vm_packages": vm.get("packages", []), + "vm_archive_snapshot": vm.get("archive_snapshot"), + "vm_timezone": vm["timezone"], } inventory["vms"]["hosts"].append(name) for role in vm.get("ansible_roles", []): diff --git a/ansible/roles/base/handlers/main.yaml b/ansible/roles/base/handlers/main.yaml new file mode 100644 index 0000000..4aa5a96 --- /dev/null +++ b/ansible/roles/base/handlers/main.yaml @@ -0,0 +1,6 @@ +--- +- name: Reload sshd + become: true + ansible.builtin.systemd_service: + name: ssh + state: reloaded diff --git a/ansible/roles/base/tasks/main.yaml b/ansible/roles/base/tasks/main.yaml new file mode 100644 index 0000000..593b1ed --- /dev/null +++ b/ansible/roles/base/tasks/main.yaml @@ -0,0 +1,102 @@ +# Everything every VM gets, applied to every host before its spec-declared +# roles. This is what the cloud-init snippet used to do at first boot, moved +# to a layer that can be re-applied: change a VM's archive_snapshot, packages +# or the fleet timezone and the next ansible-playbook run reaches the running +# guest, instead of needing a rebuild. +# +# Not here, and deliberately: qemu-guest-agent. Apply blocks until the agent +# reports an address and the dynamic inventory reads that address from the +# agent, so ansible cannot be what installs it. It comes from the golden image +# (scripts/build-image.sh). + +- name: Apply the fleet baseline + become: true + block: + # Both apt.conf.d filenames below are the ones cloud-init wrote, kept + # verbatim. "cloudinit" is a misnomer now, but scripts/vm-fingerprint.sh + # records apt configuration by filename - renaming them would make every + # fingerprint captured so far incomparable for no gain. + - name: Pin apt to the spec's archive snapshot + ansible.builtin.copy: + dest: /etc/apt/apt.conf.d/50cloudinit-snapshot + content: | + APT::Snapshot "{{ vm_archive_snapshot }}"; + mode: "0644" + when: vm_archive_snapshot is not none + + - name: Remove the apt snapshot pin when the spec has none + ansible.builtin.file: + path: /etc/apt/apt.conf.d/50cloudinit-snapshot + state: absent + when: vm_archive_snapshot is none + + # The golden image ships this file too. Re-asserted here so a VM built + # from a stock cloud image is still safe, and so the fleet's stance + # survives anything that rewrites 20auto-upgrades. + - name: Zero the apt periodic jobs + ansible.builtin.copy: + dest: /etc/apt/apt.conf.d/51cloudinit-no-auto-upgrades + content: | + APT::Periodic::Update-Package-Lists "0"; + APT::Periodic::Unattended-Upgrade "0"; + mode: "0644" + + # Zeroing the periodic jobs already makes these no-ops; disabling and then + # masking them is the belt-and-braces version. Masked rather than merely + # disabled because a package postinst re-running `systemctl preset` can + # undo a disable, and a test environment must not change itself. + - name: Stop and disable the apt-daily timers + ansible.builtin.systemd_service: + name: "{{ item }}" + enabled: false + state: stopped + loop: + - apt-daily.timer + - apt-daily-upgrade.timer + + - name: Mask the apt-daily timers + ansible.builtin.systemd_service: + name: "{{ item }}" + masked: true + loop: + - apt-daily.timer + - apt-daily-upgrade.timer + + # Key-only SSH used to be cloud-init's doing (ssh_pwauth: false, which + # wrote 50-cloud-init.conf). PVE's generated user-data has no equivalent, + # so without this the guarantee would rest entirely on the cloud image + # happening to ship 60-cloudimg-settings.conf - true today, and not + # something this repo would notice losing. + # + # 10-, because sshd takes the FIRST value it obtains for a keyword and + # reads the include glob in lexical order: a lower number wins, which is + # the opposite of apt.conf.d. KbdInteractiveAuthentication is included + # because disabling PasswordAuthentication alone still leaves a PAM + # keyboard-interactive path to password login. + - name: Enforce key-only SSH + ansible.builtin.copy: + dest: /etc/ssh/sshd_config.d/10-smithy.conf + content: | + PasswordAuthentication no + KbdInteractiveAuthentication no + mode: "0644" + notify: Reload sshd + + # Validates the whole assembled config, not just the fragment above. It + # runs before the handler flushes, so a bad config fails the play with the + # running sshd still on its old configuration. + - name: Check the assembled sshd config is valid + ansible.builtin.command: /usr/sbin/sshd -t + changed_when: false + + - name: Set the timezone + community.general.timezone: + name: "{{ vm_timezone }}" + + # After the snapshot pin above, so the pin is in force for this install. + - name: Install the spec's packages + ansible.builtin.apt: + name: "{{ vm_packages }}" + state: present + update_cache: true + when: vm_packages | length > 0 diff --git a/ansible/site.yaml b/ansible/site.yaml index 3cd8111..aa96ed3 100644 --- a/ansible/site.yaml +++ b/ansible/site.yaml @@ -1,10 +1,17 @@ -# Applies each VM's spec-declared ansible_roles (inventory/.yaml -> -# tofu output -> inventory/tofu.py hostvars). Adding a role to a VM never -# touches this file; per-VM ordering follows the spec's list. A role name -# with no ansible/roles/ directory fails loudly here AND at tofu plan time - -# never a silent no-op. -- name: Apply per-VM ansible roles +# Applies the base role to every VM, then each VM's spec-declared +# ansible_roles (inventory/.yaml -> tofu output -> inventory/tofu.py +# hostvars). Adding a role to a VM never touches this file; per-VM ordering +# follows the spec's list. A role name with no ansible/roles/ directory fails +# loudly here AND at tofu plan time - never a silent no-op. +# +# base is not in any spec and cannot be opted out of: it carries what used to +# be the cloud-init snippet, so every VM needs it. roles: runs ahead of +# tasks:, which is what puts it before the include loop - the apt snapshot pin +# it writes has to be in force before any role installs a package. +- name: Apply the fleet baseline and each VM's declared roles hosts: vms + roles: + - base tasks: - name: Include each role the VM's spec declares ansible.builtin.include_role: diff --git a/cloud-init/base.runcmd.json.tftpl b/cloud-init/base.runcmd.json.tftpl deleted file mode 100644 index 480fdcc..0000000 --- a/cloud-init/base.runcmd.json.tftpl +++ /dev/null @@ -1,4 +0,0 @@ -[ - ${jsonencode("systemctl enable --now qemu-guest-agent")}, - ${jsonencode("systemctl disable --now apt-daily.timer apt-daily-upgrade.timer")} -] diff --git a/cloud-init/base.yaml.tftpl b/cloud-init/base.yaml.tftpl deleted file mode 100644 index eeb4421..0000000 --- a/cloud-init/base.yaml.tftpl +++ /dev/null @@ -1,64 +0,0 @@ -#cloud-config -# Rendered by OpenTofu from cloud-init/base.yaml.tftpl - do not edit on the node. -# -# Every VM gets this, and only this: post-boot software is Ansible's job -# (ansible/, layer 2), not cloud-init's. A cloud-config is a single YAML -# mapping - cloud-init silently drops one of any duplicated top-level pair - -# so nothing may ever be appended to this render. -# -# Every scalar interpolation is jsonencode()d: JSON is a YAML subset, and a raw -# value containing ': ', '#', or a YAML-1.1 boolean word (an SSH key comment, a -# hostname like 'no') would otherwise change the document's structure instead -# of failing loudly. -hostname: ${jsonencode(vm_name)} -fqdn: ${jsonencode(fqdn)} -prefer_fqdn_over_hostname: false -manage_etc_hosts: true -timezone: ${jsonencode(timezone)} - -users: - - name: ${jsonencode(ci_user)} - groups: [adm, sudo] - shell: /bin/bash - sudo: "ALL=(ALL) NOPASSWD:ALL" - lock_passwd: true - ssh_authorized_keys: -%{~ for key in ssh_public_keys } - - ${jsonencode(key)} -%{~ endfor } - -ssh_pwauth: false -disable_root: true - -# bootcmd runs in the init stage - before apt-configure and package install - -# and on every boot, so these hold for the VM's whole life. -bootcmd: -%{~ if archive_snapshot != null } - # Pin apt to the Ubuntu archive as of this instant (snapshot.ubuntu.com), - # for repeatable package installs. Third-party sources without snapshot - # support are fetched normally, unpinned. - - ${jsonencode("echo 'APT::Snapshot \"${archive_snapshot}\";' > /etc/apt/apt.conf.d/50cloudinit-snapshot")} -%{~ endif } - # The image ships unattended-upgrades enabled; a test environment must not - # change itself. Zeroing the periodic jobs makes apt-daily a no-op (the - # stock 20auto-upgrades enables them; apt reads conf.d in order, last wins). - - ${jsonencode("printf '%s\\n' 'APT::Periodic::Update-Package-Lists \"0\";' 'APT::Periodic::Unattended-Upgrade \"0\";' > /etc/apt/apt.conf.d/51cloudinit-no-auto-upgrades")} - -# package_update is explicit, not load-bearing: cloud-init refreshes the apt -# indexes whenever `packages:` is non-empty regardless of this setting. -package_update: true -package_upgrade: ${jsonencode(package_upgrade)} -packages: -%{~ for pkg in packages } - - ${jsonencode(pkg)} -%{~ endfor } - -growpart: - mode: auto - devices: ["/"] - ignore_growroot_disabled: false - -runcmd: -%{~ for cmd in runcmd } - - ${jsonencode(cmd)} -%{~ endfor } diff --git a/inventory-example.yaml b/inventory-example.yaml index 91e0e2c..bde1301 100644 --- a/inventory-example.yaml +++ b/inventory-example.yaml @@ -2,7 +2,7 @@ # # The file name is the VM name (must be a DNS label: lowercase letters, digits, # hyphens). Only vm_id is required; everything omitted takes the optional() -# default from modules/vm/variables.tofu, so a three-line file is a complete +# default from modules/vm-pve/variables.tofu, so a three-line file is a complete # VM. Deprovision by deleting this file and running `tofu apply`. vm_id: 500 diff --git a/modules/vm/main.tofu b/modules/vm-pve/main.tofu similarity index 57% rename from modules/vm/main.tofu rename to modules/vm-pve/main.tofu index b543951..da74aef 100644 --- a/modules/vm/main.tofu +++ b/modules/vm-pve/main.tofu @@ -1,58 +1,10 @@ -# modules/vm/main.tofu +# modules/vm-pve/main.tofu locals { - fqdn = var.spec.dns_domain == null ? var.vm_name : "${var.vm_name}.${var.spec.dns_domain}" - - # The VM resource blocks on the guest agent reporting an address, so the agent - # package is not optional. - ci_packages = distinct(concat(["qemu-guest-agent"], var.spec.packages)) - # Always applied, never left to inventory: the foreign-VM guard keys off it. # Sorted because ProxMox normalizes tag order; an unsorted list here would # read back differently and produce a perpetual diff. tags = sort(distinct(concat([var.managed_tag], var.spec.tags))) - - # The runcmd list lives in cloud-init/base.runcmd.json.tftpl rather than in - # HCL so that scripts/check-cloud-init.sh renders the exact same source this - # module does; a hand-copied literal in the check script is how commands - # escaped validation before. - runcmd = jsondecode( - templatefile("${var.cloud_init_dir}/base.runcmd.json.tftpl", { - ci_user = var.ci_user - vm_name = var.vm_name - }) - ) - - base_user_data = templatefile("${var.cloud_init_dir}/base.yaml.tftpl", { - vm_name = var.vm_name - fqdn = local.fqdn - timezone = var.ci_timezone - ci_user = var.ci_user - ssh_public_keys = var.ssh_public_keys - packages = local.ci_packages - runcmd = local.runcmd - archive_snapshot = var.spec.archive_snapshot - package_upgrade = var.spec.package_upgrade - }) - - user_data = trimspace(local.base_user_data) -} - -# Uploaded over SSH: the ProxMox API exposes no endpoint for snippets. -# -# The snippet's ID is derived from its (stable) file name, so editing a -# cloud-init template re-uploads the snippet but shows "0 to change" on the VM -# - the running guest keeps whatever it booted with. Template changes reach a -# VM only by recreating it (delete the inventory file, apply, restore, apply). -resource "proxmox_virtual_environment_file" "user_data" { - node_name = var.pve_node - datastore_id = var.snippet_datastore - content_type = "snippets" - - source_raw { - file_name = "${var.vm_name}-user-data.yaml" - data = local.user_data - } } resource "proxmox_virtual_environment_vm" "this" { @@ -89,14 +41,18 @@ resource "proxmox_virtual_environment_vm" "this" { scsi_hardware = "virtio-scsi-single" boot_order = ["scsi0"] - # Pointing file_id at the existing image makes ProxMox import and convert it - # into the target datastore; the source image is left untouched. file_id is - # ignore_changes'd below: it only matters at creation, and without that a - # bump of the fleet-wide cloud_image_file_id would diff - and destroy/create - # - every existing VM's disk at once. + # import_from, not file_id. Both make ProxMox import and convert the image + # into the target datastore and leave the source untouched, but file_id goes + # through the provider's "custom disk" path, which shells out to `qm` over + # SSH as root - the very dependency this design removes. import_from uses + # PVE's native import-from API parameter instead. + # + # It is create-only (the provider ignores later changes) and ignore_changes'd + # anyway: without that, bumping the fleet-wide cloud_image_file_id would diff + # - and destroy/create - every existing VM's disk at once. disk { datastore_id = var.disk_datastore - file_id = var.cloud_image_file_id + import_from = var.cloud_image_file_id interface = "scsi0" size = var.spec.disk_gb discard = "on" @@ -104,10 +60,29 @@ resource "proxmox_virtual_environment_vm" "this" { iothread = true } + # PVE's native cloud-init: these are VM config, written to the ci drive by the + # node, so there is no snippet and no SSH to the hypervisor. Everything softer + # - packages, timezone, apt pinning - is ansible's job (layer 2). + # + # Unlike the snippet this replaced, changing a value here DOES diff and PVE + # regenerates the drive. That still does not reach a running guest: cloud-init + # consumes the drive on first boot only. Rotating a key on a live VM means + # ansible or a rebuild. initialization { - datastore_id = var.disk_datastore - interface = "ide2" - user_data_file_id = proxmox_virtual_environment_file.user_data.id + datastore_id = var.disk_datastore + interface = "ide2" + + # PVE fills in the rest from the distro default user (`users: - default`), + # which is where sudo, lock_passwd and the group list come from. + user_account { + username = var.ci_user + keys = var.ssh_public_keys + } + + # Stated explicitly because PVE defaults ciupgrade to 1 and the provider's + # schema default is Computed - leaving it unset is not a pin, and the + # default is the opposite of what spec.package_upgrade defaults to. + upgrade = var.spec.package_upgrade ip_config { ipv4 { @@ -143,7 +118,7 @@ resource "proxmox_virtual_environment_vm" "this" { # Three layers of protection for the VMs that predate this project. These fail # the plan, unlike a check block, which would only warn. lifecycle { - ignore_changes = [disk[0].file_id] + ignore_changes = [disk[0].import_from] precondition { condition = !contains(var.duplicate_vm_ids, var.spec.vm_id) diff --git a/modules/vm-pve/outputs.tofu b/modules/vm-pve/outputs.tofu new file mode 100644 index 0000000..15e5129 --- /dev/null +++ b/modules/vm-pve/outputs.tofu @@ -0,0 +1,65 @@ +# modules/vm-pve/outputs.tofu + +output "vm_id" { + description = "VMID of the provisioned VM." + value = proxmox_virtual_environment_vm.this.vm_id +} + +output "name" { + description = "Name of the provisioned VM." + value = proxmox_virtual_environment_vm.this.name +} + +locals { + # The guest agent reports one address list per interface, and + # ipv4_addresses / mac_addresses / network_interface_names share that + # ordering. Select by the MAC of the NIC this module created, rather than + # flattening every interface and filtering what is obviously not wanted. + # + # Flattening was wrong as soon as a role installed docker: the guest grew a + # docker0 at 172.17.0.1, and the ansible inventory takes element 0 as + # ansible_host. It happens to still be right here because the agent lists + # eth0 first, which is luck, not a guarantee. + # + # Matching the MAC rather than filtering addresses or names is what makes + # this robust: docker's bridge address is configurable (daemon.json bip), + # podman/libvirt/CNI each bring their own ranges, and interface names vary + # by image (eth0, ens18, enp0s18, br-*, veth*). None of them can present the + # MAC ProxMox assigned to net0. + # + # A for-expression rather than index(): it yields an empty list when the + # agent has not reported yet, where index() would raise. The inventory + # already treats "no address" as skip-with-a-notice. + nic_ipv4 = flatten([ + for i, mac in proxmox_virtual_environment_vm.this.mac_addresses : + proxmox_virtual_environment_vm.this.ipv4_addresses[i] + if lower(mac) == lower(proxmox_virtual_environment_vm.this.network_device[0].mac_address) + ]) +} + +output "ipv4_addresses" { + description = "IPv4 addresses the guest agent reports for this VM's own NIC. Excludes loopback and any bridge a layer-2 role brings with it." + value = local.nic_ipv4 +} + +output "ansible_roles" { + description = "Ansible roles the spec declares for this VM (layer 2)." + value = var.spec.ansible_roles +} + +# Passed through to the ansible inventory, which is what applies them now that +# there is no cloud-init snippet. +output "packages" { + description = "apt packages the spec declares for this VM (layer 2)." + value = var.spec.packages +} + +output "archive_snapshot" { + description = "snapshot.ubuntu.com ID the spec pins apt to, or null (layer 2)." + value = var.spec.archive_snapshot +} + +output "ssh_command" { + description = "Ready-to-run SSH command for the cloud-init user." + value = format("ssh %s@%s", var.ci_user, try(local.nic_ipv4[0], "")) +} diff --git a/modules/vm/variables.tofu b/modules/vm-pve/variables.tofu similarity index 76% rename from modules/vm/variables.tofu rename to modules/vm-pve/variables.tofu index 0d872e0..1b4c0a0 100644 --- a/modules/vm/variables.tofu +++ b/modules/vm-pve/variables.tofu @@ -1,4 +1,4 @@ -# modules/vm/variables.tofu +# modules/vm-pve/variables.tofu variable "vm_name" { description = "VM name, also used as the guest hostname. Comes from the inventory file name." @@ -28,21 +28,27 @@ variable "spec" { dns_servers = optional(list(string), []) dns_domain = optional(string) vlan_id = optional(number) - packages = optional(list(string), []) tags = optional(list(string), []) - started = optional(bool, true) - on_boot = optional(bool, false) - # snapshot.ubuntu.com ID (YYYYMMDDTHHMMSSZ). When set, apt on the guest is - # pinned to the archive as of that instant for the VM's whole life, so - # package installs are reproducible. Third-party repos (e.g. docker) have - # no snapshot service and are not covered. + # apt packages, installed by the ansible base role rather than by + # cloud-init, so editing this list reaches an existing VM on the next + # ansible-playbook run. Removing an entry does not uninstall it. + packages = optional(list(string), []) + started = optional(bool, true) + on_boot = optional(bool, false) + + # snapshot.ubuntu.com ID (YYYYMMDDTHHMMSSZ). When set, the ansible base + # role pins apt on the guest to the archive as of that instant, so package + # installs are reproducible. Third-party repos (e.g. docker) have no + # snapshot service and are not covered. Independent of the snapshot the + # golden image itself was built against. archive_snapshot = optional(string) + # Drives PVE's ciupgrade, i.e. cloud-init's package_upgrade on first boot. # Default false: with a fixed base image, skipping the upgrade is what # keeps environments identical over time (and first boot much faster). - # true is also deterministic when archive_snapshot is set - the base - # upgrades to the pinned date instead of to "now". + # Note this one is first-boot only - unlike packages above, flipping it + # later does not reach an existing VM. package_upgrade = optional(bool, false) # Ansible roles under ansible/roles/ applied post-boot (layer 2). Unlike @@ -67,6 +73,13 @@ variable "spec" { error_message = "spec.ipv4 must be 'dhcp' or CIDR notation like 10.0.0.50/24; a bare address would pass the plan and fail at the API." } + validation { + condition = alltrue([ + for p in var.spec.packages : can(regex("^[a-z0-9][a-z0-9+._-]*$", p)) + ]) + error_message = "each spec.packages entry must be a plain apt package name (lowercase letters, digits, and + . _ -). Anything else is a typo or an attempt to smuggle apt arguments through the inventory." + } + validation { condition = var.spec.archive_snapshot == null || can(regex("^[0-9]{8}T[0-9]{6}Z$", var.spec.archive_snapshot)) error_message = "spec.archive_snapshot must be a snapshot.ubuntu.com ID like 20260801T000000Z (UTC, YYYYMMDDTHHMMSSZ)." @@ -91,11 +104,6 @@ variable "spec" { } } -variable "cloud_init_dir" { - description = "Directory holding the cloud-init templates. Passed in so the module does not reach out with ../.." - type = string -} - variable "ansible_dir" { description = "Directory holding ansible/roles/. Passed in so the module does not reach out with ../.." type = string @@ -107,12 +115,7 @@ variable "pve_node" { } variable "cloud_image_file_id" { - description = "Datastore reference to the cloud image used as the disk source." - type = string -} - -variable "snippet_datastore" { - description = "Datastore holding cloud-init snippets. Must have the 'snippets' content type." + description = "Datastore reference to the golden cloud image used as the disk source, built by scripts/build-image.sh. It carries qemu-guest-agent, without which apply blocks until the 30m agent timeout." type = string } @@ -137,12 +140,7 @@ variable "ssh_public_keys" { } variable "ci_user" { - description = "Login account created by cloud-init." - type = string -} - -variable "ci_timezone" { - description = "Timezone applied by cloud-init." + description = "Login account. Becomes PVE's ciuser, which cloud-init renames the image's default user to." type = string } diff --git a/modules/vm/versions.tofu b/modules/vm-pve/versions.tofu similarity index 92% rename from modules/vm/versions.tofu rename to modules/vm-pve/versions.tofu index bf249b6..f9243d9 100644 --- a/modules/vm/versions.tofu +++ b/modules/vm-pve/versions.tofu @@ -1,4 +1,4 @@ -# modules/vm/versions.tofu +# modules/vm-pve/versions.tofu # # A child module must name its provider source; without this OpenTofu assumes # hashicorp/proxmox and init fails. diff --git a/modules/vm/outputs.tofu b/modules/vm/outputs.tofu deleted file mode 100644 index 3b22efc..0000000 --- a/modules/vm/outputs.tofu +++ /dev/null @@ -1,39 +0,0 @@ -# modules/vm/outputs.tofu - -output "vm_id" { - description = "VMID of the provisioned VM." - value = proxmox_virtual_environment_vm.this.vm_id -} - -output "name" { - description = "Name of the provisioned VM." - value = proxmox_virtual_environment_vm.this.name -} - -output "ipv4_addresses" { - description = "Addresses reported by the guest agent, excluding loopback." - value = [ - for addr in flatten(proxmox_virtual_environment_vm.this.ipv4_addresses) : - addr if addr != "127.0.0.1" - ] -} - -output "ansible_roles" { - description = "Ansible roles the spec declares for this VM (layer 2)." - value = var.spec.ansible_roles -} - -output "ssh_command" { - description = "Ready-to-run SSH command for the cloud-init user." - value = format( - "ssh %s@%s", - var.ci_user, - try( - [ - for addr in flatten(proxmox_virtual_environment_vm.this.ipv4_addresses) : - addr if addr != "127.0.0.1" - ][0], - "", - ), - ) -} diff --git a/outputs.tofu b/outputs.tofu index 7ead3bc..8551a3b 100644 --- a/outputs.tofu +++ b/outputs.tofu @@ -8,12 +8,19 @@ output "nodes" { output "vms" { description = "Provisioned VMs, keyed by inventory name." value = { - for name, vm in module.vm : name => { + for name, vm in module.vm-pve : name => { vm_id = vm.vm_id ipv4_addresses = vm.ipv4_addresses ssh_command = vm.ssh_command ansible_roles = vm.ansible_roles ansible_user = var.ci_user + + # Layer 2 inputs: ansible/inventory/tofu.py turns these into hostvars, + # and the base role applies them. They are here rather than in the + # cloud-init drive because ansible can re-apply them to a running VM. + packages = vm.packages + archive_snapshot = vm.archive_snapshot + timezone = var.ci_timezone } } } diff --git a/providers.tofu b/providers.tofu index d75f046..e5c38c1 100644 --- a/providers.tofu +++ b/providers.tofu @@ -2,12 +2,6 @@ provider "sops" {} -locals { - # Bare host for the SSH node pin, derived from the endpoint so the address - # lives in exactly one place. - pve_host = regex("^https?://([^:/]+)", var.pve_endpoint)[0] -} - provider "proxmox" { endpoint = var.pve_endpoint api_token = local.proxmox_api_token @@ -17,15 +11,8 @@ provider "proxmox" { # the node's CA into this workstation's trust store is what removes it. insecure = true - ssh { - username = "root" - private_key = file(pathexpand(var.pve_ssh_private_key_path)) - - # Pin the node's address so uploads do not depend on "pve" resolving from - # the workstation. - node { - name = var.pve_node - address = local.pve_host - } - } + # No ssh block, deliberately. The provider only ever needed SSH to upload + # cloud-init snippets, which the API cannot do; with the snippet gone there + # is nothing left that is not an API call, and root on the hypervisor is no + # longer a credential this project holds. } diff --git a/scripts/build-image.sh b/scripts/build-image.sh new file mode 100755 index 0000000..22c1266 --- /dev/null +++ b/scripts/build-image.sh @@ -0,0 +1,320 @@ +#!/usr/bin/env bash +# Build the golden Ubuntu cloud image and upload it to the ProxMox node. +# +# The bpg provider can only upload cloud-init snippets over SSH (the PVE API +# has no snippets endpoint), which is what forced a root SSH key for the +# hypervisor into providers.tofu. Everything that snippet did can move to PVE's +# native cloud-init settings or to the ansible base role - except installing +# qemu-guest-agent, because apply blocks until the agent reports an address and +# the ansible inventory reads the address from the agent. Ansible therefore +# cannot be what installs it, so it is baked in here instead. +# +# Deliberately API-only: the finished image goes up through the storage upload +# endpoint, not scp. Losing the SSH path is the entire point of the exercise - +# which is also why it is uploaded as `import` content rather than `iso`. A VM +# disk sourced from an iso-content volume goes through the provider's "custom +# disk" path, which shells out to `qm` over SSH as root; PVE's native +# import-from only accepts `images` or `import` content. +# +# ./scripts/build-image.sh # build only +# ./scripts/build-image.sh --upload # build, then upload +# ./scripts/build-image.sh --serial 20260823 --upload +# +# The build needs no secrets. --upload decrypts the API token from +# secrets.enc.json with sops, so it needs the age key - but not `source +# tofu.env`, which is only about the state passphrase. +# +# The output is named for both the upstream serial and the archive snapshot it +# was built against, and an uploaded image is never overwritten: disk file_id +# is under ignore_changes in modules/vm-pve, so a new name is a no-op for +# running VMs while an in-place replacement would silently change what they +# were built from. Build a new name, repoint cloud_image_file_id, delete the +# old volume by hand once nothing references it. + +set -euo pipefail + +cd "$(dirname "$0")/.." + +# Same serial as the plain image already on the node. Bumping it is a separate +# change: a new serial moves every package version at once, which drowns any +# refactor being verified with scripts/vm-fingerprint.sh. +release="resolute" +serial="20260720" +snapshot="" +outdir="private/images" +datastore="local" +endpoint="" +node="" +do_upload=0 +force=0 + +usage="usage: build-image.sh [options] + + --serial YYYYMMDD upstream cloud image serial (default: $serial) + --snapshot STAMP snapshot.ubuntu.com timestamp to build against + (default: T000000Z) + --outdir DIR where to download and build (default: $outdir) + --upload upload the finished image to the node over the API + --datastore ID upload target datastore, must allow the 'import' + content type (default: $datastore) + --endpoint URL PVE API endpoint (default: variables.tofu's pve_endpoint) + --node NAME PVE node name (default: variables.tofu's pve_node) + --force rebuild an existing local image / overwrite on the node + -h, --help this" + +die() { echo "build-image: $*" >&2; exit 1; } +warn() { echo "build-image: warning: $*" >&2; } +step() { echo; echo "==> $*"; } + +while [ $# -gt 0 ]; do + case "$1" in + --serial) serial="${2:?$usage}"; shift 2 ;; + --snapshot) snapshot="${2:?$usage}"; shift 2 ;; + --outdir) outdir="${2:?$usage}"; shift 2 ;; + --datastore) datastore="${2:?$usage}"; shift 2 ;; + --endpoint) endpoint="${2:?$usage}"; shift 2 ;; + --node) node="${2:?$usage}"; shift 2 ;; + --upload) do_upload=1; shift ;; + --force) force=1; shift ;; + -h|--help) echo "$usage"; exit 0 ;; + *) die "unknown argument: $1"$'\n'"$usage" ;; + esac +done + +[[ "$serial" =~ ^[0-9]{8}(\.[0-9]+)?$ ]] || die "--serial must look like YYYYMMDD (got '$serial')" +: "${snapshot:=${serial%%.*}T000000Z}" +[[ "$snapshot" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || die "--snapshot must be YYYYMMDDTHHMMSSZ (got '$snapshot')" + +base_url="https://cloud-images.ubuntu.com/${release}/${serial}" +upstream_name="${release}-server-cloudimg-amd64.img" +cached="${outdir}/${release}-server-cloudimg-amd64-${serial}.img" +# .qcow2, not .img: the image really is qcow2, and PVE only accepts +# .ova/.qcow2/.raw/.vmdk for an `import` upload. The extension is load-bearing. +golden_name="${release}-server-cloudimg-amd64-${serial}-golden-${snapshot}.qcow2" +golden="${outdir}/${golden_name}" +volid="${datastore}:import/${golden_name}" + +# --------------------------------------------------------------------------- +# Preflight. Every failure names its own remedy - this script is run rarely +# enough that nobody remembers the setup. +# --------------------------------------------------------------------------- +step "Preflight" + +command -v virt-customize >/dev/null || die "virt-customize not found; apt install libguestfs-tools" +command -v curl >/dev/null || die "curl not found" +command -v sha256sum >/dev/null || die "sha256sum not found" + +# libguestfs boots its appliance with the host kernel, which Ubuntu ships mode +# 0600. Without the statoverride every guestfs call fails to launch. +kernel="/boot/vmlinuz-$(uname -r)" +[ -r "$kernel" ] || die "$kernel is not readable; run: + sudo dpkg-statoverride --update --add root root 0644 $kernel" + +# Not fatal, just slow: libguestfs silently falls back to TCG emulation, which +# turns a two-minute --install into a very long one. +if [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then + warn "/dev/kvm is not accessible - the build will run under TCG emulation and be slow. + Fix with: sudo usermod -aG kvm $USER (then log out and back in)" +fi + +mkdir -p "$outdir" + +# The upstream image is ~860 MB and the golden copy grows past 1 GB while apt +# runs inside it. Only count the download if it is not already cached. +need_kb=$((1536 * 1024)) +[ -f "$cached" ] || need_kb=$((need_kb + 900 * 1024)) +avail_kb="$(df -Pk "$outdir" | awk 'NR==2 {print $4}')" +[ "$avail_kb" -ge "$need_kb" ] || die "only $((avail_kb / 1024)) MB free in $outdir, need ~$((need_kb / 1024)) MB. + Note /tmp is a tmpfs here - building there spends RAM, not disk." + +echo "ok virt-customize, readable $kernel, $((avail_kb / 1024)) MB free in $outdir" + +# --------------------------------------------------------------------------- +# Download and verify. The dated serial directory, never current/, which is a +# moving target: a golden image has to be reproducible from its own filename. +# --------------------------------------------------------------------------- +step "Fetching ${release}/${serial}" + +if [ -f "$cached" ]; then + echo "ok already downloaded: $cached" +else + curl -fSL --retry 3 -o "${cached}.part" "${base_url}/${upstream_name}" \ + || die "download failed: ${base_url}/${upstream_name} + Serial ${serial} may have aged off the mirror; check https://cloud-images.ubuntu.com/${release}/" + mv "${cached}.part" "$cached" +fi + +expected="$(curl -fsSL "${base_url}/SHA256SUMS" | awk -v n="$upstream_name" '$2 == "*" n || $2 == n {print $1}')" +[ -n "$expected" ] || die "no $upstream_name line in ${base_url}/SHA256SUMS" +actual="$(sha256sum "$cached" | awk '{print $1}')" +[ "$actual" = "$expected" ] || die "checksum mismatch for $cached + expected $expected + actual $actual + Delete it and re-run." +echo "ok sha256 $actual" + +# --------------------------------------------------------------------------- +# Customise. virt-customize applies operations in command-line order. +# --------------------------------------------------------------------------- +step "Building $golden_name" + +if [ -f "$golden" ] && [ "$force" -eq 0 ]; then + echo "ok already built: $golden (--force to rebuild)" +else + # Build under .part and rename only on success, so an aborted build is never + # mistaken for a finished one on the next run. + rm -f "$golden" "${golden}.part" + cp --reflink=auto "$cached" "${golden}.part" + + # The image's root filesystem is 2.2 GB with ~366 MB free, and apt's index + # for universe alone is ~120 MB unpacked. Two things follow, and skipping + # either one fills the guest disk mid-unpack: + # + # - Fetch ONE index set, not two. APT::Snapshot adds the snapshot mirror + # alongside the configured one and refreshes both, so the sources file is + # rewritten to point at snapshot.ubuntu.com instead, and restored after. + # Same effect - the agent comes from the archive as of $snapshot - at + # half the disk cost, and the pin cannot leak into the built image. + # - Fetch only Packages. Translations, DEP-11 app-stream metadata and + # command-not-found indexes are pure waste in a headless image. + # + # The pin must NOT survive into the image under any spelling: baked into + # apt.conf.d it would override every VM's own archive_snapshot, and left in + # ubuntu.sources it would freeze the whole fleet at the build's snapshot. + # Hence the restore, and the grep that fails the build if it did not happen. + # + # There is deliberately no `systemctl enable qemu-guest-agent`: the unit ships + # an empty [Install] section, so enabling it is a no-op. It is started by + # /usr/lib/udev/rules.d/60-qemu-guest-agent.rules when the virtio-serial port + # org.qemu.guest_agent.0 appears. That rule is the whole reason apply + # terminates - without the agent it blocks for the full 30m timeout waiting + # for an address - so its presence is asserted rather than assumed. + # + # The 51cloudinit-no-auto-upgrades filename below is the one cloud-init used + # to write. It is kept verbatim - "cloudinit" is a misnomer now, but + # scripts/vm-fingerprint.sh records apt config by filename, so renaming it + # would make every fingerprint captured so far incomparable. + LIBGUESTFS_BACKEND=direct virt-customize -a "${golden}.part" \ + --run-command 'cp /etc/apt/sources.list.d/ubuntu.sources /root/ubuntu.sources.orig' \ + --run-command "sed -i 's|^URIs:.*|URIs: https://snapshot.ubuntu.com/ubuntu/${snapshot}|' /etc/apt/sources.list.d/ubuntu.sources" \ + --write '/etc/apt/apt.conf.d/50build-lean:Acquire::Languages "none"; +Acquire::IndexTargets::deb::Translations::DefaultEnabled "false"; +Acquire::IndexTargets::deb::DEP-11::DefaultEnabled "false"; +Acquire::IndexTargets::deb::CNF::DefaultEnabled "false";' \ + --run-command 'rm -rf /var/lib/apt/lists/*' \ + --install qemu-guest-agent \ + --run-command 'mv /root/ubuntu.sources.orig /etc/apt/sources.list.d/ubuntu.sources' \ + --delete /etc/apt/apt.conf.d/50build-lean \ + --run-command 'grep -q archive.ubuntu.com /etc/apt/sources.list.d/ubuntu.sources' \ + --run-command 'test -f /usr/lib/udev/rules.d/60-qemu-guest-agent.rules' \ + --run-command 'systemctl disable apt-daily.timer apt-daily-upgrade.timer || true' \ + --write '/etc/apt/apt.conf.d/51cloudinit-no-auto-upgrades:APT::Periodic::Update-Package-Lists "0"; +APT::Periodic::Unattended-Upgrade "0";' \ + --run-command 'apt-get clean' \ + --run-command 'rm -rf /var/lib/apt/lists/*' \ + --truncate /etc/machine-id \ + || die "virt-customize failed; the partial image is at ${golden}.part" + + mv "${golden}.part" "$golden" +fi + +golden_sha="$(sha256sum "$golden" | awk '{print $1}')" + +# --------------------------------------------------------------------------- +# Upload over the API. +# --------------------------------------------------------------------------- +if [ "$do_upload" -eq 1 ]; then + step "Uploading to $volid" + + command -v sops >/dev/null || die "sops not found; needed to decrypt the API token" + + # variables.tofu is a symlink to site-specific values outside the repo, so + # read the defaults out of it rather than hardcoding an address here. + tofu_default() { + awk -v name="$1" ' + $0 ~ "^variable[[:space:]]+\"" name "\"" { inblock = 1; next } + inblock && /^}/ { exit } + inblock && /^[[:space:]]*default[[:space:]]*=/ { + sub(/^[^=]*=[[:space:]]*/, ""); gsub(/"/, ""); sub(/[[:space:]]+$/, "") + print; exit + } + ' variables.tofu + } + : "${endpoint:=$(tofu_default pve_endpoint)}" + : "${node:=$(tofu_default pve_node)}" + [ -n "$endpoint" ] || die "could not read pve_endpoint from variables.tofu; pass --endpoint" + [ -n "$node" ] || die "could not read pve_node from variables.tofu; pass --node" + endpoint="${endpoint%/}" + + token_id="$(sops -d --extract '["proxmox"]["api_token_id"]' secrets.enc.json)" \ + || die "sops decryption failed; is the age key at ~/.config/sops/age/keys.txt?" + token_secret="$(sops -d --extract '["proxmox"]["api_token_secret"]' secrets.enc.json)" \ + || die "sops decryption failed" + [ -n "$token_id" ] && [ -n "$token_secret" ] || die "sops returned an empty API token" + auth="Authorization: PVEAPIToken=${token_id}=${token_secret}" + + # -k mirrors providers.tofu's `insecure = true`: the node serves a + # self-signed certificate. The token rides on every one of these calls. + api() { curl -k -fsS -H "$auth" "$@"; } + + if api "${endpoint}/api2/json/nodes/${node}/storage/${datastore}/content?content=import" \ + | grep -q "\"volid\":\"${volid}\""; then + if [ "$force" -eq 0 ]; then + die "$volid already exists on the node. + Never replace an uploaded image in place - VMs pin their disk source by + file_id under ignore_changes, so a same-name replacement changes what a + running VM was built from without any plan diff. Build a new snapshot or + serial instead. --force overrides." + fi + warn "$volid exists and --force was given; overwriting" + fi + + # filename=@ MUST be the last -F. PVE parses the multipart body in order and + # streams everything after the file part straight into the file, so any field + # placed after it is both silently ignored as a parameter and appended to the + # image as garbage - which qemu-img happily tolerates, so nothing complains. + # With the order below, checksum is a real parameter and the node verifies + # the upload itself; the task fails on mismatch. + echo " $(du -h "$golden" | awk '{print $1}') to ${node}/${datastore} - this takes a few minutes" + upid="$(api --max-time 1800 \ + -F content=import \ + -F checksum-algorithm=sha256 \ + -F "checksum=${golden_sha}" \ + -F "filename=@${golden}" \ + "${endpoint}/api2/json/nodes/${node}/storage/${datastore}/upload" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["data"])')" \ + || die "upload failed" + echo " task $upid" + + # The upload endpoint returns as soon as the task is queued; the node is + # still writing (and checksumming) after that. + while :; do + status_json="$(api "${endpoint}/api2/json/nodes/${node}/tasks/${upid}/status")" + read -r task_status task_exit <<<"$(printf '%s' "$status_json" | python3 -c ' +import json, sys +d = json.load(sys.stdin)["data"] +print(d.get("status", ""), d.get("exitstatus", ""))')" + [ "$task_status" = "stopped" ] && break + sleep 5 + done + [ "$task_exit" = "OK" ] || die "upload task finished with: ${task_exit:-} + Check the node: pvenode task log $upid" + echo "ok uploaded" +fi + +# --------------------------------------------------------------------------- +step "Done" +cat < ${datastore} -> Import -> Upload" +fi diff --git a/scripts/check-ansible.sh b/scripts/check-ansible.sh index fada8eb..2b95400 100755 --- a/scripts/check-ansible.sh +++ b/scripts/check-ansible.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash # Validate the ansible layer: the dynamic inventory's shape and the site -# playbook's syntax. Sibling of check-cloud-init.sh - run it after touching -# anything under ansible/. +# playbook's syntax. Run it after touching anything under ansible/. # # The inventory check needs the tofu state (that is where the fleet lives), # so this requires `source tofu.env` first, same as everything else here. @@ -36,7 +35,8 @@ hostvars = inv.get("_meta", {}).get("hostvars", {}) if not hostvars: sys.exit("no reachable VMs in inventory - is the fleet applied and running?") for host, hv in hostvars.items(): - for key in ("ansible_host", "ansible_user", "vm_ansible_roles"): + for key in ("ansible_host", "ansible_user", "vm_ansible_roles", + "vm_packages", "vm_archive_snapshot", "vm_timezone"): if key not in hv: sys.exit(f"{host}: hostvar {key!r} missing") for role in hv["vm_ansible_roles"]: diff --git a/scripts/check-cloud-init.sh b/scripts/check-cloud-init.sh deleted file mode 100755 index db71e4e..0000000 --- a/scripts/check-cloud-init.sh +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env bash -# Validate the cloud-init user-data this configuration emits. -# -# `tofu validate` checks HCL and never sees the YAML that comes out of -# templatefile(), so this closes that gap. It renders base.yaml.tftpl with the -# real runcmd list (from base.runcmd.json.tftpl, the same source the module -# reads) and checks three things: -# -# 1. the result parses as YAML with no duplicate mapping keys anywhere -# (cloud-init silently drops one of a duplicated pair) -# 2. it passes `cloud-init schema`, which catches deprecated and malformed keys -# 3. deprecation warnings are treated as failures, not warnings -# -# It then renders one adversarial combination - an SSH key comment containing -# ': ', a package containing '#', a hostname that is a YAML boolean word - and -# asserts the values survive as strings. That is the regression test for the -# jsonencode() escaping in base.yaml.tftpl: without it these inputs silently -# change the document instead of failing. -# -# Requires: python3-yaml, cloud-init, an init'ed tofu. - -set -euo pipefail - -cd "$(dirname "$0")/.." - -if ! python3 -c 'import yaml' 2>/dev/null; then - echo "missing PyYAML: sudo apt install python3-yaml" >&2 - exit 1 -fi -if ! command -v cloud-init >/dev/null; then - echo "missing cloud-init: sudo apt install cloud-init" >&2 - exit 1 -fi -if ! command -v tofu >/dev/null; then - echo "missing tofu" >&2 - exit 1 -fi - -# tofu console evaluates the whole config. Before any state exists a dummy -# passphrase satisfies it, but once terraform.tfstate is on disk the real one -# is required to decrypt it - fail with a useful message instead of a cryptic -# state-encryption error. -if [ -e terraform.tfstate ] && [ -z "${TF_VAR_state_passphrase:-}" ]; then - echo "terraform.tfstate exists; run 'source tofu.env' first so tofu console can decrypt it" >&2 - exit 1 -fi -export TF_LOG= -export TF_VAR_state_passphrase="${TF_VAR_state_passphrase:-render-only-passphrase-unused}" -export TF_VAR_ssh_public_keys="${TF_VAR_ssh_public_keys:-[\"ssh-ed25519 AAAArenderonly check@local\"]}" - -workdir="$(mktemp -d)" -trap 'rm -rf "$workdir"' EXIT - -# Renders an HCL expression to stdout. jsonencode plus console's own string -# quoting means the result is double-encoded, hence the two decode passes. -# -lock=false: once a state file exists, console prints "Acquiring state -# lock..." to STDOUT, which would corrupt the pipeline; a read-only render -# needs no lock. The python side additionally keeps only the quoted result -# line in case a future tofu adds other stdout notices. -# On failure the diagnostics go to stderr (stdout is usually redirected into -# the output file) and the caller decides whether to continue. -render() { - printf '%s\n' "$1" | tofu console -lock=false 2>"$workdir/err" \ - | python3 -c 'import json,sys; line=next(l for l in sys.stdin.read().splitlines() if l.lstrip().startswith("\"")); print(json.loads(json.loads(line)), end="")' \ - || { echo "render failed:" >&2; cat "$workdir/err" >&2; return 1; } -} - -# The runcmd list a VM actually gets, built from the same file -# modules/vm/main.tofu reads. -runcmd_expr() { - printf '%s' 'jsondecode(templatefile("./cloud-init/base.runcmd.json.tftpl", { ci_user = "ubuntu", vm_name = "checkvm" }))' -} - -# base_expr -# The first three are literal strings, injected as HCL string literals; the -# last three are raw HCL (snapshot_expr is `null` or a quoted timestamp). -base_expr() { - cat <&1 | grep -Ev 'log_util|schema\.py' || true)" - if ! grep -q '^Valid schema' <<<"$schema_out"; then - echo "FAIL ${label}: ${schema_out}" - return 1 - elif grep -qi 'deprecat' <<<"$schema_out"; then - echo "FAIL ${label}: deprecation: ${schema_out}" - return 1 - fi - return 0 -} - -status=0 -label="base" -out="$workdir/base.yaml" - -if ! render "$(base_expr checkvm "ssh-ed25519 AAAA check@local" qemu-guest-agent "$(runcmd_expr)" null false)" >"$out"; then - echo "FAIL ${label}: render (base)" - status=1 -elif ! check_yaml "$out"; then - echo "FAIL ${label}: YAML/duplicate-key check" - status=1 -elif check_schema "$out" "$label"; then - echo "ok ${label}" -else - status=1 -fi - -# Adversarial pass: values shaped to break unescaped YAML interpolation, plus -# the snapshot pin and package_upgrade=true so the conditional bootcmd block -# gets schema coverage. -adv_key='ssh-ed25519 AAAA check@local: laptop key' -adv_pkg='foo # bar' -adv_name='no' -adv_snapshot='20260801T000000Z' -out="$workdir/adversarial.yaml" -if ! render "$(base_expr "$adv_name" "$adv_key" "$adv_pkg" "$(runcmd_expr)" "\"$adv_snapshot\"" true)" >"$out"; then - echo "FAIL : render" - status=1 -elif ! check_yaml "$out"; then - echo "FAIL : YAML/duplicate-key check" - status=1 -elif ! python3 - "$out" "$adv_name" "$adv_key" "$adv_pkg" "$adv_snapshot" <<'PY' -import sys, yaml -path, name, key, pkg, snapshot = sys.argv[1:6] -with open(path) as f: - doc = yaml.safe_load(f) -assert doc["hostname"] == name, f"hostname mangled: {doc['hostname']!r}" -keys = doc["users"][0]["ssh_authorized_keys"] -assert key in keys, f"ssh key mangled: {keys!r}" -assert pkg in doc["packages"], f"package mangled: {doc['packages']!r}" -bootcmd = doc.get("bootcmd") -assert isinstance(bootcmd, list) and bootcmd and all(isinstance(c, str) for c in bootcmd), \ - f"bootcmd not a list of strings: {bootcmd!r}" -assert any(snapshot in c for c in bootcmd), f"snapshot missing from bootcmd: {bootcmd!r}" -assert any("Unattended-Upgrade" in c for c in bootcmd), \ - f"unattended-upgrades disable missing from bootcmd: {bootcmd!r}" -assert doc["package_upgrade"] is True, f"package_upgrade mangled: {doc['package_upgrade']!r}" -PY -then - echo "FAIL : a hostile value did not survive as a string - check jsonencode() in base.yaml.tftpl" - status=1 -elif check_schema "$out" ""; then - echo "ok " -else - status=1 -fi - -exit "$status" diff --git a/scripts/vm-fingerprint.sh b/scripts/vm-fingerprint.sh index f140b98..d422301 100755 --- a/scripts/vm-fingerprint.sh +++ b/scripts/vm-fingerprint.sh @@ -227,7 +227,19 @@ section "base image" [ -e /etc/cloud/build.info ] && cat /etc/cloud/build.info section "cloud-init" -cloud-init status +# NOT a bare `cloud-init status`: it exits 2 when the boot finished with +# recoverable errors ("degraded done"), and under the set -eu at the top of +# this remote script that silently aborted the whole capture. Degraded is the +# steady state on the proxmox path, because the user-data PVE generates still +# uses the top-level `user:` key, deprecated in cloud-init 22.2. Nothing in +# this repo emits that key and nothing here can suppress it. +# +# So record the long form, which names the degradation, rather than the bare +# status line, which says "done" either way. If PVE ever stops emitting the +# deprecated key, that shows up here as a diff instead of looking identical. +# last_update is dropped: it is a timestamp, and this capture has to be +# byte-stable across rebuilds. +cloud-init status --long | grep -vE "^last_update:" || true # v1.platform is the datasource discriminator: nocloud on the proxmox path, # ec2 on the aws one. Verified against a real VM rather than guessed - the # first version of this line asked for v1.datasource, which is not a key. diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 25872de..07be0dd 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -5,27 +5,26 @@ # # Per-VM settings do NOT belong here - they live in inventory/*.yaml. -# Keys authorized for the cloud-init user on every VM. NOT the provisioning -# key: id_ed25519_pve is root on the hypervisor and is deliberately kept out -# of the VMs. See README.md, "Credentials". +# Keys authorized for the cloud-init user on every VM. With the hypervisor +# provisioning key gone, these and the API token are the whole of what this +# project authenticates with. See README.md, "Credentials". ssh_public_keys = [ "ssh-ed25519 AAAA...replace-me... user@desktop", # workstation "ssh-ed25519 BBBB...replace-me... user@provisioner", # box running tofu ] -# pve_node = "pve" -# pve_endpoint = "https://10.0.0.10:8006/" -# pve_ssh_private_key_path = "~/.ssh/id_ed25519_pve" -# disk_datastore = "local-lvm" -# snippet_datastore = "local" -# network_bridge = "vmbr0" -# ci_user = "ubuntu" -# ci_timezone = "Etc/UTC" +# pve_node = "pve" +# pve_endpoint = "https://10.0.0.10:8006/" +# disk_datastore = "local-lvm" +# network_bridge = "vmbr0" +# ci_user = "ubuntu" +# ci_timezone = "Etc/UTC" -# The base image every VM's disk is imported from. This is the line to change -# when a new Ubuntu release ships; existing VMs keep the image they were -# created from (the disk source is create-only), only new VMs pick it up. -# cloud_image_file_id = "local:iso/resolute-server-cloudimg-amd64-20260720.img" +# The golden image every VM's disk is imported from, built and uploaded by +# scripts/build-image.sh. This is the line to change after building a new one; +# existing VMs keep the image they were created from (the disk source is +# create-only), only new VMs pick it up. +# cloud_image_file_id = "local:iso/resolute-server-cloudimg-amd64-20260720-golden-20260720T000000Z.img" # The floor can be raised if you start using a different VMID band, but never # lowered - a validation in guards.tofu rejects anything below it. diff --git a/variables.tofu.example b/variables.tofu.example index ae1bddb..7a1589e 100644 --- a/variables.tofu.example +++ b/variables.tofu.example @@ -10,31 +10,15 @@ variable "pve_node" { } variable "pve_endpoint" { - description = "ProxMox API endpoint. The SSH node address is derived from its host part. EDIT: point at your node." + description = "ProxMox API endpoint. EDIT: point at your node." type = string default = "https://10.0.0.10:8006/" } -variable "pve_ssh_private_key_path" { - description = <<-EOT - Private key used by the provider to SSH into the node. Required because the - ProxMox API has no endpoint for uploading snippets, so the provider falls - back to SSH for the cloud-init user-data files. - EOT - type = string - default = "~/.ssh/id_ed25519_pve" -} - variable "cloud_image_file_id" { - description = "Datastore reference to the Ubuntu cloud image used as the disk source. EDIT: upload a cloud image and point this at it." - type = string - default = "local:iso/resolute-server-cloudimg-amd64-20260720.img" -} - -variable "snippet_datastore" { - description = "Datastore holding cloud-init snippets. Must have the 'snippets' content type (lvmthin cannot hold snippets; a dir datastore like 'local' can)." + description = "Datastore reference to the golden cloud image used as the disk source, built and uploaded by scripts/build-image.sh. EDIT: build an image and point this at it." type = string - default = "local" + default = "local:import/resolute-server-cloudimg-amd64-20260720-golden-20260720T000000Z.qcow2" } variable "disk_datastore" { @@ -56,7 +40,7 @@ variable "ci_user" { } variable "ci_timezone" { - description = "Timezone applied by cloud-init on every VM." + description = "Timezone applied by the ansible base role on every VM. Re-appliable: unlike a first-boot setting, changing it reaches existing VMs." type = string default = "Etc/UTC" } diff --git a/vms.tofu b/vms.tofu index 9c18a68..1465ee1 100644 --- a/vms.tofu +++ b/vms.tofu @@ -20,24 +20,21 @@ locals { ] } -module "vm" { - source = "./modules/vm" +module "vm-pve" { + source = "./modules/vm-pve" for_each = local.vms vm_name = each.key spec = each.value # Fleet-wide settings, stated once here rather than repeated per inventory file. - cloud_init_dir = "${path.module}/cloud-init" ansible_dir = "${path.module}/ansible" pve_node = var.pve_node cloud_image_file_id = var.cloud_image_file_id - snippet_datastore = var.snippet_datastore disk_datastore = var.disk_datastore network_bridge = var.network_bridge ssh_public_keys = var.ssh_public_keys ci_user = var.ci_user - ci_timezone = var.ci_timezone managed_tag = local.managed_tag managed_vmid_min = var.managed_vmid_min