From 5600deb363806fe143a073ea9c9a25ee6550c56a Mon Sep 17 00:00:00 2001
From: vpzed-dev-lux <320706889+vpzed-dev-lux@users.noreply.github.com>
Date: Fri, 28 Aug 2026 14:33:06 +0000
Subject: [PATCH 01/13] add scripts/build-image.sh and point at the golden
 image

The bpg provider can only upload cloud-init snippets over SSH, which is what
forces a root SSH key for the hypervisor into providers.tofu. Everything that
snippet does can move to PVE's native cloud-init settings or to ansible -
except installing qemu-guest-agent, because apply blocks until the agent
reports an address and the ansible inventory reads that address from the
agent. So the agent gets baked into the image instead.

build-image.sh downloads a dated upstream serial, verifies it against
SHA256SUMS, installs the agent with virt-customize, and uploads the result to
the node over the API - deliberately not scp, since losing the SSH path is the
point of the exercise.

Two things the build has to get right and neither is obvious:

- The image's root filesystem has ~366 MB free, and apt's universe index is
  ~120 MB unpacked. APT::Snapshot would pull a second full index set alongside
  the configured one, which fills the guest disk mid-unpack. The sources file
  is rewritten to the snapshot mirror for the duration instead, and only
  Packages indexes are fetched.
- PVE streams everything after the multipart file part straight into the file,
  so filename=@ must be the last -F. With checksum after it, the checksum was
  never parsed as a parameter and its 286 bytes were appended to the image -
  which qemu-img tolerates silently. In the right order the node verifies the
  upload itself.

There is no `systemctl enable qemu-guest-agent`: the unit ships an empty
[Install] and is started by a udev rule on the virtio-serial port. The build
asserts that rule exists, since without it apply hangs for the full 30m
timeout.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MzjHsAQ3HkQjLdgv5PVi55
---
 scripts/build-image.sh | 313 +++++++++++++++++++++++++++++++++++++++++
 variables.tofu.example |   4 +-
 2 files changed, 315 insertions(+), 2 deletions(-)
 create mode 100755 scripts/build-image.sh

diff --git a/scripts/build-image.sh b/scripts/build-image.sh
new file mode 100755
index 0000000..7651d51
--- /dev/null
+++ b/scripts/build-image.sh
@@ -0,0 +1,313 @@
+#!/usr/bin/env bash
+# Build the golden Ubuntu cloud image and upload it to the ProxMox node.
+#
+# The bpg provider can only upload cloud-init snippets over SSH (the PVE API
+# has no snippets endpoint), which is what forced a root SSH key for the
+# hypervisor into providers.tofu. Everything that snippet did can move to PVE's
+# native cloud-init settings or to the ansible base role - except installing
+# qemu-guest-agent, because apply blocks until the agent reports an address and
+# the ansible inventory reads the address from the agent. Ansible therefore
+# cannot be what installs it, so it is baked in here instead.
+#
+# Deliberately API-only: the finished image goes up through the storage upload
+# endpoint, not scp. Losing the SSH path is the entire point of the exercise.
+#
+#   ./scripts/build-image.sh                     # build only
+#   ./scripts/build-image.sh --upload            # build, then upload
+#   ./scripts/build-image.sh --serial 20260823 --upload
+#
+# The build needs no secrets. --upload decrypts the API token from
+# secrets.enc.json with sops, so it needs the age key - but not `source
+# tofu.env`, which is only about the state passphrase.
+#
+# The output is named for both the upstream serial and the archive snapshot it
+# was built against, and an uploaded image is never overwritten: disk file_id
+# is under ignore_changes in modules/vm, so a new name is a no-op for running
+# VMs while an in-place replacement would silently change what they were built
+# from. Build a new name, repoint cloud_image_file_id, delete the old volume by
+# hand once nothing references it.
+
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+# Same serial as the plain image already on the node. Bumping it is a separate
+# change: a new serial moves every package version at once, which drowns any
+# refactor being verified with scripts/vm-fingerprint.sh.
+release="resolute"
+serial="20260720"
+snapshot=""
+outdir="private/images"
+datastore="local"
+endpoint=""
+node=""
+do_upload=0
+force=0
+
+usage="usage: build-image.sh [options]
+
+  --serial YYYYMMDD    upstream cloud image serial (default: $serial)
+  --snapshot STAMP     snapshot.ubuntu.com timestamp to build against
+                       (default: <serial>T000000Z)
+  --outdir DIR         where to download and build (default: $outdir)
+  --upload             upload the finished image to the node over the API
+  --datastore ID       upload target datastore (default: $datastore)
+  --endpoint URL       PVE API endpoint (default: variables.tofu's pve_endpoint)
+  --node NAME          PVE node name (default: variables.tofu's pve_node)
+  --force              rebuild an existing local image / overwrite on the node
+  -h, --help           this"
+
+die() { echo "build-image: $*" >&2; exit 1; }
+warn() { echo "build-image: warning: $*" >&2; }
+step() { echo; echo "==> $*"; }
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --serial)    serial="${2:?$usage}"; shift 2 ;;
+    --snapshot)  snapshot="${2:?$usage}"; shift 2 ;;
+    --outdir)    outdir="${2:?$usage}"; shift 2 ;;
+    --datastore) datastore="${2:?$usage}"; shift 2 ;;
+    --endpoint)  endpoint="${2:?$usage}"; shift 2 ;;
+    --node)      node="${2:?$usage}"; shift 2 ;;
+    --upload)    do_upload=1; shift ;;
+    --force)     force=1; shift ;;
+    -h|--help)   echo "$usage"; exit 0 ;;
+    *)           die "unknown argument: $1"$'\n'"$usage" ;;
+  esac
+done
+
+[[ "$serial" =~ ^[0-9]{8}(\.[0-9]+)?$ ]] || die "--serial must look like YYYYMMDD (got '$serial')"
+: "${snapshot:=${serial%%.*}T000000Z}"
+[[ "$snapshot" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || die "--snapshot must be YYYYMMDDTHHMMSSZ (got '$snapshot')"
+
+base_url="https://cloud-images.ubuntu.com/${release}/${serial}"
+upstream_name="${release}-server-cloudimg-amd64.img"
+cached="${outdir}/${release}-server-cloudimg-amd64-${serial}.img"
+golden_name="${release}-server-cloudimg-amd64-${serial}-golden-${snapshot}.img"
+golden="${outdir}/${golden_name}"
+volid="${datastore}:iso/${golden_name}"
+
+# ---------------------------------------------------------------------------
+# Preflight. Every failure names its own remedy - this script is run rarely
+# enough that nobody remembers the setup.
+# ---------------------------------------------------------------------------
+step "Preflight"
+
+command -v virt-customize >/dev/null || die "virt-customize not found; apt install libguestfs-tools"
+command -v curl >/dev/null || die "curl not found"
+command -v sha256sum >/dev/null || die "sha256sum not found"
+
+# libguestfs boots its appliance with the host kernel, which Ubuntu ships mode
+# 0600. Without the statoverride every guestfs call fails to launch.
+kernel="/boot/vmlinuz-$(uname -r)"
+[ -r "$kernel" ] || die "$kernel is not readable; run:
+  sudo dpkg-statoverride --update --add root root 0644 $kernel"
+
+# Not fatal, just slow: libguestfs silently falls back to TCG emulation, which
+# turns a two-minute --install into a very long one.
+if [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then
+  warn "/dev/kvm is not accessible - the build will run under TCG emulation and be slow.
+  Fix with: sudo usermod -aG kvm $USER   (then log out and back in)"
+fi
+
+mkdir -p "$outdir"
+
+# The upstream image is ~860 MB and the golden copy grows past 1 GB while apt
+# runs inside it. Only count the download if it is not already cached.
+need_kb=$((1536 * 1024))
+[ -f "$cached" ] || need_kb=$((need_kb + 900 * 1024))
+avail_kb="$(df -Pk "$outdir" | awk 'NR==2 {print $4}')"
+[ "$avail_kb" -ge "$need_kb" ] || die "only $((avail_kb / 1024)) MB free in $outdir, need ~$((need_kb / 1024)) MB.
+  Note /tmp is a tmpfs here - building there spends RAM, not disk."
+
+echo "ok    virt-customize, readable $kernel, $((avail_kb / 1024)) MB free in $outdir"
+
+# ---------------------------------------------------------------------------
+# Download and verify. The dated serial directory, never current/, which is a
+# moving target: a golden image has to be reproducible from its own filename.
+# ---------------------------------------------------------------------------
+step "Fetching ${release}/${serial}"
+
+if [ -f "$cached" ]; then
+  echo "ok    already downloaded: $cached"
+else
+  curl -fSL --retry 3 -o "${cached}.part" "${base_url}/${upstream_name}" \
+    || die "download failed: ${base_url}/${upstream_name}
+  Serial ${serial} may have aged off the mirror; check https://cloud-images.ubuntu.com/${release}/"
+  mv "${cached}.part" "$cached"
+fi
+
+expected="$(curl -fsSL "${base_url}/SHA256SUMS" | awk -v n="$upstream_name" '$2 == "*" n || $2 == n {print $1}')"
+[ -n "$expected" ] || die "no $upstream_name line in ${base_url}/SHA256SUMS"
+actual="$(sha256sum "$cached" | awk '{print $1}')"
+[ "$actual" = "$expected" ] || die "checksum mismatch for $cached
+  expected $expected
+  actual   $actual
+  Delete it and re-run."
+echo "ok    sha256 $actual"
+
+# ---------------------------------------------------------------------------
+# Customise. virt-customize applies operations in command-line order.
+# ---------------------------------------------------------------------------
+step "Building $golden_name"
+
+if [ -f "$golden" ] && [ "$force" -eq 0 ]; then
+  echo "ok    already built: $golden (--force to rebuild)"
+else
+  # Build under .part and rename only on success, so an aborted build is never
+  # mistaken for a finished one on the next run.
+  rm -f "$golden" "${golden}.part"
+  cp --reflink=auto "$cached" "${golden}.part"
+
+  # The image's root filesystem is 2.2 GB with ~366 MB free, and apt's index
+  # for universe alone is ~120 MB unpacked. Two things follow, and skipping
+  # either one fills the guest disk mid-unpack:
+  #
+  #   - Fetch ONE index set, not two. APT::Snapshot adds the snapshot mirror
+  #     alongside the configured one and refreshes both, so the sources file is
+  #     rewritten to point at snapshot.ubuntu.com instead, and restored after.
+  #     Same effect - the agent comes from the archive as of $snapshot - at
+  #     half the disk cost, and the pin cannot leak into the built image.
+  #   - Fetch only Packages. Translations, DEP-11 app-stream metadata and
+  #     command-not-found indexes are pure waste in a headless image.
+  #
+  # The pin must NOT survive into the image under any spelling: baked into
+  # apt.conf.d it would override every VM's own archive_snapshot, and left in
+  # ubuntu.sources it would freeze the whole fleet at the build's snapshot.
+  # Hence the restore, and the grep that fails the build if it did not happen.
+  #
+  # There is deliberately no `systemctl enable qemu-guest-agent`: the unit ships
+  # an empty [Install] section, so enabling it is a no-op. It is started by
+  # /usr/lib/udev/rules.d/60-qemu-guest-agent.rules when the virtio-serial port
+  # org.qemu.guest_agent.0 appears. That rule is the whole reason apply
+  # terminates - without the agent it blocks for the full 30m timeout waiting
+  # for an address - so its presence is asserted rather than assumed.
+  #
+  # The 51cloudinit-no-auto-upgrades filename below is the one cloud-init used
+  # to write. It is kept verbatim - "cloudinit" is a misnomer now, but
+  # scripts/vm-fingerprint.sh records apt config by filename, so renaming it
+  # would make every fingerprint captured so far incomparable.
+  LIBGUESTFS_BACKEND=direct virt-customize -a "${golden}.part" \
+    --run-command 'cp /etc/apt/sources.list.d/ubuntu.sources /root/ubuntu.sources.orig' \
+    --run-command "sed -i 's|^URIs:.*|URIs: https://snapshot.ubuntu.com/ubuntu/${snapshot}|' /etc/apt/sources.list.d/ubuntu.sources" \
+    --write '/etc/apt/apt.conf.d/50build-lean:Acquire::Languages "none";
+Acquire::IndexTargets::deb::Translations::DefaultEnabled "false";
+Acquire::IndexTargets::deb::DEP-11::DefaultEnabled "false";
+Acquire::IndexTargets::deb::CNF::DefaultEnabled "false";' \
+    --run-command 'rm -rf /var/lib/apt/lists/*' \
+    --install qemu-guest-agent \
+    --run-command 'mv /root/ubuntu.sources.orig /etc/apt/sources.list.d/ubuntu.sources' \
+    --delete /etc/apt/apt.conf.d/50build-lean \
+    --run-command 'grep -q archive.ubuntu.com /etc/apt/sources.list.d/ubuntu.sources' \
+    --run-command 'test -f /usr/lib/udev/rules.d/60-qemu-guest-agent.rules' \
+    --run-command 'systemctl disable apt-daily.timer apt-daily-upgrade.timer || true' \
+    --write '/etc/apt/apt.conf.d/51cloudinit-no-auto-upgrades:APT::Periodic::Update-Package-Lists "0";
+APT::Periodic::Unattended-Upgrade "0";' \
+    --run-command 'apt-get clean' \
+    --run-command 'rm -rf /var/lib/apt/lists/*' \
+    --truncate /etc/machine-id \
+    || die "virt-customize failed; the partial image is at ${golden}.part"
+
+  mv "${golden}.part" "$golden"
+fi
+
+golden_sha="$(sha256sum "$golden" | awk '{print $1}')"
+
+# ---------------------------------------------------------------------------
+# Upload over the API.
+# ---------------------------------------------------------------------------
+if [ "$do_upload" -eq 1 ]; then
+  step "Uploading to $volid"
+
+  command -v sops >/dev/null || die "sops not found; needed to decrypt the API token"
+
+  # variables.tofu is a symlink to site-specific values outside the repo, so
+  # read the defaults out of it rather than hardcoding an address here.
+  tofu_default() {
+    awk -v name="$1" '
+      $0 ~ "^variable[[:space:]]+\"" name "\"" { inblock = 1; next }
+      inblock && /^}/ { exit }
+      inblock && /^[[:space:]]*default[[:space:]]*=/ {
+        sub(/^[^=]*=[[:space:]]*/, ""); gsub(/"/, ""); sub(/[[:space:]]+$/, "")
+        print; exit
+      }
+    ' variables.tofu
+  }
+  : "${endpoint:=$(tofu_default pve_endpoint)}"
+  : "${node:=$(tofu_default pve_node)}"
+  [ -n "$endpoint" ] || die "could not read pve_endpoint from variables.tofu; pass --endpoint"
+  [ -n "$node" ] || die "could not read pve_node from variables.tofu; pass --node"
+  endpoint="${endpoint%/}"
+
+  token_id="$(sops -d --extract '["proxmox"]["api_token_id"]' secrets.enc.json)" \
+    || die "sops decryption failed; is the age key at ~/.config/sops/age/keys.txt?"
+  token_secret="$(sops -d --extract '["proxmox"]["api_token_secret"]' secrets.enc.json)" \
+    || die "sops decryption failed"
+  [ -n "$token_id" ] && [ -n "$token_secret" ] || die "sops returned an empty API token"
+  auth="Authorization: PVEAPIToken=${token_id}=${token_secret}"
+
+  # -k mirrors providers.tofu's `insecure = true`: the node serves a
+  # self-signed certificate. The token rides on every one of these calls.
+  api() { curl -k -fsS -H "$auth" "$@"; }
+
+  if api "${endpoint}/api2/json/nodes/${node}/storage/${datastore}/content?content=iso" \
+    | grep -q "\"volid\":\"${volid}\""; then
+    if [ "$force" -eq 0 ]; then
+      die "$volid already exists on the node.
+  Never replace an uploaded image in place - VMs pin their disk source by
+  file_id under ignore_changes, so a same-name replacement changes what a
+  running VM was built from without any plan diff. Build a new snapshot or
+  serial instead. --force overrides."
+    fi
+    warn "$volid exists and --force was given; overwriting"
+  fi
+
+  # filename=@ MUST be the last -F. PVE parses the multipart body in order and
+  # streams everything after the file part straight into the file, so any field
+  # placed after it is both silently ignored as a parameter and appended to the
+  # image as garbage - which qemu-img happily tolerates, so nothing complains.
+  # With the order below, checksum is a real parameter and the node verifies
+  # the upload itself; the task fails on mismatch.
+  echo "      $(du -h "$golden" | awk '{print $1}') to ${node}/${datastore} - this takes a few minutes"
+  upid="$(api --max-time 1800 \
+    -F content=iso \
+    -F checksum-algorithm=sha256 \
+    -F "checksum=${golden_sha}" \
+    -F "filename=@${golden}" \
+    "${endpoint}/api2/json/nodes/${node}/storage/${datastore}/upload" \
+    | python3 -c 'import json,sys; print(json.load(sys.stdin)["data"])')" \
+    || die "upload failed"
+  echo "      task $upid"
+
+  # The upload endpoint returns as soon as the task is queued; the node is
+  # still writing (and checksumming) after that.
+  while :; do
+    status_json="$(api "${endpoint}/api2/json/nodes/${node}/tasks/${upid}/status")"
+    read -r task_status task_exit <<<"$(printf '%s' "$status_json" | python3 -c '
+import json, sys
+d = json.load(sys.stdin)["data"]
+print(d.get("status", ""), d.get("exitstatus", ""))')"
+    [ "$task_status" = "stopped" ] && break
+    sleep 5
+  done
+  [ "$task_exit" = "OK" ] || die "upload task finished with: ${task_exit:-<none>}
+  Check the node: pvenode task log $upid"
+  echo "ok    uploaded"
+fi
+
+# ---------------------------------------------------------------------------
+step "Done"
+cat <<EOF
+
+  image     $golden
+  sha256    $golden_sha
+  volid     $volid
+
+Set cloud_image_file_id to the volid above, in variables.tofu and in
+variables.tofu.example.
+EOF
+if [ "$do_upload" -eq 0 ]; then
+  echo
+  echo "Not uploaded. Re-run with --upload, or upload by hand in the web UI:"
+  echo "  node -> ${datastore} -> ISO Images -> Upload"
+fi
diff --git a/variables.tofu.example b/variables.tofu.example
index ae1bddb..0c02f39 100644
--- a/variables.tofu.example
+++ b/variables.tofu.example
@@ -26,9 +26,9 @@ variable "pve_ssh_private_key_path" {
 }
 
 variable "cloud_image_file_id" {
-  description = "Datastore reference to the Ubuntu cloud image used as the disk source. EDIT: upload a cloud image and point this at it."
+  description = "Datastore reference to the golden cloud image used as the disk source, built and uploaded by scripts/build-image.sh. EDIT: build an image and point this at it."
   type        = string
-  default     = "local:iso/resolute-server-cloudimg-amd64-20260720.img"
+  default     = "local:iso/resolute-server-cloudimg-amd64-20260720-golden-20260720T000000Z.img"
 }
 
 variable "snippet_datastore" {

From 42ac8a04cc056e6915879d346aaece181189e869 Mon Sep 17 00:00:00 2001
From: vpzed-dev-lux <320706889+vpzed-dev-lux@users.noreply.github.com>
Date: Fri, 28 Aug 2026 14:52:07 +0000
Subject: [PATCH 02/13] rename modules/vm to modules/vm-pve

The module is specific to ProxMox VE - it speaks the bpg/proxmox provider's
resource schema throughout - and the bare name did not say so.

The module call is renamed too, not just the directory, so resource addresses
read module.vm-pve["name"]. That is only free right now: state currently holds
nothing but the three data sources, so there is no address to migrate. Once
Step 7 provisions a VM, the same rename would need a moved block.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MzjHsAQ3HkQjLdgv5PVi55
---
 CLAUDE.md                             | 6 +++---
 CONTRIBUTING.md                       | 2 +-
 README.md                             | 8 ++++----
 inventory-example.yaml                | 2 +-
 modules/{vm => vm-pve}/main.tofu      | 2 +-
 modules/{vm => vm-pve}/outputs.tofu   | 2 +-
 modules/{vm => vm-pve}/variables.tofu | 2 +-
 modules/{vm => vm-pve}/versions.tofu  | 2 +-
 outputs.tofu                          | 2 +-
 scripts/build-image.sh                | 8 ++++----
 scripts/check-cloud-init.sh           | 2 +-
 vms.tofu                              | 4 ++--
 12 files changed, 21 insertions(+), 21 deletions(-)
 rename modules/{vm => vm-pve}/main.tofu (99%)
 rename modules/{vm => vm-pve}/outputs.tofu (96%)
 rename modules/{vm => vm-pve}/variables.tofu (99%)
 rename modules/{vm => vm-pve}/versions.tofu (92%)

diff --git a/CLAUDE.md b/CLAUDE.md
index 9472d4e..79aa161 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -21,7 +21,7 @@ Template placeholders — fill in for your site (marked `EDIT` in the code):
 ## Hard rules
 
 **Pre-existing VMs must never be touched.** `guards.tofu` plus lifecycle
-preconditions in `modules/vm/main.tofu` enforce a VMID floor, a named protected
+preconditions in `modules/vm-pve/main.tofu` enforce a VMID floor, a named protected
 list, and a live check that rejects any VMID belonging to a VM not tagged
 `opentofu`. Validations stop the floor and the list from being weakened by
 tfvars or `TF_VAR_*` overrides, and a postcondition fails the plan if the
@@ -46,7 +46,7 @@ scanned; subdirectories are invisible). The next apply destroys the VM, its
 disk, and its snippet; moving the file back provisions a *fresh* VM. To keep a
 VM and its data but power it off, set `started: false` instead. The filename
 is the VM name (DNS label). Only `vm_id` is required — everything else takes
-an `optional()` default from the `spec` object in `modules/vm/variables.tofu`,
+an `optional()` default from the `spec` object in `modules/vm-pve/variables.tofu`,
 which is the contract worth reading first.
 
 After touching anything under `cloud-init/`, run
@@ -124,7 +124,7 @@ Rebuild verification: capture, commit, destroy + reprovision, run
 - **`cloud-init schema` catches deprecations that still "work."** The check
   script treats deprecation warnings as failures.
 - **A child module must declare its own `required_providers`** naming
-  `bpg/proxmox`. Without `modules/vm/versions.tofu`, `tofu init` assumes
+  `bpg/proxmox`. Without `modules/vm-pve/versions.tofu`, `tofu init` assumes
   `hashicorp/proxmox` and fails.
 - **`agent { enabled = true }` makes apply block** until the guest agent
   reports an address. Quick with the default `package_upgrade: false`; with a
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index d8d3f67..6b50bf3 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -64,5 +64,5 @@ Notes on the two check scripts:
 - Ansible role names use underscores; each is a directory under
   `ansible/roles/`. Pin software versions in the role's `defaults/main.yaml`.
 - Keep guard behavior intact: `guards.tofu` and the validations in
-  `modules/vm/variables.tofu` exist to protect pre-existing VMs and must not
+  `modules/vm-pve/variables.tofu` exist to protect pre-existing VMs and must not
   be weakened.
diff --git a/README.md b/README.md
index 04306cf..e83bec8 100644
--- a/README.md
+++ b/README.md
@@ -26,7 +26,7 @@ overall testing workflow.
 - **One YAML file per VM.** Provisioning is adding a file to `inventory/`;
   deprovisioning is deleting one, or moving it to the destroy subdiredtory. 
   Only `vm_id` is required — everything else inherits a typed `optional()` 
-  default from the `spec` contract in `modules/vm/variables.tofu`.
+  default from the `spec` contract in `modules/vm-pve/variables.tofu`.
 - **Repeatable environments.** Per-VM `archive_snapshot:` pins apt to
   [snapshot.ubuntu.com](https://snapshot.ubuntu.com) at a chosen instant, so
   package installs resolve identically forever; `package_upgrade` defaults to
@@ -60,10 +60,10 @@ overall testing workflow.
 ├── versions.tofu  providers.tofu  encryption.tofu
 ├── main.tofu              # sops secrets, node data source
 ├── guards.tofu            # protected VMIDs + live foreign-VM lookup   [EDIT]
-├── vms.tofu               # inventory/ -> module.vm, for_each
+├── vms.tofu               # inventory/ -> module.vm-pve, for_each
 ├── variables.tofu         # fleet-wide defaults                        [EDIT]
 ├── outputs.tofu
-├── modules/vm/            # the contract: what a VM is
+├── modules/vm-pve/        # the contract: what a VM is
 ├── cloud-init/
 │   ├── base.yaml.tftpl            # every VM gets this - the whole document
 │   └── base.runcmd.json.tftpl     # commands every VM runs
@@ -296,7 +296,7 @@ can never be the SSH identity — that split is structural, not a choice.
 
 ## Protecting pre-existing VMs
 
-Three plan-time layers in `guards.tofu` + `modules/vm/main.tofu`
+Three plan-time layers in `guards.tofu` + `modules/vm-pve/main.tofu`
 (preconditions, not `check` blocks — they fail the plan rather than warn):
 
 1. **VMID floor** — managed VMs live at or above `managed_vmid_min`.
diff --git a/inventory-example.yaml b/inventory-example.yaml
index 91e0e2c..bde1301 100644
--- a/inventory-example.yaml
+++ b/inventory-example.yaml
@@ -2,7 +2,7 @@
 #
 # The file name is the VM name (must be a DNS label: lowercase letters, digits,
 # hyphens). Only vm_id is required; everything omitted takes the optional()
-# default from modules/vm/variables.tofu, so a three-line file is a complete
+# default from modules/vm-pve/variables.tofu, so a three-line file is a complete
 # VM. Deprovision by deleting this file and running `tofu apply`.
 
 vm_id: 500
diff --git a/modules/vm/main.tofu b/modules/vm-pve/main.tofu
similarity index 99%
rename from modules/vm/main.tofu
rename to modules/vm-pve/main.tofu
index b543951..7a2905a 100644
--- a/modules/vm/main.tofu
+++ b/modules/vm-pve/main.tofu
@@ -1,4 +1,4 @@
-# modules/vm/main.tofu
+# modules/vm-pve/main.tofu
 
 locals {
   fqdn = var.spec.dns_domain == null ? var.vm_name : "${var.vm_name}.${var.spec.dns_domain}"
diff --git a/modules/vm/outputs.tofu b/modules/vm-pve/outputs.tofu
similarity index 96%
rename from modules/vm/outputs.tofu
rename to modules/vm-pve/outputs.tofu
index 3b22efc..a026147 100644
--- a/modules/vm/outputs.tofu
+++ b/modules/vm-pve/outputs.tofu
@@ -1,4 +1,4 @@
-# modules/vm/outputs.tofu
+# modules/vm-pve/outputs.tofu
 
 output "vm_id" {
   description = "VMID of the provisioned VM."
diff --git a/modules/vm/variables.tofu b/modules/vm-pve/variables.tofu
similarity index 99%
rename from modules/vm/variables.tofu
rename to modules/vm-pve/variables.tofu
index 0d872e0..76dc9f8 100644
--- a/modules/vm/variables.tofu
+++ b/modules/vm-pve/variables.tofu
@@ -1,4 +1,4 @@
-# modules/vm/variables.tofu
+# modules/vm-pve/variables.tofu
 
 variable "vm_name" {
   description = "VM name, also used as the guest hostname. Comes from the inventory file name."
diff --git a/modules/vm/versions.tofu b/modules/vm-pve/versions.tofu
similarity index 92%
rename from modules/vm/versions.tofu
rename to modules/vm-pve/versions.tofu
index bf249b6..f9243d9 100644
--- a/modules/vm/versions.tofu
+++ b/modules/vm-pve/versions.tofu
@@ -1,4 +1,4 @@
-# modules/vm/versions.tofu
+# modules/vm-pve/versions.tofu
 #
 # A child module must name its provider source; without this OpenTofu assumes
 # hashicorp/proxmox and init fails.
diff --git a/outputs.tofu b/outputs.tofu
index 7ead3bc..ae71f7e 100644
--- a/outputs.tofu
+++ b/outputs.tofu
@@ -8,7 +8,7 @@ output "nodes" {
 output "vms" {
   description = "Provisioned VMs, keyed by inventory name."
   value = {
-    for name, vm in module.vm : name => {
+    for name, vm in module.vm-pve : name => {
       vm_id          = vm.vm_id
       ipv4_addresses = vm.ipv4_addresses
       ssh_command    = vm.ssh_command
diff --git a/scripts/build-image.sh b/scripts/build-image.sh
index 7651d51..8a57a69 100755
--- a/scripts/build-image.sh
+++ b/scripts/build-image.sh
@@ -22,10 +22,10 @@
 #
 # The output is named for both the upstream serial and the archive snapshot it
 # was built against, and an uploaded image is never overwritten: disk file_id
-# is under ignore_changes in modules/vm, so a new name is a no-op for running
-# VMs while an in-place replacement would silently change what they were built
-# from. Build a new name, repoint cloud_image_file_id, delete the old volume by
-# hand once nothing references it.
+# is under ignore_changes in modules/vm-pve, so a new name is a no-op for
+# running VMs while an in-place replacement would silently change what they
+# were built from. Build a new name, repoint cloud_image_file_id, delete the
+# old volume by hand once nothing references it.
 
 set -euo pipefail
 
diff --git a/scripts/check-cloud-init.sh b/scripts/check-cloud-init.sh
index db71e4e..34a6642 100755
--- a/scripts/check-cloud-init.sh
+++ b/scripts/check-cloud-init.sh
@@ -66,7 +66,7 @@ render() {
 }
 
 # The runcmd list a VM actually gets, built from the same file
-# modules/vm/main.tofu reads.
+# modules/vm-pve/main.tofu reads.
 runcmd_expr() {
   printf '%s' 'jsondecode(templatefile("./cloud-init/base.runcmd.json.tftpl", { ci_user = "ubuntu", vm_name = "checkvm" }))'
 }
diff --git a/vms.tofu b/vms.tofu
index 9c18a68..c68dff0 100644
--- a/vms.tofu
+++ b/vms.tofu
@@ -20,8 +20,8 @@ locals {
   ]
 }
 
-module "vm" {
-  source   = "./modules/vm"
+module "vm-pve" {
+  source   = "./modules/vm-pve"
   for_each = local.vms
 
   vm_name = each.key

From 22e560c74bef7dab76e49d1b53d7bf9b753aa615 Mon Sep 17 00:00:00 2001
From: vpzed-dev-lux <320706889+vpzed-dev-lux@users.noreply.github.com>
Date: Fri, 28 Aug 2026 14:55:27 +0000
Subject: [PATCH 03/13] move cloud-init from a snippet to PVE's native settings

The snippet was the only reason the provider needed SSH to the hypervisor.
Everything it carried now lands in one of two places: identity at first boot
goes into the VM's own cloud-init config over the API, and everything softer -
packages, apt pinning, timezone - becomes ansible's job.

So this drops the fqdn/ci_packages/runcmd/user_data locals and the
proxmox_virtual_environment_file resource, and adds user_account plus an
explicit upgrade to the initialization block. upgrade is stated rather than
left unset because PVE defaults ciupgrade to 1 while spec.package_upgrade
defaults to false, and the provider's schema default is Computed - so an unset
value would silently take PVE's side.

The dns block stays as it was: dns_domain still drives searchdomain, which is
what PVE turns into the guest's generated fqdn, so dropping the local fqdn
computation changes nothing the guest sees.

packages and archive_snapshot become module outputs, since the ansible
inventory is what applies them now. spec.packages gains a name validation to
replace the protection jsonencode() used to give it inside the YAML template.

vms.tofu drops the three arguments those variables backed; the rest of the
root config is a separate change.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MzjHsAQ3HkQjLdgv5PVi55
---
 modules/vm-pve/main.tofu      | 73 +++++++++++------------------------
 modules/vm-pve/outputs.tofu   | 12 ++++++
 modules/vm-pve/variables.tofu | 50 ++++++++++++------------
 vms.tofu                      |  3 --
 4 files changed, 58 insertions(+), 80 deletions(-)

diff --git a/modules/vm-pve/main.tofu b/modules/vm-pve/main.tofu
index 7a2905a..cc0709e 100644
--- a/modules/vm-pve/main.tofu
+++ b/modules/vm-pve/main.tofu
@@ -1,58 +1,10 @@
 # modules/vm-pve/main.tofu
 
 locals {
-  fqdn = var.spec.dns_domain == null ? var.vm_name : "${var.vm_name}.${var.spec.dns_domain}"
-
-  # The VM resource blocks on the guest agent reporting an address, so the agent
-  # package is not optional.
-  ci_packages = distinct(concat(["qemu-guest-agent"], var.spec.packages))
-
   # Always applied, never left to inventory: the foreign-VM guard keys off it.
   # Sorted because ProxMox normalizes tag order; an unsorted list here would
   # read back differently and produce a perpetual diff.
   tags = sort(distinct(concat([var.managed_tag], var.spec.tags)))
-
-  # The runcmd list lives in cloud-init/base.runcmd.json.tftpl rather than in
-  # HCL so that scripts/check-cloud-init.sh renders the exact same source this
-  # module does; a hand-copied literal in the check script is how commands
-  # escaped validation before.
-  runcmd = jsondecode(
-    templatefile("${var.cloud_init_dir}/base.runcmd.json.tftpl", {
-      ci_user = var.ci_user
-      vm_name = var.vm_name
-    })
-  )
-
-  base_user_data = templatefile("${var.cloud_init_dir}/base.yaml.tftpl", {
-    vm_name          = var.vm_name
-    fqdn             = local.fqdn
-    timezone         = var.ci_timezone
-    ci_user          = var.ci_user
-    ssh_public_keys  = var.ssh_public_keys
-    packages         = local.ci_packages
-    runcmd           = local.runcmd
-    archive_snapshot = var.spec.archive_snapshot
-    package_upgrade  = var.spec.package_upgrade
-  })
-
-  user_data = trimspace(local.base_user_data)
-}
-
-# Uploaded over SSH: the ProxMox API exposes no endpoint for snippets.
-#
-# The snippet's ID is derived from its (stable) file name, so editing a
-# cloud-init template re-uploads the snippet but shows "0 to change" on the VM
-# - the running guest keeps whatever it booted with. Template changes reach a
-# VM only by recreating it (delete the inventory file, apply, restore, apply).
-resource "proxmox_virtual_environment_file" "user_data" {
-  node_name    = var.pve_node
-  datastore_id = var.snippet_datastore
-  content_type = "snippets"
-
-  source_raw {
-    file_name = "${var.vm_name}-user-data.yaml"
-    data      = local.user_data
-  }
 }
 
 resource "proxmox_virtual_environment_vm" "this" {
@@ -104,10 +56,29 @@ resource "proxmox_virtual_environment_vm" "this" {
     iothread     = true
   }
 
+  # PVE's native cloud-init: these are VM config, written to the ci drive by the
+  # node, so there is no snippet and no SSH to the hypervisor. Everything softer
+  # - packages, timezone, apt pinning - is ansible's job (layer 2).
+  #
+  # Unlike the snippet this replaced, changing a value here DOES diff and PVE
+  # regenerates the drive. That still does not reach a running guest: cloud-init
+  # consumes the drive on first boot only. Rotating a key on a live VM means
+  # ansible or a rebuild.
   initialization {
-    datastore_id      = var.disk_datastore
-    interface         = "ide2"
-    user_data_file_id = proxmox_virtual_environment_file.user_data.id
+    datastore_id = var.disk_datastore
+    interface    = "ide2"
+
+    # PVE fills in the rest from the distro default user (`users: - default`),
+    # which is where sudo, lock_passwd and the group list come from.
+    user_account {
+      username = var.ci_user
+      keys     = var.ssh_public_keys
+    }
+
+    # Stated explicitly because PVE defaults ciupgrade to 1 and the provider's
+    # schema default is Computed - leaving it unset is not a pin, and the
+    # default is the opposite of what spec.package_upgrade defaults to.
+    upgrade = var.spec.package_upgrade
 
     ip_config {
       ipv4 {
diff --git a/modules/vm-pve/outputs.tofu b/modules/vm-pve/outputs.tofu
index a026147..e233d4f 100644
--- a/modules/vm-pve/outputs.tofu
+++ b/modules/vm-pve/outputs.tofu
@@ -23,6 +23,18 @@ output "ansible_roles" {
   value       = var.spec.ansible_roles
 }
 
+# Passed through to the ansible inventory, which is what applies them now that
+# there is no cloud-init snippet.
+output "packages" {
+  description = "apt packages the spec declares for this VM (layer 2)."
+  value       = var.spec.packages
+}
+
+output "archive_snapshot" {
+  description = "snapshot.ubuntu.com ID the spec pins apt to, or null (layer 2)."
+  value       = var.spec.archive_snapshot
+}
+
 output "ssh_command" {
   description = "Ready-to-run SSH command for the cloud-init user."
   value = format(
diff --git a/modules/vm-pve/variables.tofu b/modules/vm-pve/variables.tofu
index 76dc9f8..1b4c0a0 100644
--- a/modules/vm-pve/variables.tofu
+++ b/modules/vm-pve/variables.tofu
@@ -28,21 +28,27 @@ variable "spec" {
     dns_servers = optional(list(string), [])
     dns_domain  = optional(string)
     vlan_id     = optional(number)
-    packages    = optional(list(string), [])
     tags        = optional(list(string), [])
-    started     = optional(bool, true)
-    on_boot     = optional(bool, false)
 
-    # snapshot.ubuntu.com ID (YYYYMMDDTHHMMSSZ). When set, apt on the guest is
-    # pinned to the archive as of that instant for the VM's whole life, so
-    # package installs are reproducible. Third-party repos (e.g. docker) have
-    # no snapshot service and are not covered.
+    # apt packages, installed by the ansible base role rather than by
+    # cloud-init, so editing this list reaches an existing VM on the next
+    # ansible-playbook run. Removing an entry does not uninstall it.
+    packages = optional(list(string), [])
+    started  = optional(bool, true)
+    on_boot  = optional(bool, false)
+
+    # snapshot.ubuntu.com ID (YYYYMMDDTHHMMSSZ). When set, the ansible base
+    # role pins apt on the guest to the archive as of that instant, so package
+    # installs are reproducible. Third-party repos (e.g. docker) have no
+    # snapshot service and are not covered. Independent of the snapshot the
+    # golden image itself was built against.
     archive_snapshot = optional(string)
 
+    # Drives PVE's ciupgrade, i.e. cloud-init's package_upgrade on first boot.
     # Default false: with a fixed base image, skipping the upgrade is what
     # keeps environments identical over time (and first boot much faster).
-    # true is also deterministic when archive_snapshot is set - the base
-    # upgrades to the pinned date instead of to "now".
+    # Note this one is first-boot only - unlike packages above, flipping it
+    # later does not reach an existing VM.
     package_upgrade = optional(bool, false)
 
     # Ansible roles under ansible/roles/ applied post-boot (layer 2). Unlike
@@ -67,6 +73,13 @@ variable "spec" {
     error_message = "spec.ipv4 must be 'dhcp' or CIDR notation like 10.0.0.50/24; a bare address would pass the plan and fail at the API."
   }
 
+  validation {
+    condition = alltrue([
+      for p in var.spec.packages : can(regex("^[a-z0-9][a-z0-9+._-]*$", p))
+    ])
+    error_message = "each spec.packages entry must be a plain apt package name (lowercase letters, digits, and + . _ -). Anything else is a typo or an attempt to smuggle apt arguments through the inventory."
+  }
+
   validation {
     condition     = var.spec.archive_snapshot == null || can(regex("^[0-9]{8}T[0-9]{6}Z$", var.spec.archive_snapshot))
     error_message = "spec.archive_snapshot must be a snapshot.ubuntu.com ID like 20260801T000000Z (UTC, YYYYMMDDTHHMMSSZ)."
@@ -91,11 +104,6 @@ variable "spec" {
   }
 }
 
-variable "cloud_init_dir" {
-  description = "Directory holding the cloud-init templates. Passed in so the module does not reach out with ../.."
-  type        = string
-}
-
 variable "ansible_dir" {
   description = "Directory holding ansible/roles/. Passed in so the module does not reach out with ../.."
   type        = string
@@ -107,12 +115,7 @@ variable "pve_node" {
 }
 
 variable "cloud_image_file_id" {
-  description = "Datastore reference to the cloud image used as the disk source."
-  type        = string
-}
-
-variable "snippet_datastore" {
-  description = "Datastore holding cloud-init snippets. Must have the 'snippets' content type."
+  description = "Datastore reference to the golden cloud image used as the disk source, built by scripts/build-image.sh. It carries qemu-guest-agent, without which apply blocks until the 30m agent timeout."
   type        = string
 }
 
@@ -137,12 +140,7 @@ variable "ssh_public_keys" {
 }
 
 variable "ci_user" {
-  description = "Login account created by cloud-init."
-  type        = string
-}
-
-variable "ci_timezone" {
-  description = "Timezone applied by cloud-init."
+  description = "Login account. Becomes PVE's ciuser, which cloud-init renames the image's default user to."
   type        = string
 }
 
diff --git a/vms.tofu b/vms.tofu
index c68dff0..1465ee1 100644
--- a/vms.tofu
+++ b/vms.tofu
@@ -28,16 +28,13 @@ module "vm-pve" {
   spec    = each.value
 
   # Fleet-wide settings, stated once here rather than repeated per inventory file.
-  cloud_init_dir      = "${path.module}/cloud-init"
   ansible_dir         = "${path.module}/ansible"
   pve_node            = var.pve_node
   cloud_image_file_id = var.cloud_image_file_id
-  snippet_datastore   = var.snippet_datastore
   disk_datastore      = var.disk_datastore
   network_bridge      = var.network_bridge
   ssh_public_keys     = var.ssh_public_keys
   ci_user             = var.ci_user
-  ci_timezone         = var.ci_timezone
 
   managed_tag      = local.managed_tag
   managed_vmid_min = var.managed_vmid_min

From 923353a488740fee58d3283a5d850cb5c21563a0 Mon Sep 17 00:00:00 2001
From: vpzed-dev-lux <320706889+vpzed-dev-lux@users.noreply.github.com>
Date: Fri, 28 Aug 2026 15:00:23 +0000
Subject: [PATCH 04/13] drop the hypervisor SSH credential from the root config

With the snippet gone there is nothing the provider cannot do over the API, so
the ssh block goes, and with it local.pve_host and pve_ssh_private_key_path.
Root on the hypervisor is no longer a credential this project holds.
snippet_datastore goes the same way - nothing writes snippets any more.

The root vms output gains packages, archive_snapshot and timezone, which is
how the three settings that used to be baked into the cloud-init drive reach
the ansible inventory instead. ci_timezone survives for that reason, but it
now describes a re-appliable setting rather than a first-boot one.

variables.tofu is a symlink to site-specific values outside the repo, so the
same edits are mirrored into variables.tofu.example by hand.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MzjHsAQ3HkQjLdgv5PVi55
---
 outputs.tofu             |  7 +++++++
 providers.tofu           | 21 ++++-----------------
 terraform.tfvars.example | 29 ++++++++++++++---------------
 variables.tofu.example   | 20 ++------------------
 4 files changed, 27 insertions(+), 50 deletions(-)

diff --git a/outputs.tofu b/outputs.tofu
index ae71f7e..8551a3b 100644
--- a/outputs.tofu
+++ b/outputs.tofu
@@ -14,6 +14,13 @@ output "vms" {
       ssh_command    = vm.ssh_command
       ansible_roles  = vm.ansible_roles
       ansible_user   = var.ci_user
+
+      # Layer 2 inputs: ansible/inventory/tofu.py turns these into hostvars,
+      # and the base role applies them. They are here rather than in the
+      # cloud-init drive because ansible can re-apply them to a running VM.
+      packages         = vm.packages
+      archive_snapshot = vm.archive_snapshot
+      timezone         = var.ci_timezone
     }
   }
 }
diff --git a/providers.tofu b/providers.tofu
index d75f046..e5c38c1 100644
--- a/providers.tofu
+++ b/providers.tofu
@@ -2,12 +2,6 @@
 
 provider "sops" {}
 
-locals {
-  # Bare host for the SSH node pin, derived from the endpoint so the address
-  # lives in exactly one place.
-  pve_host = regex("^https?://([^:/]+)", var.pve_endpoint)[0]
-}
-
 provider "proxmox" {
   endpoint  = var.pve_endpoint
   api_token = local.proxmox_api_token
@@ -17,15 +11,8 @@ provider "proxmox" {
   # the node's CA into this workstation's trust store is what removes it.
   insecure = true
 
-  ssh {
-    username    = "root"
-    private_key = file(pathexpand(var.pve_ssh_private_key_path))
-
-    # Pin the node's address so uploads do not depend on "pve" resolving from
-    # the workstation.
-    node {
-      name    = var.pve_node
-      address = local.pve_host
-    }
-  }
+  # No ssh block, deliberately. The provider only ever needed SSH to upload
+  # cloud-init snippets, which the API cannot do; with the snippet gone there
+  # is nothing left that is not an API call, and root on the hypervisor is no
+  # longer a credential this project holds.
 }
diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index 25872de..07be0dd 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -5,27 +5,26 @@
 #
 # Per-VM settings do NOT belong here - they live in inventory/*.yaml.
 
-# Keys authorized for the cloud-init user on every VM. NOT the provisioning
-# key: id_ed25519_pve is root on the hypervisor and is deliberately kept out
-# of the VMs. See README.md, "Credentials".
+# Keys authorized for the cloud-init user on every VM. With the hypervisor
+# provisioning key gone, these and the API token are the whole of what this
+# project authenticates with. See README.md, "Credentials".
 ssh_public_keys = [
   "ssh-ed25519 AAAA...replace-me... user@desktop",      # workstation
   "ssh-ed25519 BBBB...replace-me... user@provisioner",  # box running tofu
 ]
 
-# pve_node                 = "pve"
-# pve_endpoint             = "https://10.0.0.10:8006/"
-# pve_ssh_private_key_path = "~/.ssh/id_ed25519_pve"
-# disk_datastore           = "local-lvm"
-# snippet_datastore        = "local"
-# network_bridge           = "vmbr0"
-# ci_user                  = "ubuntu"
-# ci_timezone              = "Etc/UTC"
+# pve_node       = "pve"
+# pve_endpoint   = "https://10.0.0.10:8006/"
+# disk_datastore = "local-lvm"
+# network_bridge = "vmbr0"
+# ci_user        = "ubuntu"
+# ci_timezone    = "Etc/UTC"
 
-# The base image every VM's disk is imported from. This is the line to change
-# when a new Ubuntu release ships; existing VMs keep the image they were
-# created from (the disk source is create-only), only new VMs pick it up.
-# cloud_image_file_id = "local:iso/resolute-server-cloudimg-amd64-20260720.img"
+# The golden image every VM's disk is imported from, built and uploaded by
+# scripts/build-image.sh. This is the line to change after building a new one;
+# existing VMs keep the image they were created from (the disk source is
+# create-only), only new VMs pick it up.
+# cloud_image_file_id = "local:iso/resolute-server-cloudimg-amd64-20260720-golden-20260720T000000Z.img"
 
 # The floor can be raised if you start using a different VMID band, but never
 # lowered - a validation in guards.tofu rejects anything below it.
diff --git a/variables.tofu.example b/variables.tofu.example
index 0c02f39..71e012e 100644
--- a/variables.tofu.example
+++ b/variables.tofu.example
@@ -10,33 +10,17 @@ variable "pve_node" {
 }
 
 variable "pve_endpoint" {
-  description = "ProxMox API endpoint. The SSH node address is derived from its host part. EDIT: point at your node."
+  description = "ProxMox API endpoint. EDIT: point at your node."
   type        = string
   default     = "https://10.0.0.10:8006/"
 }
 
-variable "pve_ssh_private_key_path" {
-  description = <<-EOT
-    Private key used by the provider to SSH into the node. Required because the
-    ProxMox API has no endpoint for uploading snippets, so the provider falls
-    back to SSH for the cloud-init user-data files.
-  EOT
-  type        = string
-  default     = "~/.ssh/id_ed25519_pve"
-}
-
 variable "cloud_image_file_id" {
   description = "Datastore reference to the golden cloud image used as the disk source, built and uploaded by scripts/build-image.sh. EDIT: build an image and point this at it."
   type        = string
   default     = "local:iso/resolute-server-cloudimg-amd64-20260720-golden-20260720T000000Z.img"
 }
 
-variable "snippet_datastore" {
-  description = "Datastore holding cloud-init snippets. Must have the 'snippets' content type (lvmthin cannot hold snippets; a dir datastore like 'local' can)."
-  type        = string
-  default     = "local"
-}
-
 variable "disk_datastore" {
   description = "Datastore holding VM disks and cloud-init drives. EDIT: name your VM-disk datastore."
   type        = string
@@ -56,7 +40,7 @@ variable "ci_user" {
 }
 
 variable "ci_timezone" {
-  description = "Timezone applied by cloud-init on every VM."
+  description = "Timezone applied by the ansible base role on every VM. Re-appliable: unlike a first-boot setting, changing it reaches existing VMs."
   type        = string
   default     = "Etc/UTC"
 }

From e8030339f73a22ff3507b6fc72c1f824001d1a13 Mon Sep 17 00:00:00 2001
From: vpzed-dev-lux <320706889+vpzed-dev-lux@users.noreply.github.com>
Date: Fri, 28 Aug 2026 15:26:13 +0000
Subject: [PATCH 05/13] add the ansible base role, and the collections CI needs
 to lint it

base carries what the cloud-init snippet used to do at first boot: the apt
snapshot pin, the APT::Periodic zeros, the apt-daily timers, the timezone and
the spec's packages. Moving them to layer 2 makes them re-appliable - change a
VM's archive_snapshot or the fleet timezone and the next playbook run reaches
the running guest, where before it needed a rebuild.

It is applied via roles: rather than the spec's include loop, so no inventory
file can opt out and it always lands before any role installs a package - the
snapshot pin has to be in force first.

Both apt.conf.d filenames are the ones cloud-init wrote, kept verbatim:
vm-fingerprint.sh records apt config by filename, and renaming them would make
every fingerprint captured so far incomparable. The timers are masked as well
as disabled, which a stock image is not - a package postinst re-running
`systemctl preset` can undo a disable, and a test environment must not change
itself. Expect that to show in the first post-cutover fingerprint diff.

qemu-guest-agent is deliberately absent: apply blocks on the agent reporting
an address and the inventory reads that address from the agent, so ansible
cannot be what installs it. It comes from the golden image.

The timezone task is the repo's first non-builtin FQCN, which CI could not
have resolved - it installs bare ansible-core while requirements.txt pins the
batteries-included ansible package, so community.* passes locally and fails
there. ansible/requirements.yml pins what is used and the workflow installs
from it, which closes that asymmetry rather than working around it. Measured
at 8s and 27 MB, against 12s of apt work Step 6 removes from the same job.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MzjHsAQ3HkQjLdgv5PVi55
---
 .github/workflows/validate.yaml    |  4 ++
 ansible/inventory/tofu.py          |  8 +++-
 ansible/requirements.yml           | 21 +++++++++
 ansible/roles/base/tasks/main.yaml | 75 ++++++++++++++++++++++++++++++
 ansible/site.yaml                  | 19 +++++---
 scripts/check-ansible.sh           |  3 +-
 6 files changed, 122 insertions(+), 8 deletions(-)
 create mode 100644 ansible/requirements.yml
 create mode 100644 ansible/roles/base/tasks/main.yaml

diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml
index 0520b5e..4cea76f 100644
--- a/.github/workflows/validate.yaml
+++ b/.github/workflows/validate.yaml
@@ -33,6 +33,10 @@ jobs:
           # venvs; ansible-lint pulls ansible-core into its venv itself.
           pipx install ansible-core
           pipx install ansible-lint
+          # ansible-core ships no community collections, so without this the
+          # roles' non-builtin FQCNs resolve locally (requirements.txt pins the
+          # batteries-included `ansible` package) and fail here.
+          ansible-galaxy collection install -r ansible/requirements.yml
 
       - name: copy examples
         run: |
diff --git a/ansible/inventory/tofu.py b/ansible/inventory/tofu.py
index f8ac991..07fdae4 100755
--- a/ansible/inventory/tofu.py
+++ b/ansible/inventory/tofu.py
@@ -10,7 +10,8 @@
   - group "vms" holding every reachable VM
   - one group per declared role, for ad-hoc targeting (ansible bun -m ...)
   - hostvars: ansible_host (first agent-reported IPv4), ansible_user,
-    vm_id, vm_ansible_roles (what site.yaml applies)
+    vm_id, vm_ansible_roles (what site.yaml applies), and the three the base
+    role consumes: vm_packages, vm_archive_snapshot (may be None), vm_timezone
 
 A VM with no reported address (powered off, or agent not up yet) is skipped
 with a notice on stderr: an unreachable-by-design host in inventory would
@@ -70,6 +71,11 @@ def build_inventory():
             "ansible_user": vm["ansible_user"],
             "vm_id": vm["vm_id"],
             "vm_ansible_roles": vm.get("ansible_roles", []),
+            # Applied by the base role rather than at first boot, which is why
+            # they ride the inventory instead of the cloud-init drive.
+            "vm_packages": vm.get("packages", []),
+            "vm_archive_snapshot": vm.get("archive_snapshot"),
+            "vm_timezone": vm["timezone"],
         }
         inventory["vms"]["hosts"].append(name)
         for role in vm.get("ansible_roles", []):
diff --git a/ansible/requirements.yml b/ansible/requirements.yml
new file mode 100644
index 0000000..211a343
--- /dev/null
+++ b/ansible/requirements.yml
@@ -0,0 +1,21 @@
+---
+# Collections used beyond ansible.builtin, pinned like every other version in
+# this repo.
+#
+# CI installs exactly these (.github/workflows/validate.yaml). Locally they
+# arrive incidentally, because requirements.txt pins the batteries-included
+# `ansible` package rather than `ansible-core` - so the versions below are the
+# ones that package currently ships. If the two ever diverge, local checks and
+# CI stop agreeing; the fix is to move requirements.txt to ansible-core and
+# install from this file locally too.
+collections:
+  # timezone. The only declarative option for it - ansible.builtin has none.
+  - name: community.general
+    version: "13.3.0"
+
+  # Not used yet. Declared with community.general because it is the other
+  # collection a system-level role reaches for first (sysctl, mount,
+  # pam_limits), and adding it now costs one pinned line instead of a second
+  # round of CI plumbing later.
+  - name: ansible.posix
+    version: "2.2.2"
diff --git a/ansible/roles/base/tasks/main.yaml b/ansible/roles/base/tasks/main.yaml
new file mode 100644
index 0000000..a7d0483
--- /dev/null
+++ b/ansible/roles/base/tasks/main.yaml
@@ -0,0 +1,75 @@
+# Everything every VM gets, applied to every host before its spec-declared
+# roles. This is what the cloud-init snippet used to do at first boot, moved
+# to a layer that can be re-applied: change a VM's archive_snapshot, packages
+# or the fleet timezone and the next ansible-playbook run reaches the running
+# guest, instead of needing a rebuild.
+#
+# Not here, and deliberately: qemu-guest-agent. Apply blocks until the agent
+# reports an address and the dynamic inventory reads that address from the
+# agent, so ansible cannot be what installs it. It comes from the golden image
+# (scripts/build-image.sh).
+
+- name: Apply the fleet baseline
+  become: true
+  block:
+    # Both apt.conf.d filenames below are the ones cloud-init wrote, kept
+    # verbatim. "cloudinit" is a misnomer now, but scripts/vm-fingerprint.sh
+    # records apt configuration by filename - renaming them would make every
+    # fingerprint captured so far incomparable for no gain.
+    - name: Pin apt to the spec's archive snapshot
+      ansible.builtin.copy:
+        dest: /etc/apt/apt.conf.d/50cloudinit-snapshot
+        content: |
+          APT::Snapshot "{{ vm_archive_snapshot }}";
+        mode: "0644"
+      when: vm_archive_snapshot is not none
+
+    - name: Remove the apt snapshot pin when the spec has none
+      ansible.builtin.file:
+        path: /etc/apt/apt.conf.d/50cloudinit-snapshot
+        state: absent
+      when: vm_archive_snapshot is none
+
+    # The golden image ships this file too. Re-asserted here so a VM built
+    # from a stock cloud image is still safe, and so the fleet's stance
+    # survives anything that rewrites 20auto-upgrades.
+    - name: Zero the apt periodic jobs
+      ansible.builtin.copy:
+        dest: /etc/apt/apt.conf.d/51cloudinit-no-auto-upgrades
+        content: |
+          APT::Periodic::Update-Package-Lists "0";
+          APT::Periodic::Unattended-Upgrade "0";
+        mode: "0644"
+
+    # Zeroing the periodic jobs already makes these no-ops; disabling and then
+    # masking them is the belt-and-braces version. Masked rather than merely
+    # disabled because a package postinst re-running `systemctl preset` can
+    # undo a disable, and a test environment must not change itself.
+    - name: Stop and disable the apt-daily timers
+      ansible.builtin.systemd_service:
+        name: "{{ item }}"
+        enabled: false
+        state: stopped
+      loop:
+        - apt-daily.timer
+        - apt-daily-upgrade.timer
+
+    - name: Mask the apt-daily timers
+      ansible.builtin.systemd_service:
+        name: "{{ item }}"
+        masked: true
+      loop:
+        - apt-daily.timer
+        - apt-daily-upgrade.timer
+
+    - name: Set the timezone
+      community.general.timezone:
+        name: "{{ vm_timezone }}"
+
+    # After the snapshot pin above, so the pin is in force for this install.
+    - name: Install the spec's packages
+      ansible.builtin.apt:
+        name: "{{ vm_packages }}"
+        state: present
+        update_cache: true
+      when: vm_packages | length > 0
diff --git a/ansible/site.yaml b/ansible/site.yaml
index 3cd8111..aa96ed3 100644
--- a/ansible/site.yaml
+++ b/ansible/site.yaml
@@ -1,10 +1,17 @@
-# Applies each VM's spec-declared ansible_roles (inventory/<vm>.yaml ->
-# tofu output -> inventory/tofu.py hostvars). Adding a role to a VM never
-# touches this file; per-VM ordering follows the spec's list. A role name
-# with no ansible/roles/ directory fails loudly here AND at tofu plan time -
-# never a silent no-op.
-- name: Apply per-VM ansible roles
+# Applies the base role to every VM, then each VM's spec-declared
+# ansible_roles (inventory/<vm>.yaml -> tofu output -> inventory/tofu.py
+# hostvars). Adding a role to a VM never touches this file; per-VM ordering
+# follows the spec's list. A role name with no ansible/roles/ directory fails
+# loudly here AND at tofu plan time - never a silent no-op.
+#
+# base is not in any spec and cannot be opted out of: it carries what used to
+# be the cloud-init snippet, so every VM needs it. roles: runs ahead of
+# tasks:, which is what puts it before the include loop - the apt snapshot pin
+# it writes has to be in force before any role installs a package.
+- name: Apply the fleet baseline and each VM's declared roles
   hosts: vms
+  roles:
+    - base
   tasks:
     - name: Include each role the VM's spec declares
       ansible.builtin.include_role:
diff --git a/scripts/check-ansible.sh b/scripts/check-ansible.sh
index fada8eb..689f341 100755
--- a/scripts/check-ansible.sh
+++ b/scripts/check-ansible.sh
@@ -36,7 +36,8 @@ hostvars = inv.get("_meta", {}).get("hostvars", {})
 if not hostvars:
     sys.exit("no reachable VMs in inventory - is the fleet applied and running?")
 for host, hv in hostvars.items():
-    for key in ("ansible_host", "ansible_user", "vm_ansible_roles"):
+    for key in ("ansible_host", "ansible_user", "vm_ansible_roles",
+                "vm_packages", "vm_archive_snapshot", "vm_timezone"):
         if key not in hv:
             sys.exit(f"{host}: hostvar {key!r} missing")
     for role in hv["vm_ansible_roles"]:

From 2e4ca5f51b4cdeaa3d6848b056b5300fd3585ee6 Mon Sep 17 00:00:00 2001
From: vpzed-dev-lux <320706889+vpzed-dev-lux@users.noreply.github.com>
Date: Fri, 28 Aug 2026 15:33:50 +0000
Subject: [PATCH 06/13] delete the cloud-init layer and rewrite the docs around
 it

Nothing reads cloud-init/ any more, so the templates and the check script that
guarded them go. With check-cloud-init.sh gone, CI loses the `cloud-init` and
`python3-yaml` apt install, its own step, and tofu_wrapper: false - the wrapper
was disabled solely because it buffered stdio and hung that script's
`tofu console` pipe.

The docs carried the old three-layer story throughout, so this rewrites rather
than patches: a golden image before first boot, PVE's native cloud-init for
identity, ansible for everything else. Specifically - Prerequisites and Setup
lose the snippets datastore, the Datastore.Allocate paragraph, the
OpenTofuSnippetStore role and the whole provisioning-SSH-key step; Credentials
drops to two rows and says why the third is gone; the repeatability notes name
the base role instead of bootcmd, and gain the distinction that now matters -
packages, archive_snapshot and the timezone reach a running VM, package_upgrade
does not. A new "Building the golden image" section sits between Setup and
startup, which is where a reader following the setup needs it.

CONTRIBUTING.md's "run what CI runs" block matched CI, so it had to move with
it, and it gains the rule that any collection beyond ansible.builtin must be
pinned in the same change that uses it.

Opportunistic fixes found while reading: "subdiredtory", optel-lgtm ->
otel-lgtm (the image really is grafana/otel-lgtm), inventory/example.yaml ->
inventory-example.yaml, and the Fingerprints section, which described a
tracked file and a -build1/-build2 naming the repo does not use - fingerprints/
is gitignored and the workflow captures to one path and diffs it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MzjHsAQ3HkQjLdgv5PVi55
---
 .github/workflows/validate.yaml   |  16 +--
 CLAUDE.md                         | 142 ++++++++++---------
 CONTRIBUTING.md                   |  24 ++--
 README.md                         | 223 ++++++++++++++++++------------
 cloud-init/base.runcmd.json.tftpl |   4 -
 cloud-init/base.yaml.tftpl        |  64 ---------
 scripts/check-ansible.sh          |   3 +-
 scripts/check-cloud-init.sh       | 181 ------------------------
 8 files changed, 228 insertions(+), 429 deletions(-)
 delete mode 100644 cloud-init/base.runcmd.json.tftpl
 delete mode 100644 cloud-init/base.yaml.tftpl
 delete mode 100755 scripts/check-cloud-init.sh

diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml
index 4cea76f..e433620 100644
--- a/.github/workflows/validate.yaml
+++ b/.github/workflows/validate.yaml
@@ -17,20 +17,11 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: opentofu/setup-opentofu@v1
-        with:
-          # The wrapper buffers tofu's stdio to capture outputs, which breaks
-          # the `printf ... | tofu console` pipe inside check-cloud-init.sh
-          # (the pipe hangs). We don't consume outputs, so run tofu directly.
-          tofu_wrapper: false
 
       - name: Install validation dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends cloud-init python3-yaml
-          # pipx, not pip --user: a --user-site jsonschema shadows the system
-          # one and crashes the runner's cloud-init (`schema` hits a
-          # registry= kwarg mismatch). pipx keeps ansible's deps in their own
-          # venvs; ansible-lint pulls ansible-core into its venv itself.
+          # pipx, not pip --user: it keeps ansible's dependencies in their
+          # own venvs. ansible-lint pulls ansible-core into its venv itself.
           pipx install ansible-core
           pipx install ansible-lint
           # ansible-core ships no community collections, so without this the
@@ -50,9 +41,6 @@ jobs:
           tofu init -input=false
           tofu validate
 
-      - name: check-cloud-init
-        run: ./scripts/check-cloud-init.sh
-
       # check-ansible.sh needs an applied tofu state (it inspects the live
       # dynamic inventory), so CI runs only its state-independent parts.
       - name: ansible syntax check
diff --git a/CLAUDE.md b/CLAUDE.md
index 79aa161..a0ec7cf 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -1,8 +1,9 @@
 # OpenTofu provisioning for ProxMox VE
 
 Infrastructure as code for provisioning and deprovisioning VMs on ProxMox VE.
-Three layers: OpenTofu drives the hypervisor, cloud-init gives the guest its
-boot-time identity, and Ansible installs software post-boot.
+Three layers: a golden image carries what has to exist before first boot,
+OpenTofu drives the hypervisor and PVE's native cloud-init settings, and
+Ansible owns everything after boot.
 See README.md for setup; this file is what a working session needs that is not
 obvious from reading the code.
 
@@ -11,10 +12,12 @@ obvious from reading the code.
 Template placeholders — fill in for your site (marked `EDIT` in the code):
 
 - ProxMox node name/endpoint: `variables.tofu` (`pve_node`, `pve_endpoint`).
-- Datastores: snippets need a dir datastore with the `snippets` content type;
-  lvmthin cannot hold them.
-- Base image: referenced directly as a disk `file_id`, so ProxMox imports and
-  converts it; there is no `download_file` resource.
+- Datastores: `disk_datastore` holds VM disks and cloud-init drives; the
+  golden image lives on any datastore with the `iso` content type.
+- Golden image: built by `scripts/build-image.sh` and referenced directly as a
+  disk `file_id`, so ProxMox imports and converts it; there is no
+  `download_file` resource. It exists to carry `qemu-guest-agent` — see the
+  gotcha below.
 - Secrets: `secrets.enc.json`, age via sops, key at
   `~/.config/sops/age/keys.txt`. Holds the API token and the state passphrase.
 
@@ -42,19 +45,30 @@ the only required variable. See `terraform.tfvars.example`.
 
 Adding a VM is adding a file to `inventory/`; deprovisioning is removing one —
 by convention, `git mv` it to `inventory/destroy/` (only `inventory/*.yaml` is
-scanned; subdirectories are invisible). The next apply destroys the VM, its
-disk, and its snippet; moving the file back provisions a *fresh* VM. To keep a
+scanned; subdirectories are invisible). The next apply destroys the VM and its
+disk; moving the file back provisions a *fresh* VM. To keep a
 VM and its data but power it off, set `started: false` instead. The filename
 is the VM name (DNS label). Only `vm_id` is required — everything else takes
 an `optional()` default from the `spec` object in `modules/vm-pve/variables.tofu`,
 which is the contract worth reading first.
 
-After touching anything under `cloud-init/`, run
-`./scripts/check-cloud-init.sh`. `tofu validate` checks HCL and structurally
-never sees the YAML that comes out of `templatefile()`.
-
-**Layer 2 — Ansible.** Cloud-init is decided at first boot and reachable only
-by recreating the VM; everything softer — installed software and its config —
+**Layer 0 — the golden image.** `./scripts/build-image.sh [--upload]` builds
+it: an upstream Ubuntu cloud image with `qemu-guest-agent` installed by
+`virt-customize`, uploaded over the API. Rebuild when the upstream serial
+should move; then repoint `cloud_image_file_id` at the new name. Never replace
+an uploaded image in place — `disk[0].file_id` is under `ignore_changes`, so a
+same-name replacement changes what a running VM was built from with no plan
+diff. Prerequisites are `libguestfs-tools`, a readable
+`/boot/vmlinuz-$(uname -r)` (`dpkg-statoverride`), and membership of `kvm`;
+the script's preflight names each remedy.
+
+**Layer 1 — PVE cloud-init.** `initialization` in `modules/vm-pve/main.tofu`:
+user, keys, addresses, DNS, and `upgrade`. These are VM config over the API, so
+editing one *does* diff — but cloud-init only reads the drive on first boot, so
+a diff still does not reach a running guest. Rotating a key on a live VM means
+ansible or a rebuild.
+
+**Layer 2 — Ansible.** Everything softer — installed software and its config —
 lives in `ansible/` and is re-applied any time with
 `ansible-playbook ansible/site.yaml [--limit <vm>]` (after `source tofu.env`,
 which exports `ANSIBLE_CONFIG`; no plan diff, no recreation). A spec opts in
@@ -65,7 +79,15 @@ which serves interactive shells only). Underscore names,
 each a directory under `ansible/roles/`, typos fail at plan time. The dynamic
 inventory (`ansible/inventory/tofu.py`) reads `tofu output -json vms`, so it
 needs an applied state; versions are pinned in each role's
-`defaults/main.yaml`. Removing a role from the list does **not** uninstall it —
+`defaults/main.yaml`, and collections in `ansible/requirements.yml` (CI
+installs from it — `ansible-core` alone ships none, so a `community.*` task
+without a pin there passes locally and fails CI). The `base` role is in no
+spec and cannot be opted out of: `site.yaml` applies it to every host via
+`roles:`, ahead of the include loop. It carries what the cloud-init snippet
+used to — the apt snapshot pin, the `APT::Periodic` zeros, the apt-daily
+timers, the timezone, and `spec.packages` — which means those **are** now
+reachable without recreating the VM. Removing a role from the list does
+**not** uninstall it —
 clean removal is a rebuild. Runs are idempotent (second run reports
 changed=0). After touching anything under `ansible/`, run
 `./scripts/check-ansible.sh`.
@@ -80,62 +102,54 @@ Rebuild verification: capture, commit, destroy + reprovision, run
 
 ## Gotchas that have already cost time
 
-- **Snippets need SSH.** The ProxMox API has no snippets endpoint, so the
-  provider uploads cloud-init user-data over SSH as root using
-  `pve_ssh_private_key_path`. Everything else goes over the API token.
-- **The file resource also needs `Datastore.Allocate` on the snippet
-  datastore.** Before the SSH upload, the provider reads the storage config
-  via `GET /storage/<id>`, which PVE gates behind the full admin privilege —
-  `AllocateSpace`/`AllocateTemplate` are not enough (HTTP 403 at apply).
-  Grant it via a role scoped to the snippet datastore, and give that role the
-  *complete* `Datastore.*` set: PVE ACLs on a more specific path **override**
-  the propagated role instead of merging with it.
+- **`ciupgrade` is not root-gated, whatever the provider says.** The bpg
+  schema calls `initialization.upgrade` *"only allowed for `root@pam`"*; that
+  is stale. In `PVE/API2/Qemu.pm` (9.2.3) `ciupgrade` sits in
+  `$cloudinitoptions`, which needs `VM.Config.Cloudinit` **or**
+  `VM.Config.Network`. It is set explicitly because the provider's schema
+  default is Computed and PVE's own default is 1 — the opposite of
+  `spec.package_upgrade`'s default.
 - **An `@pve`-realm token cannot SSH anywhere.** It exists only in ProxMox's
-  user database — it is not a Linux account. The API identity and the SSH
-  identity can never be unified.
-- **A cloud-config is one YAML mapping; duplicate top-level keys get silently
-  dropped.** That is why `base.yaml.tftpl` is the entire document and nothing
-  is ever appended to it — post-boot software belongs in an ansible role, not
-  in cloud-init.
-- **Every scalar interpolated into a cloud-init template must be
-  `jsonencode()`d.** JSON is a YAML subset; a raw SSH key comment containing
-  `: ` becomes a YAML mapping (VM boots with no authorized keys), a package
-  containing `#` is silently truncated, and a VM named `no` becomes a boolean
-  hostname. `check-cloud-init.sh` has an adversarial pass that regresses this.
-- **Editing cloud-init templates does not diff the VM.** The snippet
-  re-uploads but its ID (derived from the stable file name) is unchanged, so
-  the plan shows "0 to change" and running guests keep what they booted with.
-  Reaching an existing VM means recreating it: delete its inventory file,
-  apply, restore, apply.
+  user database — it is not a Linux account. Worth knowing before anyone
+  proposes reuniting the API and SSH identities to bring snippets back.
+- **`filename=@` must be the last `-F` in a PVE upload.** PVE parses the
+  multipart body in order and streams everything after the file part into the
+  file. A `checksum` field placed after it is never parsed *and* its bytes are
+  appended to the image — which `qemu-img` reads without complaint, so the
+  task reports OK and the corruption is silent. `scripts/build-image.sh` has
+  the order right and the node then verifies the checksum itself.
+- **`APT::Snapshot` cannot be used while building the image.** It adds the
+  snapshot mirror alongside the configured one and refreshes both, and the
+  cloud image's root filesystem has ~366 MB free against a ~120 MB unpacked
+  universe index — the build dies in `dpkg --unpack`. `build-image.sh`
+  rewrites `ubuntu.sources` to the snapshot mirror for the duration instead.
+  Unrelated to a VM's own `archive_snapshot`, which still goes via apt.conf.d.
 - **Repeatability knobs live per-VM in the spec.** `archive_snapshot:
-  YYYYMMDDTHHMMSSZ` pins apt to snapshot.ubuntu.com via a `bootcmd` that
-  writes `APT::Snapshot` into apt.conf.d (bootcmd runs init-stage, before
-  apt-configure and package install; an apt.conf.d file survives cloud-init
-  regenerating `ubuntu.sources`). Official archive only — third-party repos
-  like docker are not snapshotted, which is why the docker ansible role pins
-  exact package versions in its defaults instead.
-  `package_upgrade` defaults to **false**; `package_update: true`
-  is explicit but not load-bearing — cloud-init refreshes indexes whenever
-  `packages:` is non-empty. unattended-upgrades is disabled fleet-wide (the
-  image ships it enabled): base's `bootcmd` zeros the `APT::Periodic` jobs and
-  a base runcmd disables the apt-daily timers.
-- **PyYAML's `safe_load` does not error on duplicate keys**, it keeps the last
-  one. `scripts/check-cloud-init.sh` installs a custom loader for this reason.
-- **`cloud-init schema` catches deprecations that still "work."** The check
-  script treats deprecation warnings as failures.
+  YYYYMMDDTHHMMSSZ` pins apt to snapshot.ubuntu.com; the ansible `base` role
+  writes `APT::Snapshot` into apt.conf.d before it installs anything, so the
+  pin is in force for the spec's own packages. Official archive only —
+  third-party repos like docker are not snapshotted, which is why the docker
+  role pins exact package versions in its defaults instead. `package_upgrade`
+  defaults to **false** and is the one first-boot-only knob left: it becomes
+  PVE's `ciupgrade`, so flipping it does not reach an existing VM, where
+  `packages` and `archive_snapshot` now do. unattended-upgrades is disabled
+  fleet-wide (the image ships it enabled): `base` zeros the `APT::Periodic`
+  jobs and masks the apt-daily timers — masked, not merely disabled, because a
+  postinst re-running `systemctl preset` can undo a disable.
+- **The apt.conf.d filenames still say "cloudinit"** (`50cloudinit-snapshot`,
+  `51cloudinit-no-auto-upgrades`) and must stay that way.
+  `scripts/vm-fingerprint.sh` records apt configuration by filename; renaming
+  them would make every fingerprint captured so far incomparable.
 - **A child module must declare its own `required_providers`** naming
   `bpg/proxmox`. Without `modules/vm-pve/versions.tofu`, `tofu init` assumes
   `hashicorp/proxmox` and fails.
 - **`agent { enabled = true }` makes apply block** until the guest agent
   reports an address. Quick with the default `package_upgrade: false`; with a
-  VM that sets it true, expect several minutes (timeout is 30m).
-- **zsh's `echo` expands `\n`**, which corrupts expressions piped to
-  `tofu console`. Use a quoted heredoc.
-- **`tofu console` output is double-encoded** when wrapping `jsonencode` — HCL
-  string quoting on top of the JSON. Two decode passes.
-- **Once a state file exists, `tofu console` prints "Acquiring state lock..."
-  to STDOUT**, corrupting anything piped from it — hence `-lock=false` and
-  the result-line filter in `check-cloud-init.sh`.
+  VM that sets it true, expect several minutes (timeout is 30m). The agent
+  comes from the golden image, and nothing else can supply it — ansible needs
+  the address the agent reports in order to connect at all. Point
+  `cloud_image_file_id` at a stock cloud image and apply hangs for the full
+  30m, then fails.
 - **pbkdf2 welds its salt to the key provider's block name** unless
   `encrypted_metadata_alias` is set — renaming a provider (or moving a
   passphrase between blocks) makes existing state undecryptable. Both
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 6b50bf3..8ce03f9 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -31,7 +31,9 @@ or tokens.
 With a ProxMox node of your own: copy `terraform.tfvars.example` to
 `terraform.tfvars`, fill in the `EDIT` values (see README.md for the full
 setup, including `secrets.enc.json` — layout in `secrets.enc.json.example`),
-then `source tofu.env` and use `inventory/example.yaml` as a starting spec.
+then `source tofu.env` and use `inventory-example.yaml` as a starting spec.
+You will also need a golden image on the node — `./scripts/build-image.sh
+--upload` builds and uploads one; see README.md, "Building the golden image".
 
 Without a node, run what CI runs:
 
@@ -39,7 +41,7 @@ Without a node, run what CI runs:
 export TF_VAR_state_passphrase=ci-dummy-passphrase-render-only
 tofu init -input=false
 tofu validate
-./scripts/check-cloud-init.sh          # needs: cloud-init, python3-yaml
+ansible-galaxy collection install -r ansible/requirements.yml
 ansible-playbook ansible/site.yaml --syntax-check -i localhost,
 (cd ansible && ansible-lint)
 ```
@@ -47,20 +49,16 @@ ansible-playbook ansible/site.yaml --syntax-check -i localhost,
 The dummy passphrase satisfies the state-encryption config for read-only
 rendering; no state is created or read.
 
-Notes on the two check scripts:
-
-- `check-cloud-init.sh` renders the real cloud-init output and validates it —
-  `tofu validate` alone never sees the YAML that comes out of
-  `templatefile()`. Run it after touching anything under `cloud-init/`.
-- `check-ansible.sh` needs an applied state (it inspects the live inventory),
-  so it only runs against a real environment; CI covers the state-independent
-  parts (syntax check and lint). Run the full script if you have a node.
+`check-ansible.sh` needs an applied state (it inspects the live inventory), so
+it only runs against a real environment; CI covers the state-independent parts
+(syntax check and lint). Run the full script if you have a node.
 
 ## Style
 
-- Every scalar interpolated into a cloud-init template must be
-  `jsonencode()`d — see the comments in `cloud-init/base.yaml.tftpl` and the
-  adversarial pass in `check-cloud-init.sh` that regresses this.
+- Any collection beyond `ansible.builtin` must be pinned in
+  `ansible/requirements.yml` in the same change that uses it. CI installs
+  `ansible-core`, which ships none, so an unpinned `community.*` task passes
+  locally and fails there.
 - Ansible role names use underscores; each is a directory under
   `ansible/roles/`. Pin software versions in the role's `defaults/main.yaml`.
 - Keep guard behavior intact: `guards.tofu` and the validations in
diff --git a/README.md b/README.md
index e83bec8..13269cd 100644
--- a/README.md
+++ b/README.md
@@ -10,13 +10,14 @@ the environment around the software under test stays fixed — same image,
 same package versions. The VMs are designed to be ephemeral.
 
 Core components for VM management: 
-- OpenTofu drives the ProxMox VE hypervisor (bpg/proxmox provider)
-- cloud-init gives the guest its boot-time identity
-- Ansible installs software post-boot — re-appliable at any time without 
+- A golden image carries what has to exist before first boot (the guest agent)
+- OpenTofu drives the ProxMox VE hypervisor (bpg/proxmox provider) and its 
+native cloud-init settings — identity, keys, addressing
+- Ansible owns everything after boot — re-appliable at any time without 
 touching the VM lifecycle.
 
 Persistent backend components:
-- A grafana/optel-lgtm container provides an OpenTelemetry endpoint to 
+- A grafana/otel-lgtm container provides an OpenTelemetry endpoint to 
 capture and store Claude Code telemetry from agents running during testing. 
 - A windmill-labs/windmill container is being evaluated as an orchestrator for the
 overall testing workflow.
@@ -24,7 +25,7 @@ overall testing workflow.
 ## Overview 
 
 - **One YAML file per VM.** Provisioning is adding a file to `inventory/`;
-  deprovisioning is deleting one, or moving it to the destroy subdiredtory. 
+  deprovisioning is deleting one, or moving it to the destroy subdirectory. 
   Only `vm_id` is required — everything else inherits a typed `optional()` 
   default from the `spec` contract in `modules/vm-pve/variables.tofu`.
 - **Repeatable environments.** Per-VM `archive_snapshot:` pins apt to
@@ -46,13 +47,12 @@ overall testing workflow.
 - **Encrypted state and plans** (pbkdf2 + AES-GCM, `enforced = true`), secrets
   via sops/age, and a `.gitignore` that covers the sharp edges (`crash.log`
   contains a TRACE-level dump including the API token, regardless of TF_LOG).
-- **Real validation gates.** `tofu validate` never sees the YAML that comes
-  out of `templatefile()`; `scripts/check-cloud-init.sh` renders the real
-  user-data plus an adversarial fixture (SSH key comment containing `: `,
-  package containing `#`, hostname that is a YAML boolean) and runs
-  duplicate-key and `cloud-init schema` checks, treating deprecation warnings
-  as failures. `scripts/check-ansible.sh` asserts the dynamic inventory's
-  shape and syntax-checks (and, when installed, lints) the playbook.
+- **Real validation gates.** `scripts/check-ansible.sh` asserts the dynamic
+  inventory's shape — every hostvar the roles rely on, every declared role
+  backed by a real directory — and syntax-checks and lints the playbook.
+  `scripts/build-image.sh` verifies the upstream image against its published
+  `SHA256SUMS`, asserts inside the built image that the build-time apt pin did
+  not leak, and has the node verify the upload's checksum.
 
 ## Layout
 
@@ -64,17 +64,15 @@ overall testing workflow.
 ├── variables.tofu         # fleet-wide defaults                        [EDIT]
 ├── outputs.tofu
 ├── modules/vm-pve/        # the contract: what a VM is
-├── cloud-init/
-│   ├── base.yaml.tftpl            # every VM gets this - the whole document
-│   └── base.runcmd.json.tftpl     # commands every VM runs
 ├── ansible/               # post-boot software
 │   ├── ansible.cfg
-│   ├── site.yaml                  # one hostvar-driven play
+│   ├── requirements.yml           # pinned collections (CI installs these)
+│   ├── site.yaml                  # base, then each VM's declared roles
 │   ├── inventory/tofu.py          # dynamic inventory from tofu output
-│   └── roles/<role>/              # nats_server, bun, claude, docker,
-│                                  # metafactory_arc
+│   └── roles/<role>/              # base (every VM), then nats_server, bun,
+│                                  # claude, docker, metafactory_arc
 ├── inventory/             # one YAML file per VM                       [EDIT]
-├── scripts/               # check-cloud-init.sh, check-ansible.sh,
+├── scripts/               # build-image.sh, check-ansible.sh,
 │                          # vm-fingerprint.sh
 ├── otel-lgtm/             # grafana/otel-lgtm observability stack      [EDIT]
 │                          # (docker compose; see its README.md)
@@ -90,8 +88,8 @@ same system that is running opentofu and ansible, but that is not required.
 ## Prerequisites
 
 - OpenTofu >= 1.10, `sops`, `age`, `ansible` (ansible-core >= 2.15;
-  `ansible-lint` optional), and (for the cloud-init check script)
-  `python3-yaml` and `cloud-init` on the workstation.
+  `ansible-lint` optional) on the workstation, plus `libguestfs-tools` if you
+  build the golden image there (see "Building the golden image").
 
 - OpenTofu, https://github.com/opentofu/opentofu/releases/tag/v1.12.6
 - sops, https://github.com/getsops/sops/releases/tag/v3.13.3
@@ -99,18 +97,12 @@ same system that is running opentofu and ansible, but that is not required.
 - uv, https://github.com/astral-sh/uv/releases/tag/0.12.6
 
 - A ProxMox VE node (built against 9.x) with:
-  - a datastore that allows the `snippets` content type (`local` by default;
-    lvmthin cannot hold snippets),
-  - an Ubuntu cloud image uploaded (e.g.
-    `local:iso/resolute-server-cloudimg-amd64-20260720.img`),
-  - an API token for provisioning, and root SSH access for snippet upload
-    (see Credentials below). Besides the usual `VM.*`/`Datastore.*`/`SDN.Use`
-    privileges, the token needs **`Datastore.Allocate` on the snippet
-    datastore** — the provider reads the storage config (`GET /storage/<id>`)
-    before uploading, and PVE gates that behind the full admin privilege.
-    Grant it via a role scoped to `/storage/<snippet-datastore>`, and give
-    that role the *complete* `Datastore.*` set: PVE ACLs on a specific path
-    override propagated ones instead of merging.
+  - a datastore that allows the `iso` content type for the golden image
+    (`local` by default),
+  - a golden image uploaded (see "Building the golden image"), and
+  - an API token for provisioning, with the usual
+    `VM.*`/`Datastore.*`/`SDN.Use` privileges. That is the only credential
+    this project needs — nothing here has, or wants, SSH to the node.
 
 ## Setup
 
@@ -126,8 +118,7 @@ same system that is running opentofu and ansible, but that is not required.
    `secrets.enc.json` holds the ProxMox API token and the state passphrase
    (16 characters minimum). Once encrypted it is safe to commit.
 
-2. **API user, roles, and token.** As root on the node (adjust the snippet
-   datastore path if yours is not `local`):
+2. **API user, role, and token.** As root on the node:
 
    ```bash
    pveum user add opentofu-prov@pve --comment "OpenTofu provisioning (API token only)"
@@ -135,24 +126,59 @@ same system that is running opentofu and ansible, but that is not required.
    pveum role add OpenTofuProv -privs "Datastore.AllocateSpace,Datastore.AllocateTemplate,Datastore.Audit,Pool.Allocate,Pool.Audit,SDN.Audit,SDN.Use,Sys.AccessNetwork,Sys.Audit,Sys.Console,Sys.Modify,VM.Allocate,VM.Audit,VM.Clone,VM.Config.CDROM,VM.Config.CPU,VM.Config.Cloudinit,VM.Config.Disk,VM.Config.HWType,VM.Config.Memory,VM.Config.Network,VM.Config.Options,VM.GuestAgent.Unrestricted,VM.Migrate,VM.PowerMgmt"
    pveum acl modify / -user opentofu-prov@pve -role OpenTofuProv
 
-   # Scoped role for the snippet datastore; must carry the FULL Datastore.*
-   # set - an ACL on a specific path overrides the propagated role, it does
-   # not merge with it.
-   pveum role add OpenTofuSnippetStore -privs "Datastore.Allocate,Datastore.AllocateSpace,Datastore.AllocateTemplate,Datastore.Audit"
-   pveum acl modify /storage/local -user opentofu-prov@pve -role OpenTofuSnippetStore
-
    # privsep=0: the token inherits the user's ACLs. The secret prints ONCE -
    # it goes into secrets.enc.json (proxmox.api_token_secret).
    pveum user token add opentofu-prov@pve provisioning --privsep 0
    ```
 
-3. **Provisioning SSH key.** The ProxMox API has no snippets endpoint, so the
-   provider uploads cloud-init user-data over SSH as root:
+   `Datastore.AllocateTemplate` is what lets the same token upload the golden
+   image, so no second identity is needed.
 
-   ```bash
-   ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519_pve -N ""
-   ssh-copy-id -i ~/.ssh/id_ed25519_pve.pub root@<your-node>
-   ```
+3. **Golden image.** Build and upload one before the first `tofu apply` — see
+   "Building the golden image". A stock cloud image will not do: it has no
+   guest agent, and apply blocks for the full 30-minute timeout waiting for an
+   address that never arrives.
+
+## Building the golden image
+
+Every VM's disk is imported from a golden image: an upstream Ubuntu cloud image
+with `qemu-guest-agent` baked in. The agent is the one thing ansible cannot
+install — `tofu apply` blocks until the agent reports an address, and the
+dynamic inventory reads that same address to connect at all — so it has to be
+present before first boot.
+
+`scripts/build-image.sh` does the whole job. On the machine that builds:
+
+```bash
+sudo apt install libguestfs-tools
+# libguestfs boots its appliance with the host kernel, which ships mode 0600
+sudo dpkg-statoverride --update --add root root 0644 /boot/vmlinuz-$(uname -r)
+sudo usermod -aG kvm "$USER"     # then log out and back in
+```
+
+The script's preflight checks each of those and names the fix if one is
+missing; without `kvm` it still works, just slowly under emulation. Then:
+
+```bash
+./scripts/build-image.sh --upload
+```
+
+which downloads the dated upstream serial, verifies it against that
+directory's `SHA256SUMS`, installs the agent with `virt-customize`, and uploads
+the result through the storage API — the same token as everything else, no scp.
+It prints the `volid` to put in `cloud_image_file_id`. Build without `--upload`
+first if you want to inspect the image (`virt-df`, `guestfish --ro`).
+
+Options: `--serial YYYYMMDD` picks the upstream build (default is the one this
+repo currently pins), `--snapshot YYYYMMDDTHHMMSSZ` the snapshot.ubuntu.com
+instant the agent is installed from, and `--outdir` where to work (default
+`private/images/`, gitignored; budget ~2.5 GB).
+
+Rebuild when the upstream serial should move, then repoint
+`cloud_image_file_id` at the new file name. **Never overwrite an uploaded
+image**: the disk source is create-only, so a same-name replacement silently
+changes what a running VM was built from without any plan diff. Build a new
+name, repoint, and delete the old volume once nothing references it.
 
 ## Opentofu and Ansible startup
 
@@ -168,12 +194,12 @@ Note: Edit all the copied files with your specific data.
 - `tofu init` # the first init installs the providers
 - `tofu plan` # verify the output looks like what you expect
 
-Note: The `ssh_public_keys` in terraform.tfvars are the keys that reach the *VMs* 
-NOT the Proxmox VE provisioning key. Override any fleet defaults there too.
+Note: The `ssh_public_keys` in terraform.tfvars are the keys that reach the 
+*VMs*. Override any fleet defaults there too.
 
-## optel-lgtm startup
+## otel-lgtm startup
 
-- `cd optel-lgtm`
+- `cd otel-lgtm`
 - `cp .env.example .env`
 - `docker compose up -d`
 
@@ -201,11 +227,11 @@ NOT the Proxmox VE provisioning key. Override any fleet defaults there too.
 
 - **Add a VM:** create `inventory/<name>.yaml` (the file name becomes the VM
   name and hostname — DNS label rules apply) and `tofu apply`. See
-  `inventory/example.yaml` for every knob including snapshot pinning and
+  `inventory-example.yaml` for every knob including snapshot pinning and
   ansible roles.
 - **Remove a VM:** `git mv inventory/<name>.yaml inventory/destroy/` and
   `tofu apply` — only `inventory/*.yaml` is scanned, so the spec stays in the
-  repo while the VM, its disk, and its snippet are destroyed. Moving it back
+  repo while the VM and its disk are destroyed. Moving it back
   provisions a fresh VM (all guest data is gone). To keep a VM but power it
   off, set `started: false` instead.
 - **Install software:** list roles in the spec
@@ -213,33 +239,39 @@ NOT the Proxmox VE provisioning key. Override any fleet defaults there too.
   `ansible-playbook ansible/site.yaml [--limit <vm>]` — no plan diff, no
   recreation, idempotent (second run reports changed=0). Removing a role from
   the list does *not* uninstall it; clean removal is a rebuild.
-- **After touching anything under `cloud-init/`:** run
-  `./scripts/check-cloud-init.sh`. **Under `ansible/`:**
+- **After touching anything under `ansible/`:** run
   `./scripts/check-ansible.sh`.
 - **Prove a rebuild is identical:**
   `./scripts/vm-fingerprint.sh ubuntu@<ip> fingerprints/<name>.txt` captures
   the VM's environment (packages, apt config, enabled units, users, and the
   ansible-installed files — minus per-instance noise like host keys and
-  machine-id) into a tracked file. Capture, commit, destroy + reprovision,
-  re-run ansible, capture again to the same path: an empty `git diff` is the
-  proof.
+  machine-id). Capture, destroy + reprovision, re-run ansible, capture again
+  to the same path: an empty `git diff` is the proof. `fingerprints/**` is
+  gitignored, so captures stay local unless you commit one deliberately.
 
 ## Fingerprints
 
-Short version of the command sequence to destroy, recreate, and validate the fingerprint assuming the inventory entry is for a VM called ubuntu-test at an IP of 10.0.0.50. 
-This test assumes the first build fingerprint was saved as fingerprints/ubuntu-test-build1.txt. Rebuilding the same configuration should be an empty diff.
+Short version of the command sequence to destroy, recreate, and validate the
+fingerprint, assuming an inventory entry for a VM called ubuntu-test at
+10.0.0.50. Capture to one path throughout and let `git diff` do the comparing;
+rebuilding the same configuration should produce no diff.
 
 **Important**: Always start a session with `source tofu.env`
 
 ```sh
 source tofu.env
+./scripts/vm-fingerprint.sh ubuntu@10.0.0.50 fingerprints/ubuntu-test.txt
+git add -f fingerprints/ubuntu-test.txt   # fingerprints/** is gitignored
+git commit -m "baseline"
+
 git mv inventory/ubuntu-test.yaml inventory/destroy/
 tofu apply
 git mv inventory/destroy/ubuntu-test.yaml inventory/
 tofu apply
 ansible-playbook ansible/site.yaml --limit ubuntu-test
-./scripts/vm-fingerprint.sh ubuntu@10.0.0.50 fingerprints/ubuntu-test-build2.txt
-diff fingerprints/ubuntu-test-build1.txt fingerprints/ubuntu-test-build2.txt
+
+./scripts/vm-fingerprint.sh ubuntu@10.0.0.50 fingerprints/ubuntu-test.txt
+git diff -- fingerprints/ubuntu-test.txt   # empty = identical rebuild
 ```
 
 NOTE: The inventory/destroy directory name is intentional, so it is clear what the next "tofu apply" is expected to do.
@@ -247,9 +279,9 @@ NOTE: The inventory/destroy directory name is intentional, so it is clear what t
 ## Ansible
 
 Cloud-init is decided at first boot and reachable only by recreating the VM —
-that is the right place for identity, network, and the apt baseline, and the
-wrong place for software. Everything softer lives in `ansible/` and follows
-the same declarative grammar as the rest of the repo:
+that is the right place for identity and network, and the wrong place for
+anything else. Everything softer lives in `ansible/` and follows the same
+declarative grammar as the rest of the repo:
 
 - **Specs declare, roles implement.** `ansible_roles:` in a VM's YAML rides
   through the module into `tofu output -json vms`;
@@ -258,6 +290,14 @@ the same declarative grammar as the rest of the repo:
   `site.yaml` is a single play that `include_role`s each host's declared
   list, so adding a role never touches it, and a typo'd role name fails at
   `tofu plan` time via a `fileexists()` validation in the spec contract.
+- **The `base` role is implicit.** No spec lists it and none can opt out:
+  `site.yaml` applies it to every host via `roles:`, ahead of the include
+  loop. It owns the apt snapshot pin, the `APT::Periodic` zeros, the masked
+  apt-daily timers, the timezone, and the spec's `packages:` — the apt
+  baseline that used to be cloud-init's, now re-appliable to a running VM.
+- **Collections are pinned** in `ansible/requirements.yml`, and CI installs
+  from it. `ansible-core` ships none, so a `community.*` task without an entry
+  there passes locally and fails in CI.
 - **Role names use underscores** (`nats_server`, not `nats-server`): they
   double as Ansible group names, which must be valid identifiers.
 - **Every download is verified, every version pinned** in the role's
@@ -281,18 +321,22 @@ the same declarative grammar as the rest of the repo:
 
 ## Credentials
 
-Deliberately separate identities, one job each:
+Two credentials, one job each:
 
 | Credential | Authenticates to | Used for |
 |---|---|---|
-| API token (in `secrets.enc.json`) | ProxMox API | everything except snippet upload |
-| `id_ed25519_pve` | `root@<node>` | snippet upload only |
+| API token (in `secrets.enc.json`) | ProxMox API | provisioning, and uploading the golden image |
 | `ssh_public_keys` (tfvars) | `<ci_user>@<vm>` | reaching the VMs |
 
-The provisioning key is root on the hypervisor and exists only to upload YAML
-files; keeping it out of the VMs means a compromised VM never saw the key that
-owns the hypervisor. The API identity (`@pve` realm) is not a Linux account and
-can never be the SSH identity — that split is structural, not a choice.
+There is deliberately no hypervisor SSH credential. One used to exist, because
+the ProxMox API has no snippets endpoint and the provider fell back to SSH as
+root to upload cloud-init user-data. Removing the snippet removed the reason:
+every call this project makes is now an API call carrying the token, and a
+compromised workstation cannot reach root on the node through anything here.
+
+The API identity lives in the `@pve` realm, which is not a Linux account, so it
+could never have been the SSH identity — that split was structural, and is now
+simply absent.
 
 ## Protecting pre-existing VMs
 
@@ -323,23 +367,28 @@ one guard OpenTofu genuinely cannot bypass.
   co-exist for different test setups. Baseline = manifest, delta = inventory
   `packages:`, delta versions = `archive_snapshot` — the whole environment is
   specified without booting anything.
-- Ubuntu cloud images ship apt *sources* but not package *indexes*; cloud-init
-  refreshes indexes automatically whenever `packages:` is non-empty, so
-  installs work regardless of `package_update`/`package_upgrade`.
+- Ubuntu cloud images ship apt *sources* but not package *indexes*; the base
+  role refreshes them (`update_cache: true`) before installing, so the spec's
+  `packages:` resolve against the pin.
 - **unattended-upgrades is disabled on every VM** (the image ships it
-  enabled). Base zeroes the `APT::Periodic` jobs via `bootcmd` and disables
-  the apt-daily timers at first boot — a test environment must not change
-  itself. Remove those lines from `base.yaml.tftpl` /
-  `base.runcmd.json.tftpl` if you *want* automatic security updates.
-- `archive_snapshot:` writes `APT::Snapshot "<ts>";` to apt.conf.d via
-  `bootcmd` (init stage — before apt configures sources and installs packages;
-  re-applied every boot, so later manual `apt install` stays pinned). An
-  apt.conf.d file survives cloud-init regenerating `ubuntu.sources`.
-- Changing a cloud-init template does **not** diff existing VMs (the snippet
-  ID is name-based and unchanged); template changes reach a VM only by
-  recreating it.
+  enabled). The base role zeroes the `APT::Periodic` jobs and *masks* the
+  apt-daily timers — masked rather than merely disabled, because a package
+  postinst re-running `systemctl preset` can undo a disable, and a test
+  environment must not change itself. Drop those tasks from
+  `ansible/roles/base/tasks/main.yaml` if you *want* automatic updates.
+- `archive_snapshot:` writes `APT::Snapshot "<ts>";` to apt.conf.d from the
+  base role, before it installs anything, so the pin is in force for the
+  spec's own packages and for any later manual `apt install`. An apt.conf.d
+  file survives cloud-init regenerating `ubuntu.sources`.
+- **What reaches a running VM, and what does not.** `packages:`,
+  `archive_snapshot:`, `ci_timezone` and every `ansible_roles` entry are
+  applied by ansible, so editing them and re-running the playbook is enough.
+  `package_upgrade` is the exception: it becomes PVE's `ciupgrade`, which
+  cloud-init reads on first boot only, so changing it reaches new VMs only.
 - The disk's image reference is create-only (`ignore_changes`): bumping
-  `cloud_image_file_id` affects new VMs, never existing ones.
+  `cloud_image_file_id` affects new VMs, never existing ones. That is also why
+  an uploaded image must never be replaced in place — same name, different
+  bytes, no plan diff.
 
 ## Rotating the state passphrase
 
diff --git a/cloud-init/base.runcmd.json.tftpl b/cloud-init/base.runcmd.json.tftpl
deleted file mode 100644
index 480fdcc..0000000
--- a/cloud-init/base.runcmd.json.tftpl
+++ /dev/null
@@ -1,4 +0,0 @@
-[
-  ${jsonencode("systemctl enable --now qemu-guest-agent")},
-  ${jsonencode("systemctl disable --now apt-daily.timer apt-daily-upgrade.timer")}
-]
diff --git a/cloud-init/base.yaml.tftpl b/cloud-init/base.yaml.tftpl
deleted file mode 100644
index eeb4421..0000000
--- a/cloud-init/base.yaml.tftpl
+++ /dev/null
@@ -1,64 +0,0 @@
-#cloud-config
-# Rendered by OpenTofu from cloud-init/base.yaml.tftpl - do not edit on the node.
-#
-# Every VM gets this, and only this: post-boot software is Ansible's job
-# (ansible/, layer 2), not cloud-init's. A cloud-config is a single YAML
-# mapping - cloud-init silently drops one of any duplicated top-level pair -
-# so nothing may ever be appended to this render.
-#
-# Every scalar interpolation is jsonencode()d: JSON is a YAML subset, and a raw
-# value containing ': ', '#', or a YAML-1.1 boolean word (an SSH key comment, a
-# hostname like 'no') would otherwise change the document's structure instead
-# of failing loudly.
-hostname: ${jsonencode(vm_name)}
-fqdn: ${jsonencode(fqdn)}
-prefer_fqdn_over_hostname: false
-manage_etc_hosts: true
-timezone: ${jsonencode(timezone)}
-
-users:
-  - name: ${jsonencode(ci_user)}
-    groups: [adm, sudo]
-    shell: /bin/bash
-    sudo: "ALL=(ALL) NOPASSWD:ALL"
-    lock_passwd: true
-    ssh_authorized_keys:
-%{~ for key in ssh_public_keys }
-      - ${jsonencode(key)}
-%{~ endfor }
-
-ssh_pwauth: false
-disable_root: true
-
-# bootcmd runs in the init stage - before apt-configure and package install -
-# and on every boot, so these hold for the VM's whole life.
-bootcmd:
-%{~ if archive_snapshot != null }
-  # Pin apt to the Ubuntu archive as of this instant (snapshot.ubuntu.com),
-  # for repeatable package installs. Third-party sources without snapshot
-  # support are fetched normally, unpinned.
-  - ${jsonencode("echo 'APT::Snapshot \"${archive_snapshot}\";' > /etc/apt/apt.conf.d/50cloudinit-snapshot")}
-%{~ endif }
-  # The image ships unattended-upgrades enabled; a test environment must not
-  # change itself. Zeroing the periodic jobs makes apt-daily a no-op (the
-  # stock 20auto-upgrades enables them; apt reads conf.d in order, last wins).
-  - ${jsonencode("printf '%s\\n' 'APT::Periodic::Update-Package-Lists \"0\";' 'APT::Periodic::Unattended-Upgrade \"0\";' > /etc/apt/apt.conf.d/51cloudinit-no-auto-upgrades")}
-
-# package_update is explicit, not load-bearing: cloud-init refreshes the apt
-# indexes whenever `packages:` is non-empty regardless of this setting.
-package_update: true
-package_upgrade: ${jsonencode(package_upgrade)}
-packages:
-%{~ for pkg in packages }
-  - ${jsonencode(pkg)}
-%{~ endfor }
-
-growpart:
-  mode: auto
-  devices: ["/"]
-  ignore_growroot_disabled: false
-
-runcmd:
-%{~ for cmd in runcmd }
-  - ${jsonencode(cmd)}
-%{~ endfor }
diff --git a/scripts/check-ansible.sh b/scripts/check-ansible.sh
index 689f341..2b95400 100755
--- a/scripts/check-ansible.sh
+++ b/scripts/check-ansible.sh
@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 # Validate the ansible layer: the dynamic inventory's shape and the site
-# playbook's syntax. Sibling of check-cloud-init.sh - run it after touching
-# anything under ansible/.
+# playbook's syntax. Run it after touching anything under ansible/.
 #
 # The inventory check needs the tofu state (that is where the fleet lives),
 # so this requires `source tofu.env` first, same as everything else here.
diff --git a/scripts/check-cloud-init.sh b/scripts/check-cloud-init.sh
deleted file mode 100755
index 34a6642..0000000
--- a/scripts/check-cloud-init.sh
+++ /dev/null
@@ -1,181 +0,0 @@
-#!/usr/bin/env bash
-# Validate the cloud-init user-data this configuration emits.
-#
-# `tofu validate` checks HCL and never sees the YAML that comes out of
-# templatefile(), so this closes that gap. It renders base.yaml.tftpl with the
-# real runcmd list (from base.runcmd.json.tftpl, the same source the module
-# reads) and checks three things:
-#
-#   1. the result parses as YAML with no duplicate mapping keys anywhere
-#      (cloud-init silently drops one of a duplicated pair)
-#   2. it passes `cloud-init schema`, which catches deprecated and malformed keys
-#   3. deprecation warnings are treated as failures, not warnings
-#
-# It then renders one adversarial combination - an SSH key comment containing
-# ': ', a package containing '#', a hostname that is a YAML boolean word - and
-# asserts the values survive as strings. That is the regression test for the
-# jsonencode() escaping in base.yaml.tftpl: without it these inputs silently
-# change the document instead of failing.
-#
-# Requires: python3-yaml, cloud-init, an init'ed tofu.
-
-set -euo pipefail
-
-cd "$(dirname "$0")/.."
-
-if ! python3 -c 'import yaml' 2>/dev/null; then
-  echo "missing PyYAML: sudo apt install python3-yaml" >&2
-  exit 1
-fi
-if ! command -v cloud-init >/dev/null; then
-  echo "missing cloud-init: sudo apt install cloud-init" >&2
-  exit 1
-fi
-if ! command -v tofu >/dev/null; then
-  echo "missing tofu" >&2
-  exit 1
-fi
-
-# tofu console evaluates the whole config. Before any state exists a dummy
-# passphrase satisfies it, but once terraform.tfstate is on disk the real one
-# is required to decrypt it - fail with a useful message instead of a cryptic
-# state-encryption error.
-if [ -e terraform.tfstate ] && [ -z "${TF_VAR_state_passphrase:-}" ]; then
-  echo "terraform.tfstate exists; run 'source tofu.env' first so tofu console can decrypt it" >&2
-  exit 1
-fi
-export TF_LOG=
-export TF_VAR_state_passphrase="${TF_VAR_state_passphrase:-render-only-passphrase-unused}"
-export TF_VAR_ssh_public_keys="${TF_VAR_ssh_public_keys:-[\"ssh-ed25519 AAAArenderonly check@local\"]}"
-
-workdir="$(mktemp -d)"
-trap 'rm -rf "$workdir"' EXIT
-
-# Renders an HCL expression to stdout. jsonencode plus console's own string
-# quoting means the result is double-encoded, hence the two decode passes.
-# -lock=false: once a state file exists, console prints "Acquiring state
-# lock..." to STDOUT, which would corrupt the pipeline; a read-only render
-# needs no lock. The python side additionally keeps only the quoted result
-# line in case a future tofu adds other stdout notices.
-# On failure the diagnostics go to stderr (stdout is usually redirected into
-# the output file) and the caller decides whether to continue.
-render() {
-  printf '%s\n' "$1" | tofu console -lock=false 2>"$workdir/err" \
-    | python3 -c 'import json,sys; line=next(l for l in sys.stdin.read().splitlines() if l.lstrip().startswith("\"")); print(json.loads(json.loads(line)), end="")' \
-    || { echo "render failed:" >&2; cat "$workdir/err" >&2; return 1; }
-}
-
-# The runcmd list a VM actually gets, built from the same file
-# modules/vm-pve/main.tofu reads.
-runcmd_expr() {
-  printf '%s' 'jsondecode(templatefile("./cloud-init/base.runcmd.json.tftpl", { ci_user = "ubuntu", vm_name = "checkvm" }))'
-}
-
-# base_expr <vm_name> <ssh_key> <package> <runcmd_expr> <snapshot_expr> <upgrade>
-# The first three are literal strings, injected as HCL string literals; the
-# last three are raw HCL (snapshot_expr is `null` or a quoted timestamp).
-base_expr() {
-  cat <<EXPR
-jsonencode(templatefile("./cloud-init/base.yaml.tftpl", {
-  vm_name = "$1", fqdn = "$1.example", timezone = "Etc/UTC",
-  ci_user = "ubuntu", ssh_public_keys = ["$2"],
-  packages = ["$3"], runcmd = $4,
-  archive_snapshot = $5, package_upgrade = $6,
-}))
-EXPR
-}
-
-check_yaml() {
-  python3 - "$1" <<'PY'
-import sys, yaml
-
-class Strict(yaml.SafeLoader):
-    pass
-
-def no_dupes(loader, node, deep=False):
-    seen = set()
-    for k, _ in node.value:
-        key = loader.construct_object(k, deep=deep)
-        if key in seen:
-            raise ValueError(f"duplicate mapping key: {key!r}")
-        seen.add(key)
-    return yaml.SafeLoader.construct_mapping(loader, node, deep)
-
-Strict.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, no_dupes)
-with open(sys.argv[1]) as f:
-    yaml.load(f.read(), Loader=Strict)
-PY
-}
-
-check_schema() {
-  local out="$1" label="$2"
-  # cloud-init exits 0 on deprecation warnings, so grep for them explicitly.
-  local schema_out
-  schema_out="$(cloud-init schema --config-file "$out" 2>&1 | grep -Ev 'log_util|schema\.py' || true)"
-  if ! grep -q '^Valid schema' <<<"$schema_out"; then
-    echo "FAIL  ${label}: ${schema_out}"
-    return 1
-  elif grep -qi 'deprecat' <<<"$schema_out"; then
-    echo "FAIL  ${label}: deprecation: ${schema_out}"
-    return 1
-  fi
-  return 0
-}
-
-status=0
-label="base"
-out="$workdir/base.yaml"
-
-if ! render "$(base_expr checkvm "ssh-ed25519 AAAA check@local" qemu-guest-agent "$(runcmd_expr)" null false)" >"$out"; then
-  echo "FAIL  ${label}: render (base)"
-  status=1
-elif ! check_yaml "$out"; then
-  echo "FAIL  ${label}: YAML/duplicate-key check"
-  status=1
-elif check_schema "$out" "$label"; then
-  echo "ok    ${label}"
-else
-  status=1
-fi
-
-# Adversarial pass: values shaped to break unescaped YAML interpolation, plus
-# the snapshot pin and package_upgrade=true so the conditional bootcmd block
-# gets schema coverage.
-adv_key='ssh-ed25519 AAAA check@local: laptop key'
-adv_pkg='foo # bar'
-adv_name='no'
-adv_snapshot='20260801T000000Z'
-out="$workdir/adversarial.yaml"
-if ! render "$(base_expr "$adv_name" "$adv_key" "$adv_pkg" "$(runcmd_expr)" "\"$adv_snapshot\"" true)" >"$out"; then
-  echo "FAIL  <adversarial>: render"
-  status=1
-elif ! check_yaml "$out"; then
-  echo "FAIL  <adversarial>: YAML/duplicate-key check"
-  status=1
-elif ! python3 - "$out" "$adv_name" "$adv_key" "$adv_pkg" "$adv_snapshot" <<'PY'
-import sys, yaml
-path, name, key, pkg, snapshot = sys.argv[1:6]
-with open(path) as f:
-    doc = yaml.safe_load(f)
-assert doc["hostname"] == name, f"hostname mangled: {doc['hostname']!r}"
-keys = doc["users"][0]["ssh_authorized_keys"]
-assert key in keys, f"ssh key mangled: {keys!r}"
-assert pkg in doc["packages"], f"package mangled: {doc['packages']!r}"
-bootcmd = doc.get("bootcmd")
-assert isinstance(bootcmd, list) and bootcmd and all(isinstance(c, str) for c in bootcmd), \
-    f"bootcmd not a list of strings: {bootcmd!r}"
-assert any(snapshot in c for c in bootcmd), f"snapshot missing from bootcmd: {bootcmd!r}"
-assert any("Unattended-Upgrade" in c for c in bootcmd), \
-    f"unattended-upgrades disable missing from bootcmd: {bootcmd!r}"
-assert doc["package_upgrade"] is True, f"package_upgrade mangled: {doc['package_upgrade']!r}"
-PY
-then
-  echo "FAIL  <adversarial>: a hostile value did not survive as a string - check jsonencode() in base.yaml.tftpl"
-  status=1
-elif check_schema "$out" "<adversarial>"; then
-  echo "ok    <adversarial>"
-else
-  status=1
-fi
-
-exit "$status"

From 27055cf4762eb7e0a4acc740f83a0686a62c1f8e Mon Sep 17 00:00:00 2001
From: vpzed-dev-lux <320706889+vpzed-dev-lux@users.noreply.github.com>
Date: Fri, 28 Aug 2026 15:55:12 +0000
Subject: [PATCH 07/13] import the disk over the API, not over SSH

Reprovisioning found the premise incomplete: the snippet was not the only
thing the provider did over SSH. A disk whose source is `file_id` goes through
the provider's "custom disk" path, which shells out to `qm` on the node as
root - so the first apply after removing the ssh block failed with "unable to
authenticate user over SSH", not on cloud-init but on disk creation.

import_from is the API-native equivalent: it passes PVE's import-from
parameter, which does the same convert-and-import server-side. It only accepts
`images` or `import` content, never `iso`, so the golden image is now uploaded
as `import` content - and named .qcow2, because PVE restricts import uploads to
.ova/.qcow2/.raw/.vmdk and the image genuinely is qcow2. The extension is
load-bearing, not cosmetic.

vm-fingerprint.sh needed a fix to run at all: `cloud-init status` exits 2 when
the boot finished with recoverable errors, and the set -eu in its remote script
turned that into a silent, message-free abort. Degraded is now the steady state
here, because the user-data PVE generates uses the top-level `user:` key that
cloud-init deprecated in 22.2 - nothing in this repo emits it and nothing can
suppress it. The capture records the long form so the degradation is visible
and a future PVE fixing it shows up as a diff, minus last_update, which is a
timestamp.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MzjHsAQ3HkQjLdgv5PVi55
---
 modules/vm-pve/main.tofu  | 18 +++++++++++-------
 scripts/build-image.sh    | 21 ++++++++++++++-------
 scripts/vm-fingerprint.sh | 14 +++++++++++++-
 variables.tofu.example    |  2 +-
 4 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/modules/vm-pve/main.tofu b/modules/vm-pve/main.tofu
index cc0709e..da74aef 100644
--- a/modules/vm-pve/main.tofu
+++ b/modules/vm-pve/main.tofu
@@ -41,14 +41,18 @@ resource "proxmox_virtual_environment_vm" "this" {
   scsi_hardware = "virtio-scsi-single"
   boot_order    = ["scsi0"]
 
-  # Pointing file_id at the existing image makes ProxMox import and convert it
-  # into the target datastore; the source image is left untouched. file_id is
-  # ignore_changes'd below: it only matters at creation, and without that a
-  # bump of the fleet-wide cloud_image_file_id would diff - and destroy/create
-  # - every existing VM's disk at once.
+  # import_from, not file_id. Both make ProxMox import and convert the image
+  # into the target datastore and leave the source untouched, but file_id goes
+  # through the provider's "custom disk" path, which shells out to `qm` over
+  # SSH as root - the very dependency this design removes. import_from uses
+  # PVE's native import-from API parameter instead.
+  #
+  # It is create-only (the provider ignores later changes) and ignore_changes'd
+  # anyway: without that, bumping the fleet-wide cloud_image_file_id would diff
+  # - and destroy/create - every existing VM's disk at once.
   disk {
     datastore_id = var.disk_datastore
-    file_id      = var.cloud_image_file_id
+    import_from  = var.cloud_image_file_id
     interface    = "scsi0"
     size         = var.spec.disk_gb
     discard      = "on"
@@ -114,7 +118,7 @@ resource "proxmox_virtual_environment_vm" "this" {
   # Three layers of protection for the VMs that predate this project. These fail
   # the plan, unlike a check block, which would only warn.
   lifecycle {
-    ignore_changes = [disk[0].file_id]
+    ignore_changes = [disk[0].import_from]
 
     precondition {
       condition     = !contains(var.duplicate_vm_ids, var.spec.vm_id)
diff --git a/scripts/build-image.sh b/scripts/build-image.sh
index 8a57a69..22c1266 100755
--- a/scripts/build-image.sh
+++ b/scripts/build-image.sh
@@ -10,7 +10,11 @@
 # cannot be what installs it, so it is baked in here instead.
 #
 # Deliberately API-only: the finished image goes up through the storage upload
-# endpoint, not scp. Losing the SSH path is the entire point of the exercise.
+# endpoint, not scp. Losing the SSH path is the entire point of the exercise -
+# which is also why it is uploaded as `import` content rather than `iso`. A VM
+# disk sourced from an iso-content volume goes through the provider's "custom
+# disk" path, which shells out to `qm` over SSH as root; PVE's native
+# import-from only accepts `images` or `import` content.
 #
 #   ./scripts/build-image.sh                     # build only
 #   ./scripts/build-image.sh --upload            # build, then upload
@@ -51,7 +55,8 @@ usage="usage: build-image.sh [options]
                        (default: <serial>T000000Z)
   --outdir DIR         where to download and build (default: $outdir)
   --upload             upload the finished image to the node over the API
-  --datastore ID       upload target datastore (default: $datastore)
+  --datastore ID       upload target datastore, must allow the 'import'
+                       content type (default: $datastore)
   --endpoint URL       PVE API endpoint (default: variables.tofu's pve_endpoint)
   --node NAME          PVE node name (default: variables.tofu's pve_node)
   --force              rebuild an existing local image / overwrite on the node
@@ -83,9 +88,11 @@ done
 base_url="https://cloud-images.ubuntu.com/${release}/${serial}"
 upstream_name="${release}-server-cloudimg-amd64.img"
 cached="${outdir}/${release}-server-cloudimg-amd64-${serial}.img"
-golden_name="${release}-server-cloudimg-amd64-${serial}-golden-${snapshot}.img"
+# .qcow2, not .img: the image really is qcow2, and PVE only accepts
+# .ova/.qcow2/.raw/.vmdk for an `import` upload. The extension is load-bearing.
+golden_name="${release}-server-cloudimg-amd64-${serial}-golden-${snapshot}.qcow2"
 golden="${outdir}/${golden_name}"
-volid="${datastore}:iso/${golden_name}"
+volid="${datastore}:import/${golden_name}"
 
 # ---------------------------------------------------------------------------
 # Preflight. Every failure names its own remedy - this script is run rarely
@@ -250,7 +257,7 @@ if [ "$do_upload" -eq 1 ]; then
   # self-signed certificate. The token rides on every one of these calls.
   api() { curl -k -fsS -H "$auth" "$@"; }
 
-  if api "${endpoint}/api2/json/nodes/${node}/storage/${datastore}/content?content=iso" \
+  if api "${endpoint}/api2/json/nodes/${node}/storage/${datastore}/content?content=import" \
     | grep -q "\"volid\":\"${volid}\""; then
     if [ "$force" -eq 0 ]; then
       die "$volid already exists on the node.
@@ -270,7 +277,7 @@ if [ "$do_upload" -eq 1 ]; then
   # the upload itself; the task fails on mismatch.
   echo "      $(du -h "$golden" | awk '{print $1}') to ${node}/${datastore} - this takes a few minutes"
   upid="$(api --max-time 1800 \
-    -F content=iso \
+    -F content=import \
     -F checksum-algorithm=sha256 \
     -F "checksum=${golden_sha}" \
     -F "filename=@${golden}" \
@@ -309,5 +316,5 @@ EOF
 if [ "$do_upload" -eq 0 ]; then
   echo
   echo "Not uploaded. Re-run with --upload, or upload by hand in the web UI:"
-  echo "  node -> ${datastore} -> ISO Images -> Upload"
+  echo "  node -> ${datastore} -> Import -> Upload"
 fi
diff --git a/scripts/vm-fingerprint.sh b/scripts/vm-fingerprint.sh
index f140b98..d422301 100755
--- a/scripts/vm-fingerprint.sh
+++ b/scripts/vm-fingerprint.sh
@@ -227,7 +227,19 @@ section "base image"
 [ -e /etc/cloud/build.info ] && cat /etc/cloud/build.info
 
 section "cloud-init"
-cloud-init status
+# NOT a bare `cloud-init status`: it exits 2 when the boot finished with
+# recoverable errors ("degraded done"), and under the set -eu at the top of
+# this remote script that silently aborted the whole capture. Degraded is the
+# steady state on the proxmox path, because the user-data PVE generates still
+# uses the top-level `user:` key, deprecated in cloud-init 22.2. Nothing in
+# this repo emits that key and nothing here can suppress it.
+#
+# So record the long form, which names the degradation, rather than the bare
+# status line, which says "done" either way. If PVE ever stops emitting the
+# deprecated key, that shows up here as a diff instead of looking identical.
+# last_update is dropped: it is a timestamp, and this capture has to be
+# byte-stable across rebuilds.
+cloud-init status --long | grep -vE "^last_update:" || true
 # v1.platform is the datasource discriminator: nocloud on the proxmox path,
 # ec2 on the aws one. Verified against a real VM rather than guessed - the
 # first version of this line asked for v1.datasource, which is not a key.
diff --git a/variables.tofu.example b/variables.tofu.example
index 71e012e..7a1589e 100644
--- a/variables.tofu.example
+++ b/variables.tofu.example
@@ -18,7 +18,7 @@ variable "pve_endpoint" {
 variable "cloud_image_file_id" {
   description = "Datastore reference to the golden cloud image used as the disk source, built and uploaded by scripts/build-image.sh. EDIT: build an image and point this at it."
   type        = string
-  default     = "local:iso/resolute-server-cloudimg-amd64-20260720-golden-20260720T000000Z.img"
+  default     = "local:import/resolute-server-cloudimg-amd64-20260720-golden-20260720T000000Z.qcow2"
 }
 
 variable "disk_datastore" {

From dd884e7ba5916764f8d9c4b156ae65aaaf5929db Mon Sep 17 00:00:00 2001
From: vpzed-dev-lux <320706889+vpzed-dev-lux@users.noreply.github.com>
Date: Fri, 28 Aug 2026 16:33:22 +0000
Subject: [PATCH 08/13] take back two guarantees the cutover had quietly
 outsourced

Both found by the Step 7 fingerprint diff rather than by reading the code.

Key-only SSH. The old snippet set ssh_pwauth: false, which made cloud-init
write 50-cloud-init.conf. PVE's generated user-data has no equivalent, so the
guarantee had come to rest entirely on the cloud image happening to ship
60-cloudimg-settings.conf - still true, still off, but no longer ours and not
something this repo would notice losing. The base role now writes its own
drop-in, numbered 10- because sshd takes the FIRST value it obtains for a
keyword and reads the include glob in lexical order, which is the opposite of
apt.conf.d. KbdInteractiveAuthentication comes with it: disabling
PasswordAuthentication alone still leaves a PAM keyboard-interactive path to
password login. A separate task runs `sshd -t` over the whole assembled
config before the reload handler flushes, so a bad fragment fails the play
with the running sshd untouched.

Guest address selection. The agent reports one address list per interface, and
flattening them was correct only until the docker role brought up a docker0 at
172.17.0.1 - the inventory takes element 0 as ansible_host, and it stayed
right by luck rather than by rule. ipv4_addresses / mac_addresses /
network_interface_names share an index, so the module now selects the entry
matching network_device[0].mac_address. That beats both alternatives on offer:
docker's bridge address is configurable via daemon.json bip and podman,
libvirt and CNI each bring their own ranges, while interface names vary by
image (eth0, ens18, enp0s18, br-*, veth*). Neither can present the MAC ProxMox
assigned to net0.

Verified on VM 500: playbook applied both, second run changed=0, `sshd -T`
reports passwordauthentication and kbdinteractiveauthentication no, and the
vms output no longer carries 172.17.0.1.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MzjHsAQ3HkQjLdgv5PVi55
---
 CLAUDE.md                             | 26 +++++++++++++++
 README.md                             |  4 +--
 ansible/inventory/tofu.py             |  3 +-
 ansible/roles/base/handlers/main.yaml |  6 ++++
 ansible/roles/base/tasks/main.yaml    | 27 ++++++++++++++++
 modules/vm-pve/outputs.tofu           | 46 +++++++++++++++++----------
 6 files changed, 93 insertions(+), 19 deletions(-)
 create mode 100644 ansible/roles/base/handlers/main.yaml

diff --git a/CLAUDE.md b/CLAUDE.md
index a0ec7cf..4a9818a 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -112,6 +112,32 @@ Rebuild verification: capture, commit, destroy + reprovision, run
 - **An `@pve`-realm token cannot SSH anywhere.** It exists only in ProxMox's
   user database — it is not a Linux account. Worth knowing before anyone
   proposes reuniting the API and SSH identities to bring snippets back.
+- **A disk `file_id` needs SSH; `import_from` does not.** Sourcing a disk from
+  `file_id` sends the provider down its "custom disk" path, which shells out
+  to `qm` on the node as root. That is a second SSH dependency, quite separate
+  from snippets, and it fails at *disk creation* with "unable to authenticate
+  user over SSH". `import_from` passes PVE's native import-from parameter
+  instead. It only accepts `images` or `import` content — never `iso` — and
+  the upload endpoint restricts `import` filenames to
+  `.ova/.qcow2/.raw/.vmdk` (`UPLOAD_IMPORT_EXT_RE_1` in `PVE/Storage.pm`).
+  Hence `local:import/<name>.qcow2`: the extension is load-bearing.
+- **`sshd_config.d` takes the FIRST value, apt.conf.d takes the last.** The
+  include glob is read in lexical order, so a *lower* number wins. The base
+  role's drop-in is `10-smithy.conf` for that reason; the image's own
+  `60-cloudimg-settings.conf` would otherwise be authoritative.
+- **`cloud-init status` exits 2 on "degraded done".** Under a `set -e` that is
+  a silent, message-free abort — it cost a debugging session in
+  `vm-fingerprint.sh`. Degraded is the *steady state* here: the user-data PVE
+  generates uses the top-level `user:` key, deprecated in cloud-init 22.2.
+  Nothing in this repo emits it and nothing here can suppress it.
+- **Pick the guest address by MAC, not by filtering.** The agent reports one
+  address list per interface, with `ipv4_addresses`, `mac_addresses` and
+  `network_interface_names` sharing an index. Flattening them was fine until
+  the docker role added a `docker0` at 172.17.0.1, which the ansible inventory
+  could have taken as `ansible_host`. `modules/vm-pve/outputs.tofu` selects the
+  entry whose MAC is `network_device[0].mac_address`; address ranges are
+  configurable and interface names vary by image, but that MAC cannot be
+  impersonated by a bridge the guest brought up itself.
 - **`filename=@` must be the last `-F` in a PVE upload.** PVE parses the
   multipart body in order and streams everything after the file part into the
   file. A `checksum` field placed after it is never parsed *and* its bytes are
diff --git a/README.md b/README.md
index 13269cd..353b300 100644
--- a/README.md
+++ b/README.md
@@ -293,8 +293,8 @@ declarative grammar as the rest of the repo:
 - **The `base` role is implicit.** No spec lists it and none can opt out:
   `site.yaml` applies it to every host via `roles:`, ahead of the include
   loop. It owns the apt snapshot pin, the `APT::Periodic` zeros, the masked
-  apt-daily timers, the timezone, and the spec's `packages:` — the apt
-  baseline that used to be cloud-init's, now re-appliable to a running VM.
+  apt-daily timers, key-only SSH, the timezone, and the spec's `packages:` —
+  the baseline that used to be cloud-init's, now re-appliable to a running VM.
 - **Collections are pinned** in `ansible/requirements.yml`, and CI installs
   from it. `ansible-core` ships none, so a `community.*` task without an entry
   there passes locally and fails in CI.
diff --git a/ansible/inventory/tofu.py b/ansible/inventory/tofu.py
index 07fdae4..98f771e 100755
--- a/ansible/inventory/tofu.py
+++ b/ansible/inventory/tofu.py
@@ -9,7 +9,8 @@
   - one host per VM, named by inventory name (so --limit <name> works)
   - group "vms" holding every reachable VM
   - one group per declared role, for ad-hoc targeting (ansible bun -m ...)
-  - hostvars: ansible_host (first agent-reported IPv4), ansible_user,
+  - hostvars: ansible_host (the VM's own NIC, per the module output),
+    ansible_user,
     vm_id, vm_ansible_roles (what site.yaml applies), and the three the base
     role consumes: vm_packages, vm_archive_snapshot (may be None), vm_timezone
 
diff --git a/ansible/roles/base/handlers/main.yaml b/ansible/roles/base/handlers/main.yaml
new file mode 100644
index 0000000..4aa5a96
--- /dev/null
+++ b/ansible/roles/base/handlers/main.yaml
@@ -0,0 +1,6 @@
+---
+- name: Reload sshd
+  become: true
+  ansible.builtin.systemd_service:
+    name: ssh
+    state: reloaded
diff --git a/ansible/roles/base/tasks/main.yaml b/ansible/roles/base/tasks/main.yaml
index a7d0483..593b1ed 100644
--- a/ansible/roles/base/tasks/main.yaml
+++ b/ansible/roles/base/tasks/main.yaml
@@ -62,6 +62,33 @@
         - apt-daily.timer
         - apt-daily-upgrade.timer
 
+    # Key-only SSH used to be cloud-init's doing (ssh_pwauth: false, which
+    # wrote 50-cloud-init.conf). PVE's generated user-data has no equivalent,
+    # so without this the guarantee would rest entirely on the cloud image
+    # happening to ship 60-cloudimg-settings.conf - true today, and not
+    # something this repo would notice losing.
+    #
+    # 10-, because sshd takes the FIRST value it obtains for a keyword and
+    # reads the include glob in lexical order: a lower number wins, which is
+    # the opposite of apt.conf.d. KbdInteractiveAuthentication is included
+    # because disabling PasswordAuthentication alone still leaves a PAM
+    # keyboard-interactive path to password login.
+    - name: Enforce key-only SSH
+      ansible.builtin.copy:
+        dest: /etc/ssh/sshd_config.d/10-smithy.conf
+        content: |
+          PasswordAuthentication no
+          KbdInteractiveAuthentication no
+        mode: "0644"
+      notify: Reload sshd
+
+    # Validates the whole assembled config, not just the fragment above. It
+    # runs before the handler flushes, so a bad config fails the play with the
+    # running sshd still on its old configuration.
+    - name: Check the assembled sshd config is valid
+      ansible.builtin.command: /usr/sbin/sshd -t
+      changed_when: false
+
     - name: Set the timezone
       community.general.timezone:
         name: "{{ vm_timezone }}"
diff --git a/modules/vm-pve/outputs.tofu b/modules/vm-pve/outputs.tofu
index e233d4f..15e5129 100644
--- a/modules/vm-pve/outputs.tofu
+++ b/modules/vm-pve/outputs.tofu
@@ -10,12 +10,36 @@ output "name" {
   value       = proxmox_virtual_environment_vm.this.name
 }
 
+locals {
+  # The guest agent reports one address list per interface, and
+  # ipv4_addresses / mac_addresses / network_interface_names share that
+  # ordering. Select by the MAC of the NIC this module created, rather than
+  # flattening every interface and filtering what is obviously not wanted.
+  #
+  # Flattening was wrong as soon as a role installed docker: the guest grew a
+  # docker0 at 172.17.0.1, and the ansible inventory takes element 0 as
+  # ansible_host. It happens to still be right here because the agent lists
+  # eth0 first, which is luck, not a guarantee.
+  #
+  # Matching the MAC rather than filtering addresses or names is what makes
+  # this robust: docker's bridge address is configurable (daemon.json bip),
+  # podman/libvirt/CNI each bring their own ranges, and interface names vary
+  # by image (eth0, ens18, enp0s18, br-*, veth*). None of them can present the
+  # MAC ProxMox assigned to net0.
+  #
+  # A for-expression rather than index(): it yields an empty list when the
+  # agent has not reported yet, where index() would raise. The inventory
+  # already treats "no address" as skip-with-a-notice.
+  nic_ipv4 = flatten([
+    for i, mac in proxmox_virtual_environment_vm.this.mac_addresses :
+    proxmox_virtual_environment_vm.this.ipv4_addresses[i]
+    if lower(mac) == lower(proxmox_virtual_environment_vm.this.network_device[0].mac_address)
+  ])
+}
+
 output "ipv4_addresses" {
-  description = "Addresses reported by the guest agent, excluding loopback."
-  value = [
-    for addr in flatten(proxmox_virtual_environment_vm.this.ipv4_addresses) :
-    addr if addr != "127.0.0.1"
-  ]
+  description = "IPv4 addresses the guest agent reports for this VM's own NIC. Excludes loopback and any bridge a layer-2 role brings with it."
+  value       = local.nic_ipv4
 }
 
 output "ansible_roles" {
@@ -37,15 +61,5 @@ output "archive_snapshot" {
 
 output "ssh_command" {
   description = "Ready-to-run SSH command for the cloud-init user."
-  value = format(
-    "ssh %s@%s",
-    var.ci_user,
-    try(
-      [
-        for addr in flatten(proxmox_virtual_environment_vm.this.ipv4_addresses) :
-        addr if addr != "127.0.0.1"
-      ][0],
-      "<pending>",
-    ),
-  )
+  value       = format("ssh %s@%s", var.ci_user, try(local.nic_ipv4[0], "<pending>"))
 }

From 4753c506256918fb8718704a7b928ed3289aaf2d Mon Sep 17 00:00:00 2001
From: vpzed-dev-lux <320706889+vpzed-dev-lux@users.noreply.github.com>
Date: Fri, 28 Aug 2026 16:37:37 +0000
Subject: [PATCH 09/13] add AGENTS.md as a symlink to CLAUDE.md

Not everyone using this repo drives it with Claude Code, and AGENTS.md is the
name most other coding agents look for. A symlink rather than a copy so the
two can never drift.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MzjHsAQ3HkQjLdgv5PVi55
---
 AGENTS.md | 1 +
 README.md | 1 +
 2 files changed, 2 insertions(+)
 create mode 120000 AGENTS.md

diff --git a/AGENTS.md b/AGENTS.md
new file mode 120000
index 0000000..681311e
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1 @@
+CLAUDE.md
\ No newline at end of file
diff --git a/README.md b/README.md
index 353b300..f09c57e 100644
--- a/README.md
+++ b/README.md
@@ -57,6 +57,7 @@ overall testing workflow.
 ## Layout
 
 ```
+├── CLAUDE.md              # notes for coding agents; AGENTS.md is a symlink
 ├── versions.tofu  providers.tofu  encryption.tofu
 ├── main.tofu              # sops secrets, node data source
 ├── guards.tofu            # protected VMIDs + live foreign-VM lookup   [EDIT]

From 688e8c1cdedc8f63355828f91f82cf9b638a6cd3 Mon Sep 17 00:00:00 2001
From: vpzed-dev-lux <320706889+vpzed-dev-lux@users.noreply.github.com>
Date: Fri, 28 Aug 2026 18:29:10 +0000
Subject: [PATCH 10/13] correct the docs against the finished code

A review pass turned up claims that were true when written and stopped being
true during Step 7, plus two instructions that never worked here.

Stale after the import_from change: the golden image needs a datastore with
the `import` content type, not `iso`, and the disk names it as `import_from`,
not `file_id` - which also means `disk[0].import_from`, not
`disk[0].file_id`, is what `ignore_changes` covers.

Never worked: `git mv inventory/<name>.yaml inventory/destroy/`, in both the
usage list and the fingerprint walkthrough. This template gitignores
`inventory/*.yaml` and `inventory/destroy/*.yaml`, so git mv fails with "not
under version control". Plain mv, with a note that a fork tracking its own
specs can use the git form. Same class of error in the fingerprint section,
which reached for `git add -f` and `git diff` against a gitignored
`fingerprints/**` - now cp and diff.

Also: CLAUDE.md still described fingerprint captures as landing in a tracked
file, and omitted key-only SSH from the base role's inventory of what it owns.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MzjHsAQ3HkQjLdgv5PVi55
---
 CLAUDE.md | 62 +++++++++++++++++++++++++++++++------------------------
 README.md | 40 +++++++++++++++++++++--------------
 2 files changed, 59 insertions(+), 43 deletions(-)

diff --git a/CLAUDE.md b/CLAUDE.md
index 4a9818a..21735fa 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -13,9 +13,10 @@ Template placeholders — fill in for your site (marked `EDIT` in the code):
 
 - ProxMox node name/endpoint: `variables.tofu` (`pve_node`, `pve_endpoint`).
 - Datastores: `disk_datastore` holds VM disks and cloud-init drives; the
-  golden image lives on any datastore with the `iso` content type.
-- Golden image: built by `scripts/build-image.sh` and referenced directly as a
-  disk `file_id`, so ProxMox imports and converts it; there is no
+  golden image needs a datastore with the **`import`** content type (not
+  `iso` — see the `import_from` gotcha below).
+- Golden image: built by `scripts/build-image.sh` and named as the disk's
+  `import_from`, so ProxMox imports and converts it server-side; there is no
   `download_file` resource. It exists to carry `qemu-guest-agent` — see the
   gotcha below.
 - Secrets: `secrets.enc.json`, age via sops, key at
@@ -24,8 +25,9 @@ Template placeholders — fill in for your site (marked `EDIT` in the code):
 ## Hard rules
 
 **Pre-existing VMs must never be touched.** `guards.tofu` plus lifecycle
-preconditions in `modules/vm-pve/main.tofu` enforce a VMID floor, a named protected
-list, and a live check that rejects any VMID belonging to a VM not tagged
+preconditions in `modules/vm-pve/main.tofu` enforce a VMID floor, a named
+protected list, and a live check that rejects any VMID belonging to a VM not
+tagged
 `opentofu`. Validations stop the floor and the list from being weakened by
 tfvars or `TF_VAR_*` overrides, and a postcondition fails the plan if the
 protected VMs are not all visible in the API listing (fail closed).
@@ -44,20 +46,24 @@ the repo root.
 the only required variable. See `terraform.tfvars.example`.
 
 Adding a VM is adding a file to `inventory/`; deprovisioning is removing one —
-by convention, `git mv` it to `inventory/destroy/` (only `inventory/*.yaml` is
-scanned; subdirectories are invisible). The next apply destroys the VM and its
-disk; moving the file back provisions a *fresh* VM. To keep a
-VM and its data but power it off, set `started: false` instead. The filename
-is the VM name (DNS label). Only `vm_id` is required — everything else takes
-an `optional()` default from the `spec` object in `modules/vm-pve/variables.tofu`,
-which is the contract worth reading first.
+by convention, move it to `inventory/destroy/` (only `inventory/*.yaml` is
+scanned; subdirectories are invisible). Plain `mv`, not `git mv`: this
+template gitignores `inventory/*.yaml` and `inventory/destroy/*.yaml`, so
+`git mv` fails with "not under version control" — a fork that tracks its own
+specs can use `git mv`. The next apply destroys the VM and its disk; moving
+the file back provisions a *fresh* VM. To keep a VM and its data but power it
+off, set `started: false` instead. The filename is the VM name (DNS label).
+Only `vm_id` is required — everything else takes an `optional()` default from
+the `spec` object in `modules/vm-pve/variables.tofu`, which is the contract
+worth reading first.
 
 **Layer 0 — the golden image.** `./scripts/build-image.sh [--upload]` builds
 it: an upstream Ubuntu cloud image with `qemu-guest-agent` installed by
 `virt-customize`, uploaded over the API. Rebuild when the upstream serial
-should move; then repoint `cloud_image_file_id` at the new name. Never replace
-an uploaded image in place — `disk[0].file_id` is under `ignore_changes`, so a
-same-name replacement changes what a running VM was built from with no plan
+should move; then repoint `cloud_image_file_id` at the new name. It is
+uploaded as `import` content, not `iso`. Never replace an uploaded image in
+place — `disk[0].import_from` is create-only *and* under `ignore_changes`, so
+a same-name replacement changes what a running VM was built from with no plan
 diff. Prerequisites are `libguestfs-tools`, a readable
 `/boot/vmlinuz-$(uname -r)` (`dpkg-statoverride`), and membership of `kvm`;
 the script's preflight names each remedy.
@@ -85,20 +91,21 @@ without a pin there passes locally and fails CI). The `base` role is in no
 spec and cannot be opted out of: `site.yaml` applies it to every host via
 `roles:`, ahead of the include loop. It carries what the cloud-init snippet
 used to — the apt snapshot pin, the `APT::Periodic` zeros, the apt-daily
-timers, the timezone, and `spec.packages` — which means those **are** now
-reachable without recreating the VM. Removing a role from the list does
-**not** uninstall it —
-clean removal is a rebuild. Runs are idempotent (second run reports
-changed=0). After touching anything under `ansible/`, run
+timers, key-only SSH, the timezone, and `spec.packages` — which means those
+**are** now reachable without recreating the VM. Removing a role from the list
+does **not** uninstall it; clean removal is a rebuild. Runs are idempotent
+(second run reports changed=0). After touching anything under `ansible/`, run
 `./scripts/check-ansible.sh`.
 
 `./scripts/vm-fingerprint.sh <user>@<ip> fingerprints/<name>.txt` captures a
 VM's environment fingerprint (packages, apt config, enabled units, users, and
 the ansible-installed layer-2 files under `~/.local` and `~/.bun` — excluding
-per-instance noise like host keys and machine-id) into a tracked file.
-Rebuild verification: capture, commit, destroy + reprovision, run
-`ansible-playbook ansible/site.yaml`, capture to the same path — an empty
-`git diff` proves the environment is identical.
+per-instance noise like host keys and machine-id). `fingerprints/**` is
+gitignored, so a capture is local unless `git add -f`d deliberately. Rebuild
+verification: capture, destroy + reprovision, run `ansible-playbook
+ansible/site.yaml`, capture to the same path — an empty diff proves the
+environment is identical. Verified on this design: two rebuilds produced
+byte-identical captures.
 
 ## Gotchas that have already cost time
 
@@ -108,7 +115,8 @@ Rebuild verification: capture, commit, destroy + reprovision, run
   `$cloudinitoptions`, which needs `VM.Config.Cloudinit` **or**
   `VM.Config.Network`. It is set explicitly because the provider's schema
   default is Computed and PVE's own default is 1 — the opposite of
-  `spec.package_upgrade`'s default.
+  `spec.package_upgrade`'s default. Confirmed on the node, not just in
+  source: `qm config <id>` shows `ciupgrade: 0` after an apply by the token.
 - **An `@pve`-realm token cannot SSH anywhere.** It exists only in ProxMox's
   user database — it is not a Linux account. Worth knowing before anyone
   proposes reuniting the API and SSH identities to bring snippets back.
@@ -174,8 +182,8 @@ Rebuild verification: capture, commit, destroy + reprovision, run
   VM that sets it true, expect several minutes (timeout is 30m). The agent
   comes from the golden image, and nothing else can supply it — ansible needs
   the address the agent reports in order to connect at all. Point
-  `cloud_image_file_id` at a stock cloud image and apply hangs for the full
-  30m, then fails.
+  `cloud_image_file_id` at a stock cloud image (uploaded as `import`) and
+  apply hangs for the full 30m, then fails.
 - **pbkdf2 welds its salt to the key provider's block name** unless
   `encrypted_metadata_alias` is set — renaming a provider (or moving a
   passphrase between blocks) makes existing state undecryptable. Both
diff --git a/README.md b/README.md
index f09c57e..fe74712 100644
--- a/README.md
+++ b/README.md
@@ -98,8 +98,9 @@ same system that is running opentofu and ansible, but that is not required.
 - uv, https://github.com/astral-sh/uv/releases/tag/0.12.6
 
 - A ProxMox VE node (built against 9.x) with:
-  - a datastore that allows the `iso` content type for the golden image
-    (`local` by default),
+  - a datastore that allows the **`import`** content type for the golden
+    image (`local` by default; `iso` will not do — PVE only imports a disk
+    from `images` or `import` content),
   - a golden image uploaded (see "Building the golden image"), and
   - an API token for provisioning, with the usual
     `VM.*`/`Datastore.*`/`SDN.Use` privileges. That is the only credential
@@ -166,8 +167,9 @@ missing; without `kvm` it still works, just slowly under emulation. Then:
 
 which downloads the dated upstream serial, verifies it against that
 directory's `SHA256SUMS`, installs the agent with `virt-customize`, and uploads
-the result through the storage API — the same token as everything else, no scp.
-It prints the `volid` to put in `cloud_image_file_id`. Build without `--upload`
+the result through the storage API as `import` content — the same token as
+everything else, no scp. It prints the `volid` to put in
+`cloud_image_file_id`. Build without `--upload`
 first if you want to inspect the image (`virt-df`, `guestfish --ro`).
 
 Options: `--serial YYYYMMDD` picks the upstream build (default is the one this
@@ -230,11 +232,13 @@ Note: The `ssh_public_keys` in terraform.tfvars are the keys that reach the
   name and hostname — DNS label rules apply) and `tofu apply`. See
   `inventory-example.yaml` for every knob including snapshot pinning and
   ansible roles.
-- **Remove a VM:** `git mv inventory/<name>.yaml inventory/destroy/` and
-  `tofu apply` — only `inventory/*.yaml` is scanned, so the spec stays in the
-  repo while the VM and its disk are destroyed. Moving it back
-  provisions a fresh VM (all guest data is gone). To keep a VM but power it
-  off, set `started: false` instead.
+- **Remove a VM:** `mv inventory/<name>.yaml inventory/destroy/` and
+  `tofu apply` — only `inventory/*.yaml` is scanned, so the spec stays on disk
+  while the VM and its disk are destroyed. Moving it back provisions a fresh
+  VM (all guest data is gone). To keep a VM but power it off, set
+  `started: false` instead. (Plain `mv`: this template gitignores
+  `inventory/*.yaml`, so `git mv` fails. A fork that tracks its own specs can
+  use `git mv`.)
 - **Install software:** list roles in the spec
   (`ansible_roles: [docker, ...]`) and run
   `ansible-playbook ansible/site.yaml [--limit <vm>]` — no plan diff, no
@@ -247,7 +251,7 @@ Note: The `ssh_public_keys` in terraform.tfvars are the keys that reach the
   the VM's environment (packages, apt config, enabled units, users, and the
   ansible-installed files — minus per-instance noise like host keys and
   machine-id). Capture, destroy + reprovision, re-run ansible, capture again
-  to the same path: an empty `git diff` is the proof. `fingerprints/**` is
+  to the same path: an empty diff is the proof. `fingerprints/**` is
   gitignored, so captures stay local unless you commit one deliberately.
 
 ## Fingerprints
@@ -262,20 +266,24 @@ rebuilding the same configuration should produce no diff.
 ```sh
 source tofu.env
 ./scripts/vm-fingerprint.sh ubuntu@10.0.0.50 fingerprints/ubuntu-test.txt
-git add -f fingerprints/ubuntu-test.txt   # fingerprints/** is gitignored
-git commit -m "baseline"
+cp fingerprints/ubuntu-test.txt /tmp/baseline.txt
 
-git mv inventory/ubuntu-test.yaml inventory/destroy/
+mv inventory/ubuntu-test.yaml inventory/destroy/
 tofu apply
-git mv inventory/destroy/ubuntu-test.yaml inventory/
+mv inventory/destroy/ubuntu-test.yaml inventory/
 tofu apply
 ansible-playbook ansible/site.yaml --limit ubuntu-test
 
 ./scripts/vm-fingerprint.sh ubuntu@10.0.0.50 fingerprints/ubuntu-test.txt
-git diff -- fingerprints/ubuntu-test.txt   # empty = identical rebuild
+diff /tmp/baseline.txt fingerprints/ubuntu-test.txt   # empty = identical rebuild
 ```
 
-NOTE: The inventory/destroy directory name is intentional, so it is clear what the next "tofu apply" is expected to do.
+`fingerprints/**` and `inventory/*.yaml` are both gitignored, so this uses
+`cp`/`diff` and plain `mv` rather than `git add -f`/`git diff` and `git mv`.
+A fork that tracks its specs and captures can use the git forms throughout.
+
+NOTE: The inventory/destroy directory name is intentional, so it is clear what
+the next "tofu apply" is expected to do.
 
 ## Ansible
 

From cf5601ef18922412c9c4831486e8b83a7444b3d7 Mon Sep 17 00:00:00 2001
From: vpzed-dev-lux <320706889+vpzed-dev-lux@users.noreply.github.com>
Date: Fri, 28 Aug 2026 20:06:04 +0000
Subject: [PATCH 11/13] clarifications in README

---
 README.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index fe74712..346c644 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,9 @@ overall testing workflow.
 
 ```
 ├── CLAUDE.md              # notes for coding agents; AGENTS.md is a symlink
-├── versions.tofu  providers.tofu  encryption.tofu
+├── versions.tofu
+├── providers.tofu
+├── encryption.tofu
 ├── main.tofu              # sops secrets, node data source
 ├── guards.tofu            # protected VMIDs + live foreign-VM lookup   [EDIT]
 ├── vms.tofu               # inventory/ -> module.vm-pve, for_each
@@ -88,15 +90,15 @@ same system that is running opentofu and ansible, but that is not required.
 
 ## Prerequisites
 
-- OpenTofu >= 1.10, `sops`, `age`, `ansible` (ansible-core >= 2.15;
-  `ansible-lint` optional) on the workstation, plus `libguestfs-tools` if you
-  build the golden image there (see "Building the golden image").
+Download and install on PATH, for example ~/.local/bin
 
 - OpenTofu, https://github.com/opentofu/opentofu/releases/tag/v1.12.6
 - sops, https://github.com/getsops/sops/releases/tag/v3.13.3
 - age, https://github.com/FiloSottile/age/releases/tag/v1.2.1
 - uv, https://github.com/astral-sh/uv/releases/tag/0.12.6
 
+- `libguestfs-tools` (Debian/Ubuntu package name) if you build the golden image there (see "Building the golden image"). On Arch Linux the package name is `libguestfs`.
+
 - A ProxMox VE node (built against 9.x) with:
   - a datastore that allows the **`import`** content type for the golden
     image (`local` by default; `iso` will not do — PVE only imports a disk

From 8f610dd966ad13378e6efdda9e5338006faa96af Mon Sep 17 00:00:00 2001
From: vpzed-dev-lux <320706889+vpzed-dev-lux@users.noreply.github.com>
Date: Fri, 28 Aug 2026 21:05:53 +0000
Subject: [PATCH 12/13] give every ansible tool in CI the same collections path

ansible-lint failed on PR #22 with "couldn't resolve module/action
'community.general.timezone'" even though the galaxy install step ran. The
step was a no-op: the runner already has both collections somewhere its own
ansible-galaxy can see, so it reported "nothing to do" - and ansible-lint,
which runs from its own pipx venv, reads none of those paths.

ansible-lint's fallback of installing requirements.yml itself cannot help
either: ansible_compat looks for requirements.yml relative to the directory
holding .git, so ansible/requirements.yml is invisible to it no matter what
working-directory the step uses.

Export ANSIBLE_COLLECTIONS_PATH for the job and install into it with --force,
which ansible_compat preserves and prepends its cache to. Reproduced the
failure in a venv holding only ansible-core 2.21.3 and ansible-lint 26.8.0,
and confirmed this makes the same run pass.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MzjHsAQ3HkQjLdgv5PVi55
---
 .github/workflows/validate.yaml | 20 ++++++++++++++++----
 .gitignore                      |  4 ++++
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml
index e433620..4fd3244 100644
--- a/.github/workflows/validate.yaml
+++ b/.github/workflows/validate.yaml
@@ -13,6 +13,14 @@ jobs:
       # Read-only rendering only: satisfies the state-encryption config and
       # its >=16-char validation. No state is created or read in CI.
       TF_VAR_state_passphrase: ci-dummy-passphrase-render-only
+      # The one collections path every ansible tool here agrees on. Each has
+      # its own default view of where collections live - the runner ships some
+      # preinstalled, ansible-lint runs from its own pipx venv and sees neither
+      # those nor ansible-core's - and ansible-compat keeps (and prepends its
+      # cache to) whatever this names. Singular _PATH: ansible-compat hard
+      # errors on the plural spelling. Outside the workspace, so ansible-lint
+      # never walks it as project files.
+      ANSIBLE_COLLECTIONS_PATH: ${{ runner.temp }}/ansible-collections
     steps:
       - uses: actions/checkout@v4
 
@@ -24,10 +32,14 @@ jobs:
           # own venvs. ansible-lint pulls ansible-core into its venv itself.
           pipx install ansible-core
           pipx install ansible-lint
-          # ansible-core ships no community collections, so without this the
-          # roles' non-builtin FQCNs resolve locally (requirements.txt pins the
-          # batteries-included `ansible` package) and fail here.
-          ansible-galaxy collection install -r ansible/requirements.yml
+          # The roles' non-builtin FQCNs resolve locally because requirements.txt
+          # pins the batteries-included `ansible` package; here they have to be
+          # fetched. --force and an explicit -p because without them galaxy sees
+          # the runner's own preinstalled copies, reports "nothing to do", and
+          # installs into a path ansible-lint does not read - a silent no-op.
+          ansible-galaxy collection install -r ansible/requirements.yml \
+            -p "$ANSIBLE_COLLECTIONS_PATH" --force
+          ansible-galaxy collection list
 
       - name: copy examples
         run: |
diff --git a/.gitignore b/.gitignore
index 51a1a0e..eaf1594 100644
--- a/.gitignore
+++ b/.gitignore
@@ -47,3 +47,7 @@ private/**
 !private/.gitkeep
 # scratch
 tmp/
+# ansible-lint's cache and its symlinked copy of the project, written into
+# whichever directory holds .git - so the repo root, whatever `ansible-lint`
+# is run from.
+.ansible/

From 5ede096e79fcb898122cb6c2924b8b1e4ca36452 Mon Sep 17 00:00:00 2001
From: vpzed-dev-lux <320706889+vpzed-dev-lux@users.noreply.github.com>
Date: Fri, 28 Aug 2026 21:07:44 +0000
Subject: [PATCH 13/13] export the collections path from a step, not the job
 env

The job-level env: block cannot see the runner context, so
${{ runner.temp }} there made the whole workflow invalid: the run failed in
0s with "this run likely failed because of a workflow file issue", and no
pull_request run was created for the commit at all. Set the variable inside
the step via $RUNNER_TEMP and GITHUB_ENV, which every later step inherits.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MzjHsAQ3HkQjLdgv5PVi55
---
 .github/workflows/validate.yaml | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml
index 4fd3244..42dc37a 100644
--- a/.github/workflows/validate.yaml
+++ b/.github/workflows/validate.yaml
@@ -13,14 +13,6 @@ jobs:
       # Read-only rendering only: satisfies the state-encryption config and
       # its >=16-char validation. No state is created or read in CI.
       TF_VAR_state_passphrase: ci-dummy-passphrase-render-only
-      # The one collections path every ansible tool here agrees on. Each has
-      # its own default view of where collections live - the runner ships some
-      # preinstalled, ansible-lint runs from its own pipx venv and sees neither
-      # those nor ansible-core's - and ansible-compat keeps (and prepends its
-      # cache to) whatever this names. Singular _PATH: ansible-compat hard
-      # errors on the plural spelling. Outside the workspace, so ansible-lint
-      # never walks it as project files.
-      ANSIBLE_COLLECTIONS_PATH: ${{ runner.temp }}/ansible-collections
     steps:
       - uses: actions/checkout@v4
 
@@ -34,12 +26,22 @@ jobs:
           pipx install ansible-lint
           # The roles' non-builtin FQCNs resolve locally because requirements.txt
           # pins the batteries-included `ansible` package; here they have to be
-          # fetched. --force and an explicit -p because without them galaxy sees
-          # the runner's own preinstalled copies, reports "nothing to do", and
-          # installs into a path ansible-lint does not read - a silent no-op.
+          # fetched - into one path every later step agrees on. Each tool has
+          # its own default view of where collections live: the runner ships
+          # some preinstalled, and ansible-lint runs from its own pipx venv and
+          # reads neither those nor ansible-core's. Without the explicit -p and
+          # --force, galaxy finds the runner's copies, reports "nothing to do",
+          # and ansible-lint still cannot resolve community.general.
+          #
+          # Exported through GITHUB_ENV rather than the job's env: block
+          # because that block cannot see the runner context. Singular _PATH -
+          # ansible-compat hard errors on the plural spelling - and outside the
+          # workspace, so ansible-lint never walks it as project files.
+          collections="$RUNNER_TEMP/ansible-collections"
+          echo "ANSIBLE_COLLECTIONS_PATH=$collections" >>"$GITHUB_ENV"
           ansible-galaxy collection install -r ansible/requirements.yml \
-            -p "$ANSIBLE_COLLECTIONS_PATH" --force
-          ansible-galaxy collection list
+            -p "$collections" --force
+          ansible-galaxy collection list -p "$collections"
 
       - name: copy examples
         run: |