diff --git a/community/front-end/ofe/deploy.sh b/community/front-end/ofe/deploy.sh index fe7e6e2668..6841b96e61 100755 --- a/community/front-end/ofe/deploy.sh +++ b/community/front-end/ofe/deploy.sh @@ -494,6 +494,342 @@ create_service_account() { fi } +# +# Check the state bucket name is a valid Cloud Storage bucket name. +# - A safety net: tf/state_bucket/main.tf derives and shortens the name from +# the deployment name; this stops before anything is created or imported +# if that ever yields an invalid name. +# - '.' is not allowed, as such names need domain verification. +# - Prints the reason to stderr and returns 1 if the name is invalid. +# +BUCKET_NAME_MAX_LENGTH=63 + +validate_bucket_name() { + + local name=${1} + local validname='^[a-z0-9][a-z0-9_-]*[a-z0-9]$' + + if [[ ${#name} -lt 3 || ${#name} -gt ${BUCKET_NAME_MAX_LENGTH} || ! ${name} =~ ${validname} ]]; then + error " Error: State bucket name '${name}' is invalid. It must:" + error " - Be 3 to ${BUCKET_NAME_MAX_LENGTH} characters long" + error " - Only contain characters a-z (lowercase), 0-9, '-' and '_'" + error " - Start and end with a letter or digit" + return 1 + fi +} + +# +# Prepare this deployment's state bucket working directory +# - Each deployment name has its own tf/state_bucket/deployments// +# holding its local state, terraform.tfvars and admin.auto.tfvars, so +# deployments from one checkout never collide or need files moved by hand. +# - Symlinks the tracked .tf files rather than copying them, so it never +# runs a stale copy. +# - Moves files an earlier release left in tf/state_bucket/ (which held a +# single deployment) into the owning deployment's directory. +# - Sets ${state_bucket_dir}. +# +STATE_BUCKET_ROOT=tf/state_bucket + +prepare_state_bucket_dir() { + + if [[ -z ${deployment_name} || ${deployment_name} == . || + ${deployment_name} == .. || ${deployment_name} == */* ]]; then + echo "Error: invalid deployment name '${deployment_name}'." + return 1 + fi + + # -- The owner is the deployment named in the terraform.tfvars deploy.sh + # wrote. If it is unknown, or its directory already holds any of these + # files, stop rather than guess: a wrong move could make a later deploy + # create a second bucket and orphan the state in the first. + local -a legacy_files=(terraform.tfstate terraform.tfstate.backup terraform.tfvars admin.auto.tfvars) + local f found=0 + for f in "${legacy_files[@]}"; do + [[ -e ${STATE_BUCKET_ROOT}/${f} ]] && found=1 + done + if [[ ${found} -eq 1 ]]; then + local owner="" owner_dir + if [[ -f ${STATE_BUCKET_ROOT}/terraform.tfvars ]]; then + owner=$(sed -n 's|^deployment_name = "\([^"/]*\)"$|\1|p' \ + "${STATE_BUCKET_ROOT}/terraform.tfvars") + fi + # The bucket in the state must carry the same deployment label: an + # earlier release rewrote terraform.tfvars before applying, so after a + # failed deploy under another name it can name the wrong deployment. + if [[ -n ${owner} && -f ${STATE_BUCKET_ROOT}/terraform.tfstate ]]; then + local label + label=$(python3 -c ' +import json, sys +for r in json.load(open(sys.argv[1])).get("resources", []): + if r.get("type") == "google_storage_bucket" and r.get("name") == "tfstate": + for i in r.get("instances", []): + print(i["attributes"]["labels"]["ghpcfe_id"]) +' "${STATE_BUCKET_ROOT}/terraform.tfstate" 2>/dev/null) || label="?" + [[ -n ${label} && ${label} != "${owner}" ]] && owner="" + fi + owner_dir=${STATE_BUCKET_ROOT}/deployments/${owner} + for f in "${legacy_files[@]}"; do + [[ -n ${owner} && -e ${owner_dir}/${f} ]] && owner="" + done + if [[ -z ${owner} ]]; then + echo "Error: ${STATE_BUCKET_ROOT}/ holds state bucket files from an earlier" + echo " release, and deploy.sh cannot tell which deployment they belong to" + echo " or that deployment's directory already has its own. Move" + echo " terraform.tfstate*, terraform.tfvars and admin.auto.tfvars from" + echo " ${STATE_BUCKET_ROOT}/ into ${STATE_BUCKET_ROOT}/deployments//" + echo " for the deployment they belong to (or aside, if none), and rerun." + return 1 + fi + echo " Moving deployment ${owner}'s state bucket files to ${owner_dir}/" + mkdir -p "${owner_dir}" || return 1 + for f in "${legacy_files[@]}"; do + if [[ -e ${STATE_BUCKET_ROOT}/${f} ]]; then + mv "${STATE_BUCKET_ROOT}/${f}" "${owner_dir}/${f}" || return 1 + fi + done + fi + + state_bucket_dir=${STATE_BUCKET_ROOT}/deployments/${deployment_name} + mkdir -p "${state_bucket_dir}" || return 1 + # Relinked on every deploy, so a .tf file added or removed upstream is + # picked up. + find "${state_bucket_dir}" -maxdepth 1 -name '*.tf' -type l -delete || return 1 + local tf_file + for tf_file in "${STATE_BUCKET_ROOT}"/*.tf; do + ln -s "../../${tf_file##*/}" "${state_bucket_dir}/" || return 1 + done +} + +# +# Deploy the Terraform state bucket +# - A separate Terraform root from tf/, applied first. It holds the state of +# every cluster, filesystem, network, image and workbench the FrontEnd +# provisions; if that state were lost, Terraform could no longer destroy +# them and they would keep running and billing. So teardown.sh keeps it +# unless given --delete-state-bucket, and then only if it is empty. +# - Works in this deployment's own directory (prepare_state_bucket_dir). +# - Sets ${state_bucket_name} for the main deployment to consume. +# +deploy_state_bucket() { + + local sa="${deployment_name}-tkfe-sa@${project_id}.iam.gserviceaccount.com" + local -a tfvar_args=( + -var="project_id=${project_id}" + -var="region=${region}" + -var="deployment_name=${deployment_name}" + ) + + echo "" + echo "Creating the Terraform state bucket..." + + prepare_state_bucket_dir || return 1 + + # The writer list is split by owner so deploy.sh never rebuilds an + # authoritative IAM member list from a file an administrator edits: + # terraform.tfvars is deploy.sh's and is regenerated on every deploy + # (default_writers); admin.auto.tfvars is the administrator's + # (state_bucket_writers) and is created once and never read or rewritten. + local admin_file=${state_bucket_dir}/admin.auto.tfvars + if [[ ! -f ${admin_file} && -f ${state_bucket_dir}/terraform.tfvars ]] && + grep -q '^[[:space:]]*state_bucket_writers' "${state_bucket_dir}/terraform.tfvars"; then + echo "Error: state_bucket_writers is set in ${state_bucket_dir}/terraform.tfvars, which" + echo " deploy.sh regenerates. Move that line into ${admin_file}," + echo " which deploy.sh never overwrites, and rerun." + return 1 + fi + local -a seed_members=() + + # -- Adopt an existing bucket. Its name is a pure function of project and + # deployment name (tf/state_bucket/main.tf), so a deploy from a machine + # without this root's local state must import it, not create a second, + # empty one and orphan the state the first one holds. + # -upgrade: init rejects a lock file from an earlier provider pin unless + # it is re-resolved within versions.tf's range, as on a new machine. + terraform -chdir="${state_bucket_dir}" init -upgrade -input=false || return 1 + local bucket + bucket=$(echo 'local.bucket_name' | + terraform -chdir="${state_bucket_dir}" console "${tfvar_args[@]}") || return 1 + bucket=${bucket//\"/} + if [[ -z ${bucket} ]]; then + echo "Error: could not compute the Terraform state bucket name." + return 1 + fi + if ! validate_bucket_name "${bucket}"; then + return 1 + fi + + # -- No local state is normal on a new machine. State that exists but + # cannot be read must stop the deploy with Terraform's own error, not be + # mistaken for "no state" and lead to a confusing failed import. + local state_file=${state_bucket_dir}/terraform.tfstate + local resources="" + # 1 once the bucket is in this machine's state (already there, or imported + # below); the apply-failure diagnosis depends on it. + local managed=0 + if [[ -f ${state_file} ]] && + ! resources=$(terraform -chdir="${state_bucket_dir}" state list 2>&1); then + echo "Error: could not read ${state_file}:" + echo " ${resources//$'\n'/$'\n' }" + echo " It records only the state bucket itself, not any provisioned" + echo " resource. Fix it, or move it aside and rerun to adopt the bucket:" + echo " mv ${state_file} ${state_file}.unreadable" + return 1 + fi + + if grep -qx 'google_storage_bucket.tfstate' <<<"${resources}"; then + # Local state from a different deployment name or project, or from a + # bucket named before the deterministic naming, would make Terraform + # try to replace a prevent_destroy bucket. Stop with a remedy instead. + # An adoption whose apply failed leaves the bucket in state but no + # outputs, so the name is then read from the state. + local local_bucket + local_bucket=$(terraform -chdir="${state_bucket_dir}" output -raw bucket_name 2>/dev/null) || + local_bucket=$(terraform -chdir="${state_bucket_dir}" state show -no-color \ + google_storage_bucket.tfstate 2>/dev/null | + sed -n 's/^[[:space:]]*name[[:space:]]*=[[:space:]]*"\(.*\)"$/\1/p' | head -n 1) + if [[ ${local_bucket} != "${bucket}" ]]; then + echo "Error: ${state_bucket_dir}/terraform.tfstate on this machine belongs to" + echo " gs://${local_bucket:-}, but this deployment's state bucket" + echo " is gs://${bucket}. Move the old local files aside and rerun:" + echo " mv ${state_bucket_dir}/terraform.tfstate ${state_bucket_dir}/terraform.tfstate.old" + echo " mv ${state_bucket_dir}/terraform.tfvars ${state_bucket_dir}/terraform.tfvars.old" + echo " mv ${state_bucket_dir}/admin.auto.tfvars ${state_bucket_dir}/admin.auto.tfvars.old" + echo " The old bucket and the state it holds are not touched." + return 1 + fi + managed=1 + else + # -- Is the bucket already in this project? Decided from the exit + # status and the listed names, never from gcloud's error wording: + # a failed listing (auth, permission, network) stops the deploy, + # a successful listing without the bucket means it is absent. + # Bucket names are global; if another project owns the name, the + # create below fails with a 409, reported before the server is built. + local listed + if ! listed=$(gcloud storage buckets list --project="${project_id}" \ + --filter="name=${bucket}" --format='value(name)' 2>&1); then + echo "Error: could not determine whether gs://${bucket} already exists:" + echo " ${listed}" + return 1 + fi + if grep -qxF "${bucket}" <<<"${listed}"; then + echo " Found existing state bucket gs://${bucket}; adopting it." + terraform -chdir="${state_bucket_dir}" import -input=false "${tfvar_args[@]}" \ + google_storage_bucket.tfstate "${project_id}/${bucket}" || return 1 + managed=1 + + # The writer binding is authoritative, and this machine has no + # admin.auto.tfvars recording who was granted access before. Seed + # it from the current holders (used only if that file is absent). + # Buckets created before writers moved to objectUser hold objectAdmin. + local policy live_members member + policy=$(gcloud storage buckets get-iam-policy "gs://${bucket}" \ + --format=json) || return 1 + live_members=$(python3 -c ' +import json, sys +seen = set() +for b in json.load(sys.stdin).get("bindings", []): + if b.get("role") in ("roles/storage.objectUser", "roles/storage.objectAdmin"): + for m in b.get("members", []): + if not m.startswith("deleted:") and m not in seen: + seen.add(m) + print(m) +' <<<"${policy}") || return 1 + while IFS= read -r member; do + [[ -z ${member} || ${member} == "serviceAccount:${sa}" ]] && continue + seed_members+=("${member}") + done <<<"${live_members}" + fi + fi + + if [[ ! -f ${admin_file} ]]; then + { + echo "# Administrator-maintained settings for the state bucket. deploy.sh creates" + echo "# this file once and never reads or overwrites it. Add the service account of" + echo "# any provisioning credential registered from another project here, then" + echo "# run: terraform -chdir=${state_bucket_dir} apply" + echo "state_bucket_writers = [" + local m + for m in "${seed_members[@]}"; do echo " \"${m}\","; done + echo "]" + } >"${admin_file}" || return 1 + fi + + # The default provisioning service account is optional (an administrator + # may register a different or pre-existing credential instead; see + # AdvancedAdmin.md), and GCS rejects an IAM binding that names a + # nonexistent service account, so grant it only if it exists. + local default_writers="[]" + if bash "${SCRIPT_DIR}/script/service_account.sh" check \ + "${project_id}" "${deployment_name}-tkfe-sa" &>/dev/null; then + default_writers="[\"serviceAccount:${sa}\"]" + else + echo " Note: ${sa} does not exist yet, so the state bucket grants it no" + echo " writer access. Once it exists, rerun deploy.sh, or add it to" + echo " state_bucket_writers in ${admin_file} and run:" + echo " terraform -chdir=${state_bucket_dir} apply" + fi + + cat >"${state_bucket_dir}/terraform.tfvars" </dev/null) && [[ -n ${owner} ]]; then + this_project=$(gcloud projects describe "${project_id}" \ + --format='value(projectNumber)' 2>/dev/null) || this_project="" + if [[ ${owner} == "${this_project}" ]]; then + echo " gs://${bucket} already exists in ${project_id}; the lookup" + echo " above did not list it. Rerun deploy.sh to adopt it." + else + echo " gs://${bucket} already exists in another project (number" + echo " ${owner}). Bucket names are global, so this deployment name" + echo " cannot be used in ${project_id}; choose another." + fi + else + echo " If the error above is a 409, gs://${bucket} exists in a project" + echo " this account cannot see. Choose another deployment name." + fi + echo " The FrontEnd will not be deployed without it." + return 1 + fi + + state_bucket_name=$(terraform -chdir="${state_bucket_dir}" output -raw bucket_name) + if [[ -z ${state_bucket_name} ]]; then + echo "Error: the state bucket was applied but reported no name." + return 1 + fi + + echo " Terraform state bucket: ${state_bucket_name}" + echo " Note: this bucket is versioned and is kept by teardown.sh unless" + echo " --delete-state-bucket is given." + echo " Provisioning credentials for other projects must be added to" + echo " state_bucket_writers in ${admin_file}." + echo "" +} + # # Deploy the FrontEnd # - All deployments parameters have been obtained, so can now construct the @@ -501,6 +837,27 @@ create_service_account() { # deploy() { + # -- Confirm before anything is created: the state bucket below is + # protected from deletion, so declining after it exists would leave it + # behind. + # + if [ -z "$config_file" ]; then + case $(ask " Proceed to deploy? [y/N] ") in + [Yy]*) ;; + *) + echo "Exiting." + exit 0 + ;; + esac + fi + + # -- The state bucket must exist before the server is created, because the + # server's configuration carries its name. + # + if ! deploy_state_bucket; then + exit 1 + fi + # -- Collect deployment files # # For a tarball deployment, it is important that the 'root' directory is @@ -550,6 +907,8 @@ django_su_email = "${django_superuser_email}" deployment_mode = "${deployment_mode}" subnet = "${subnet_name}" +terraform_state_bucket = "${state_bucket_name}" + extra_labels = { creator = "${USER}" } @@ -578,17 +937,6 @@ TFVARS # TODO - upload terraform files to the FE server, so that they can be recovered if ever # needed - if [ -z "$config_file" ]; then - # Ask for user confirmation before deploying - case $(ask " Proceed to deploy? [y/N] ") in - [Yy]*) ;; - *) - echo "Exiting." - exit 0 - ;; - esac - fi - # -- Start the deployment using Terraform. # Note: Extract $? for terraform using PIPESTATUS, as $? below is # from tee @@ -958,6 +1306,7 @@ deploy_from_config() { deployment_name=${yaml_array[deployment_name]} project_id=${yaml_array[project_id]} zone=${yaml_array[zone]} + region=${zone%-*} subnet_name=${yaml_array[subnet_name]} dns_hostname=${yaml_array[dns_hostname]} ip_address=${yaml_array[ip_address]} diff --git a/community/front-end/ofe/docs/AdvancedAdmin.md b/community/front-end/ofe/docs/AdvancedAdmin.md index b513265dcc..dabb890e28 100644 --- a/community/front-end/ofe/docs/AdvancedAdmin.md +++ b/community/front-end/ofe/docs/AdvancedAdmin.md @@ -28,6 +28,23 @@ matter of registering different service account credentials, for the different projects, to the TKFE portal. The process to create additional service accounts is outlined in the following section. +A multi-project setup needs one extra step. Terraform state for every resource +TKFE provisions is held in a Cloud Storage bucket in the deployment project +(see [Terraform State](admin_guide.md#terraform-state)), and a service account +can write the state of the resources it provisions only if it has access to +that bucket. After creating an additional service account, add it to +`state_bucket_writers` in +`tf/state_bucket/deployments//admin.auto.tfvars` on the client +machine and re-apply that directory: + +```shell +terraform -chdir=tf/state_bucket/deployments/ apply +``` + +`deploy.sh` never overwrites `admin.auto.tfvars`, so the entry survives +redeploys. Without this step, provisioning with that credential fails when +Terraform tries to write state. + --- ## Service Account Management diff --git a/community/front-end/ofe/docs/admin_guide.md b/community/front-end/ofe/docs/admin_guide.md index 4cea8d88c0..d2252be670 100644 --- a/community/front-end/ofe/docs/admin_guide.md +++ b/community/front-end/ofe/docs/admin_guide.md @@ -184,6 +184,16 @@ TKFE can also be deployed automatically using a YAML configuration file with all If the DJANGO_SUPERUSER_PASSWORD environment variable is set, the script will use it to set the Django superuser password instead of the value in the YAML file. This is a more secure way of handling passwords. +A configuration-file deployment does not create the provisioning service +account (`-tkfe-sa`), so no account is granted access to the +Terraform state bucket. A credential whose service account has project-level +Cloud Storage access, such as one created with `script/service_account.sh`, +can still use the bucket. Add any other provisioning credential, including one +from another project, to `state_bucket_writers` in +`tf/state_bucket/deployments//admin.auto.tfvars`, then run +`terraform -chdir=tf/state_bucket/deployments/ apply`. See +[Terraform State](#terraform-state). + To use this configuration file for automated deployment, follow these steps: 1. Create a file called `config.yaml` (do not place inline `#` comments on value lines — `deploy.sh` parses values directly): @@ -472,6 +482,343 @@ Please see the [application installation guide](Applications.md). Please see the [Workbench Admin Guide](WorkbenchAdmin.md). +## Terraform State + +Every resource TKFE provisions (clusters, filesystems, networks, image builder +environments and workbenches) is created with Terraform, which records what it +created in a *state* file. Terraform needs that state to modify or destroy the +resource later. **A resource whose state has been +lost cannot be destroyed by Terraform**: it keeps running, keeps accruing cost, +and has to be removed by hand in the Cloud Console. + +TKFE therefore stores this state in a dedicated Cloud Storage bucket, +`-tfstate-`, rather than on the service machine's +local disk. `deploy.sh` creates the bucket from the Terraform files in +`tf/state_bucket/`, before the FrontEnd itself, and prints its name. The +suffix is derived from the project and deployment name, so running +`deploy.sh` again from any machine finds and reuses the existing bucket rather +than creating a second one. + +To make a valid bucket name, the deployment name is lowercased, any character +other than a-z, 0-9, '-' and '_' becomes '-', it is shortened to 46 characters +(bucket names are limited to 63), and leading and trailing '-' and '_' are +removed. A typical deployment name is used unchanged, and the suffix still +distinguishes deployments whose names clean or shorten to the same text. + +On the client machine, each deployment has its own working directory for +these files, `tf/state_bucket/deployments//`, holding that +deployment's local state and variables, so deployments with different names +can be deployed from the same directory one after another. Files that an +earlier release kept directly in `tf/state_bucket/` are moved into the +directory of the deployment they belong to on the next deploy. If `deploy.sh` +cannot tell which deployment that is, it stops and says how to move them. + +The bucket has three properties that matter to an administrator: + +- **It is versioned.** Every previous copy of a state file is retained (by + default for 90 days), so a state file corrupted by a crash part-way through + an operation can be rolled back. +- **It survives teardown.** It is a separate Terraform deployment from the + FrontEnd, so destroying the FrontEnd cannot destroy the state of the + resources it created, and it is protected against accidental deletion. +- **Access is limited.** The bucket itself grants access only to the + provisioning service account `deploy.sh` creates and to the accounts listed + in `state_bucket_writers` in the deployment's `admin.auto.tfvars`. Accounts + with a project-level Cloud Storage role can also reach it, including the + `storage.admin` role `script/service_account.sh` gives the provisioning + account. State contains full resource detail and can contain generated + passwords, so keep project-level storage roles to accounts you trust. + +State is filed under a namespace belonging to the TKFE database, then one +prefix per resource matching the run-time directory names on the service +machine: `gs:///20261015-3f2a9c1e/clusters/cluster_3/` holds the +state for the cluster with ID 3. The namespace is created with the database, +so a new database (for example after rebuilding the service machine) gets a +new namespace, and its resources can never be confused with those of an +earlier database. On the service machine, `python manage.py tfstate_namespace` +prints the current namespace. + +The state bucket is separate from the support bucket +(`-storage-`), which holds startup scripts, Ansible +configuration and job output. + +### Registering a credential for another project + +If you register a service account credential from a **different project** to +the one TKFE was deployed into (see +[Advanced Administration](AdvancedAdmin.md#multi-project-configuration)), that +service account also needs access to the state bucket, or provisioning with it +fails when Terraform tries to write state. + +Add it to the `state_bucket_writers` list in +`tf/state_bucket/deployments//admin.auto.tfvars` on the client +machine and re-apply that directory: + +```shell +terraform -chdir=tf/state_bucket/deployments/ apply +``` + +`deploy.sh` creates `admin.auto.tfvars` on the first deploy and never reads or +overwrites it afterwards, so entries there survive redeploys, and any valid +list layout is accepted. Do not add writers to `terraform.tfvars` in the same +directory: `deploy.sh` regenerates that file on every deploy. + +When a credential is registered or updated in the portal, TKFE uses it to +check whether its service account can read and write state in the bucket. If +it cannot, the portal shows a warning naming the account, with the line to add +to `admin.auto.tfvars` and the command above. Registration still succeeds; +TKFE never grants the access itself. + +### Recovering resources after losing the service machine + +This is the situation the state bucket exists for. If the service machine is +lost or rebuilt, its local disk goes with it, including the TKFE database +(`db.sqlite3`), which is what the portal knows about your resources, and the +per-resource run-time directories. The state bucket is unaffected, so every +resource TKFE created can still be destroyed with Terraform. + +Only the **state** is recovered, from the bucket. The **portal's records** are +not: a rebuilt TKFE will not list the old clusters, because they were in the +lost database. Recovery is therefore done from the command line, not through +the web interface. + +Old resources are under the old database's namespace. List the namespaces with +`gcloud storage ls gs:///`; the date at the start of each shows when +that database was created. + +To destroy a resource whose service machine is gone, create an empty directory +on any machine with a `gcloud` login that has access to the state bucket, and +write a single `main.tf` containing only a backend and a provider: + +```hcl +terraform { + backend "gcs" { + bucket = "-tfstate-" + prefix = "/clusters/cluster_3" + } +} + +provider "google" { + project = "" + region = "" +} +``` + +Use the prefix of the resource you want to remove; +`gcloud storage ls gs:////` lists them, and they match the +run-time directory names (`clusters/cluster_`, `fs/fs_`, +`vpcs/vpc_`, `images/image_`, `workbenches/workbench_`). Then: + +```shell +terraform init +terraform plan +terraform destroy +``` + +The resources are in the recovered state but not in this configuration, so +Terraform plans to destroy all of them. Check that the plan lists what you +expect before approving it. + +The state alone is enough, so this works even for clusters, filesystems and +image builder environments, whose Terraform configuration was generated by the +Cluster Toolkit and cannot be regenerated once the database is gone. For +networks and workbenches you can instead copy the template from +`infrastructure_files/` into a new directory and add the same backend block; +`terraform init` then adopts the existing state, allowing a normal plan and +destroy. + +**Clusters: remove compute nodes before destroying the network.** Slurm creates +compute nodes directly through the Compute Engine API when jobs are scheduled, +so they are not in Terraform state and `terraform destroy` leaves them running. +After destroying a cluster, find them by the cluster's Slurm name, which is the +part before `-controller` in the controller instance's name in the plan above: + +```shell +gcloud compute instances list --project= \ + --filter="name~^-" --format="value(name,zone)" +gcloud compute instances delete --zone= --project= +``` + +Do this before destroying the cluster's network: a network still used by +running instances cannot be deleted. In general, destroy clusters, +filesystems, workbenches and image builders before the networks they use. + +### A resource reports a stale state lock + +While Terraform works on a resource it holds a lock on that resource's state +(a `default.tflock` object beside `default.tfstate` in the state bucket). A +second operation on the same resource waits for the lock, for up to 45 +minutes, instead of interfering with the first. Terraform releases the lock +when it finishes, including when it fails. A lock is left behind, or *stale*, +only if Terraform is killed outright while holding it: for example by a +service machine reboot, the out-of-memory killer, or `kill -9`. + +Before each Terraform operation, TKFE checks the resource's lock. If the lock +was taken on the service machine and no Terraform process there is running +for that resource, the operation fails at once instead of waiting. The +resource goes into an error state, and the log (see +[Finding Log Files](#finding-log-files)) contains a message starting +`Terraform state for is locked.` It gives the lock ID, the +operation and host that took the lock, its age, and the command to clear it. +An operation that gives up after waiting 45 minutes for a lock reports the +same message. + +Before clearing the lock, confirm on the service machine that no Terraform +process is working in that resource's directory (with `sudo`, so processes run +by any user are shown): + +```shell +for pid in $(pgrep -x terraform); do echo "$pid $(sudo readlink /proc/$pid/cwd)"; done +``` + +If any line shows the directory from the message, an operation is still +running: wait for it to finish. If the lock's holder is a different host, +check that machine instead. + +When nothing is running, clear the lock as the `gcluster` user with the +command from the message: + +```shell +sudo -u gcluster bash -c 'cd && GOOGLE_APPLICATION_CREDENTIALS= terraform force-unlock -force ' +``` + +The lock ID is the number in the message (the lock object's generation), not +the `ID` field inside the lock object. Then repeat the operation in the portal. + +**Never force-unlock while an operation is running.** The lock is the only +thing that stops two Terraform runs writing the same state at once, and +removing it during an operation can corrupt the state. If state is damaged, +earlier versions are kept in the versioned bucket. + +### Upgrading an existing TKFE to Cloud Storage state + +A TKFE deployed before state was held in Cloud Storage keeps each resource's +state on the service machine's disk, and TKFE has no in-place upgrade command. +Until the steps below are done, creating or reconfiguring a resource fails: +the new code refuses to provision without a state bucket. + +These steps assume the new release's code is already in place under +`/opt/gcluster/cluster-toolkit` on the service machine. The support bucket +must also hold the new release's cluster and workbench files, or new clusters +still start with the old release's setup scripts. To copy them, run this from +the new release's `community/front-end/ofe` directory on the client machine: + +```shell +gcloud storage cp --recursive infrastructure_files/gcs_bucket/* gs://-storage-/ +``` + +1. On the client machine, create the state bucket from the deployment's own + directory, linking the Terraform files as `deploy.sh` does: + + ```shell + D=tf/state_bucket/deployments/ + mkdir -p "$D" + for f in tf/state_bucket/*.tf; do ln -s "../../${f##*/}" "$D/"; done + ``` + + Write `$D/terraform.tfvars` with `project_id`, `region` and + `deployment_name` matching the deployment, and set `default_writers` to the + provisioning service account if it exists: + + ```hcl + default_writers = ["serviceAccount:-tkfe-sa@.iam.gserviceaccount.com"] + ``` + + Put any other provisioning accounts, including those for other projects, + in `state_bucket_writers` in `$D/admin.auto.tfvars`. Then: + + ```shell + terraform -chdir="$D" init + terraform -chdir="$D" apply + terraform -chdir="$D" output -raw bucket_name + ``` + +2. On the service machine, add the bucket name under `config.server` in + `/opt/gcluster/cluster-toolkit/community/front-end/ofe/configuration.yaml`: + + ```yaml + terraform_state_bucket: "" + ``` + + On the client machine, add the same value to the FrontEnd's own + `tf/terraform.tfvars`, which an older `deploy.sh` wrote without it: + + ```hcl + terraform_state_bucket = "" + ``` + + Without it, `teardown.sh` cannot destroy the FrontEnd ("No value for + required variable"). + +3. As the `gcluster` user, create the new database table and restart TKFE: + + ```shell + cd /opt/gcluster/cluster-toolkit/community/front-end/ofe/website + /opt/gcluster/django-env/bin/python3 manage.py makemigrations ghpcfe + /opt/gcluster/django-env/bin/python3 manage.py migrate + sudo supervisorctl restart 'gcluster-uvicorn-background:*' + ``` + +4. Move existing state into the bucket, first as a preview: + + ```shell + cd /opt/gcluster/cluster-toolkit/community/front-end/ofe + ./script/migrate_tf_state_to_gcs.sh --dry-run + ./script/migrate_tf_state_to_gcs.sh --yes + ``` + + The script reports each directory as migrated, skipped or failed and exits + non-zero if any failed. It keeps the original of each migrated state file + as `terraform.tfstate.premigration` in the resource's run-time directory, + beside its `cloud_credentials` (for example `clusters/cluster_/` or + `vpcs/vpc_/`), not in the Terraform working directory, which the + Cluster Toolkit replaces whenever a cluster, filesystem or image builder is + reconfigured. + +5. Reconfigure or start resources only after the script reports no failures. + Reconfiguring a cluster whose state has not been migrated fails at + `terraform init` rather than moving its state, because the backend has + changed. + +The script only finds working directories that hold a local state file. A +network or workbench that was defined but never created, or a filesystem that +was defined but never started, has none, so it is not migrated. A +filesystem's blueprint would also still lack the state backend, and starting +it would keep its state on the service machine's disk. Delete such a resource +in the portal and create it again. + +### Reverting to an older TKFE release + +A TKFE release from before state was held in Cloud Storage expects state on +local disk. There is no automated way back. To revert, move each resource's +state back to its run-time directory **before** installing the older release; +an older release that finds a GCS backend block in a working directory will +not find the resource's state. + +For each resource, in its Terraform working directory on the service machine +(for example `vpcs/vpc_`, `workbenches/workbench_/terraform/google`, or +`clusters/cluster_//primary`): + +1. Remove the `backend "gcs"` block. If the directory has a `backend.tf`, + delete that file: networks and workbenches have one, and so does any + resource that `script/migrate_tf_state_to_gcs.sh` migrated from an older + release. Otherwise, for clusters, filesystems and image builders created by + this release, delete the `backend "gcs"` block from the generated `main.tf`. +2. Copy the state back to local disk, using the resource's own credential: + + ```shell + GOOGLE_APPLICATION_CREDENTIALS=/cloud_credentials \ + terraform init -migrate-state -force-copy + ``` + +3. Confirm `terraform.tfstate` lists the resources and `terraform plan` + reports no changes. + +If you later move these resources to Cloud Storage again, first move each +resource's existing `terraform.tfstate.premigration` out of its run-time +directory, or rename it. The migration script refuses to overwrite a backup +that differs from the current state. + ## Teardown Process The TKFE package contains a `teardown.sh` script that will destroy the running @@ -485,6 +832,30 @@ To tear down the web interface and its hosting infrastructure, run directory `./teardown.sh` on the original client machine in the same directory that was used to deploy TKFE. +The Terraform state bucket is deliberately **not** removed by the teardown +script, so that any resource still running can still be destroyed with +Terraform. The script prints the bucket name when it finishes. Once you are +certain every resource TKFE created is gone, remove it with: + +```shell +./teardown.sh --delete-state-bucket +``` + +This tears down the FrontEnd too if it is still deployed. Before destroying +anything, the script reads every state file in the bucket, under every +namespace, and stops if any still records resources, cannot be read or is +locked, listing them. It then asks you to type the bucket name to confirm, +unless `-y` is also given. The bucket's deletion protection is lifted only in a +temporary copy of `tf/state_bucket/`; the files in the repository are not +changed. If Terraform would change the bucket in any other way, for example +replace it after a change of region, the script stops and deletes nothing. + +To remove the bucket of an earlier deployment made from the same directory, +name it: `./teardown.sh --delete-state-bucket `. This touches +only that bucket. If that deployment's FrontEnd is, or may still be, deployed +from this directory (its lock file exists or its state cannot be read), the +script refuses; run it without a name to tear the FrontEnd down first. + Note: If the -y flag is passed to the script, all user questions will be answered with "yes". This is intended for automated deployments where user input is not possible. ## Troubleshooting diff --git a/community/front-end/ofe/docs/developer_guide.md b/community/front-end/ofe/docs/developer_guide.md index 1b6a4202a4..861594825f 100644 --- a/community/front-end/ofe/docs/developer_guide.md +++ b/community/front-end/ofe/docs/developer_guide.md @@ -98,6 +98,17 @@ Here are some notes from a developer's perspective: - Terraform creates a supporting GCS bucket. This bucket is not only used during deployment, but also provides a long-term storage for clusters operating within this deployment. +- Before any of the above, `deploy.sh` applies a second, separate Terraform + root, `tf/state_bucket/`, which creates a versioned bucket holding the + Terraform state of every resource the FrontEnd goes on to provision. It is a + separate root so that tearing the FrontEnd down cannot destroy it + (`teardown.sh` removes it only with `--delete-state-bucket`): state for a + running cluster must outlive the FrontEnd that created it. `deploy.sh` runs + it in `tf/state_bucket/deployments//`, which links to the + root's `.tf` files and holds that deployment's own local state and + variables. The bucket name is passed to the service machine in the + `terraform-state-bucket` instance metadata key and ends up as + `server.terraform_state_bucket` in `configuration.yaml`. - Terraform sets up a Pub/Sub topic for communication between the service machine and clusters. - Terraform provisions a compute engine virtual machine to be the service @@ -169,6 +180,22 @@ The directory contains the following: Note that life cycles of VPCs and Filestores are managed independently to those of clusters. +These run-time directories hold the generated Terraform configuration and the +log files, but **not** the Terraform state: each directory is configured with a +GCS backend, so state lives in the state bucket under a prefix made of the +database's namespace and the directory name +(`\/clusters/cluster_\`, `\/vpcs/vpc_\`, and so +on for `fs`, `images` and `workbenches`), built by +`cluster_manager.utils.terraform_state_prefix`. The namespace is created with +the TKFE database (model `StateNamespace`; `python manage.py tfstate_namespace` +prints it), so a rebuilt server never writes over an earlier database's state. +Blueprint-generated resources get the backend from `terraform_backend_defaults` +in the generated blueprint; the raw-template resources (VPCs and workbenches) +get a `backend.tf`, written into the directory by +`cluster_manager.utils.write_backend_tf` when the template is copied. Code that +needs to inspect state uses `cluster_manager.utils.get_terraform_state`, which +runs `terraform state pull`, rather than opening a local `terraform.tfstate`. + #### For applications Application data is stored in the shared filesystem `/opt/cluster`. It contains the following sub-directories: @@ -251,6 +278,7 @@ define the major components: | `.../docs/` | documentation | | `.../infrastructure_files/` | Support files for deploying cloud infrastructure | | `.../tf/` | Terraform files for deploying the HPC FrontEnd | +| `.../tf/state_bucket/` | Separate Terraform root for the Terraform state bucket | | `.../website/` | Source code for the HPC Frontend website | ##### Infrastructure Files @@ -347,10 +375,9 @@ Workbenches are created using a template configuration in `cluster-toolkit/frontend/infrastructure_files/workbench_tf`. The Terraform template was originally based on the Terraform template provided by the [Google Cloud Platform Rad-Lab git repo](https://github.com/GoogleCloudPlatform/rad-lab) -however the configuration diverged during early development. The main reason -for this divergence was to accommodate running the Jupyter notebook as a -specific OSLogin user rather than the generic Jupyter user which would make it -impossible to interact properly with any mounted shared storage. +however the configuration has since diverged. It now creates a Vertex AI +Workbench Instance (`google_workbench_instance`) whose post-startup script is +generated by the FrontEnd, and waits for the instance to become ready. The process of creating the workbench files is mostly contained within the file `cluster-toolkit/frontend/website/ghpcfe/cluster_manager/workbenchinfo.py`. The @@ -368,7 +395,6 @@ following info gathered by the FrontEnd during the workbench creation process: - boot_disk_type - boot_disk_size_gb - trusted_users -- image_family - owner_id - wb_startup_script_name - wb_startup_script_bucket @@ -396,30 +422,34 @@ social ID number set by the owner_id field. It also passes any configured mount points into the startup script before the second part of the startup script is copied from `infrastructure_files/gcs_bucket/workbench/startup_script_template.sh`. +The script is uploaded to the deployment's GCS bucket and passed to the +instance in the `post-startup-script` metadata key. On Vertex AI Workbench +Instances the managed image owns JupyterLab and its systemd unit, so the script +does not change the Jupyter configuration, its notebook directory or +`jupyter.service`; doing so stops the instance from becoming ready. JupyterLab +runs as the image's `jupyter` user, with `/home/jupyter` as the directory shown +in the web interface. + The startup script runs the following processes when the workbench instance boots: -- Query instance metadata for list of users and filter based on the users - social ID number to discover the correct format of their OSLogin username. -- Install nfs-common package via `apt-get`. -- Make temporary jupyterhome directory in /tmp/ and set user ownership. -- Make home directory for the user and set user ownership. -- Copy Jupyter configuration files from `/home/jupyter/.jupyter` to - `/tmp/jupyterhome/.jupyter`. -- Create `DATA_LOSS_WARNING.txt` file with warning message. -- Configure specified mount points in order specified on FrontEnd. -- Add symlink to `/tmp/jupyterhome/` which will serve as working directory on - the web interface. - - This process of mounting and symlinking means the mountpoint will appear - in both the jupyter notebook web interface working directory and in the - expected location in the root filesystem. -- Append mount points to `DATA_LOSS_WARNING.txt` file. -- If `/home` was not mounted as a mount point then create a symlink to - `/home` in `/tmp/jupyterhome`. -- Modify Jupyter config to reflect username and new working directory. -- Update `/lib/systemd/system/jupyter.service` systemd service file to reflect - username and new working directory. -- Run `systemctl` daemon-reload and restart Jupyter service. - - Without updating the Jupyter config and restarting the service then the - Jupyter notebook would be running as the jupyter user. This would break - permissions used on any mounted shared storage. +- Query instance metadata for the list of users and filter on the user's + social ID number to find their OSLogin username, which is logged to + `/tmp/startup.log`. +- Install the nfs-common package via `apt-get`. +- If the workbench is attached to a Slurm cluster, install the Slurm client + (built from source only if `scontrol` is not already present), copy the + cluster's authentication key (`slurm.key`, or `munge.key` for clusters using + MUNGE) from the controller, and mount the controller's Slurm configuration + at `/mnt/slurm_configs`, linking `slurm.conf` into `/etc/slurm`. +- Create `/home/jupyter/DATA_LOSS_WARNING.txt` with a warning message. +- For each mount point, in the order specified on the FrontEnd: + - mount the export at its mount path and make it writable by all users + (`chmod 777`), since JupyterLab runs as the `jupyter` user; + - symlink it into `/home/jupyter`, named after the mount path with `/` + replaced by `_` (for example `/data/shared` appears as `data_shared`), so + it appears both in the JupyterLab file browser and at its expected + location in the root filesystem; + - append it to `DATA_LOSS_WARNING.txt`. +- Log completion to `/tmp/startup.log`. No completion marker is written: the + `wait-for-startup` module detects readiness by polling the instance state. diff --git a/community/front-end/ofe/infrastructure_files/gcs_bucket/webserver/startup.sh b/community/front-end/ofe/infrastructure_files/gcs_bucket/webserver/startup.sh index 6162c61195..9ea2e484c9 100644 --- a/community/front-end/ofe/infrastructure_files/gcs_bucket/webserver/startup.sh +++ b/community/front-end/ofe/infrastructure_files/gcs_bucket/webserver/startup.sh @@ -30,6 +30,7 @@ if [ -z "${SERVER_IP_ADDRESS}" ]; then fi SERVER_HOSTNAME=$(curl --silent --fail http://metadata/computeMetadata/v1/instance/attributes/hostname -H "Metadata-Flavor: Google") config_bucket=$(curl --silent --show-error http://metadata/computeMetadata/v1/instance/attributes/webserver-config-bucket -H "Metadata-Flavor: Google") +state_bucket=$(curl --silent --show-error http://metadata/computeMetadata/v1/instance/attributes/terraform-state-bucket -H "Metadata-Flavor: Google") c2_topic=$(curl --silent --show-error http://metadata/computeMetadata/v1/instance/attributes/ghpcfe-c2-topic -H "Metadata-Flavor: Google") deploy_mode=$(curl --silent --show-error http://metadata/computeMetadata/v1/instance/attributes/deploy_mode -H "Metadata-Flavor: Google") deployment_name=$(curl --silent --show-error http://metadata/computeMetadata/v1/instance/attributes/deployment_name -H "Metadata-Flavor: Google") @@ -233,6 +234,7 @@ sudo su - gcluster -c /bin/bash <> configuration.yaml echo " gcp_project: \"$GCP_PROJECT\"" >> configuration.yaml echo " gcs_bucket: \"${config_bucket}\"" >> configuration.yaml + echo " terraform_state_bucket: \"${state_bucket}\"" >> configuration.yaml echo " c2_topic: \"${c2_topic}\"" >> configuration.yaml echo " deployment_name: \"${deployment_name}\"" >> configuration.yaml diff --git a/community/front-end/ofe/infrastructure_files/gcs_bucket/workbench/startup_script_template.sh b/community/front-end/ofe/infrastructure_files/gcs_bucket/workbench/startup_script_template.sh index 50cc62a079..780b4c25ca 100644 --- a/community/front-end/ofe/infrastructure_files/gcs_bucket/workbench/startup_script_template.sh +++ b/community/front-end/ofe/infrastructure_files/gcs_bucket/workbench/startup_script_template.sh @@ -13,40 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -if [ ! -d /tmp/jupyterhome/home ]; then ln -s /home /tmp/jupyterhome/; fi - -echo "modifying jupyter config" | tee -a /tmp/startup.log - -cat >>/tmp/jupyterhome/.jupyter/jupyter_notebook_config.py <<+ -jupyter_user = "$USER" -jupyter_home = "/tmp/jupyterhome" -sys.path.append(f"{jupyter_home}/.jupyter/") -c.ServerApp.notebook_dir = "/tmp/jupyterhome" -+ - -echo "modifying jupyter service" | tee -a /tmp/startup.log -cat >/lib/systemd/system/jupyter.service <<+ -[Unit] -Description=Jupyter Notebook Service - -[Service] -Type=simple -PIDFile=/run/jupyter.pid -MemoryHigh=3493718272 -MemoryMax=3543718272 -ExecStart=/bin/bash --login -c '/opt/conda/bin/jupyter lab --config=/tmp/jupyterhome/.jupyter/jupyter_notebook_config.py' -#User=jupyter -User=$USER -Group=$USER -WorkingDirectory=/tmp/jupyterhome -Restart=always - -[Install] -WantedBy=multi-user.target -+ - -echo "reloading and restarting service" | tee -a /tmp/startup.log -systemctl daemon-reload -service jupyter restart - -echo BRINGUP COMPLETE +# Trailer appended to the generated post-startup script. +# On Vertex AI Workbench Instances the image owns JupyterLab and its +# systemd unit: do NOT relocate the notebook directory or overwrite the +# unit, or the instance never becomes ready. Readiness is detected by +# wait-for-startup polling the instance state, so no completion marker +# is needed here. + +echo "OFE workbench post-startup script finished at $(date)" | tee -a /tmp/startup.log diff --git a/community/front-end/ofe/infrastructure_files/workbench_tf/google/README.md b/community/front-end/ofe/infrastructure_files/workbench_tf/google/README.md index f5e71fcc68..920d2f6e0c 100644 --- a/community/front-end/ofe/infrastructure_files/workbench_tf/google/README.md +++ b/community/front-end/ofe/infrastructure_files/workbench_tf/google/README.md @@ -18,7 +18,7 @@ limitations under the License. | Name | Version | | ---- | ------- | | [terraform](#requirement\_terraform) | >= 1.12.2 | -| [google](#requirement\_google) | >= 3.87.0 | +| [google](#requirement\_google) | >= 6.4.0 | | [random](#requirement\_random) | >= 2.0 | | [time](#requirement\_time) | >= 0.7.2 | @@ -26,7 +26,7 @@ limitations under the License. | Name | Version | | ---- | ------- | -| [google](#provider\_google) | >= 3.87.0 | +| [google](#provider\_google) | >= 6.4.0 | | [random](#provider\_random) | >= 2.0 | | [time](#provider\_time) | >= 0.7.2 | @@ -34,8 +34,8 @@ limitations under the License. | Name | Source | Version | | ---- | ------ | ------- | -| [project\_radlab\_ds\_analytics](#module\_project\_radlab\_ds\_analytics) | terraform-google-modules/project-factory/google | ~> 11.0 | -| [vpc\_ai\_notebook](#module\_vpc\_ai\_notebook) | terraform-google-modules/network/google | ~> 3.0 | +| [project\_radlab\_ds\_analytics](#module\_project\_radlab\_ds\_analytics) | terraform-google-modules/project-factory/google | ~> 18.0 | +| [vpc\_ai\_notebook](#module\_vpc\_ai\_notebook) | terraform-google-modules/network/google | ~> 10.0 | | [waitforstartup](#module\_waitforstartup) | ./wait-for-startup | n/a | ## Resources @@ -43,7 +43,6 @@ limitations under the License. | Name | Type | | ---- | ---- | | [google_compute_instance_iam_member.oslogin_permissions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_iam_member) | resource | -| [google_notebooks_instance.ai_notebook](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/notebooks_instance) | resource | | [google_project_iam_member.sa_p_notebook_permissions](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | | [google_project_organization_policy.external_ip_policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_organization_policy) | resource | | [google_project_organization_policy.shielded_vm_policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_organization_policy) | resource | @@ -52,6 +51,7 @@ limitations under the License. | [google_service_account.sa_p_notebook](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource | | [google_service_account_iam_member.sa_ai_notebook_user_iam](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account_iam_member) | resource | | [google_storage_bucket_object.startup_script](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource | +| [google_workbench_instance.ai_notebook](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/workbench_instance) | resource | | [random_id.default](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | | [time_sleep.wait_120_seconds](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [google_compute_network.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network) | data source | @@ -67,11 +67,11 @@ limitations under the License. | [boot\_disk\_type](#input\_boot\_disk\_type) | Disk types for notebook instances | `string` | `"PD_SSD"` | no | | [create\_network](#input\_create\_network) | If the module has to be deployed in an existing network, set this variable to false. | `bool` | `false` | no | | [create\_project](#input\_create\_project) | Set to true if the module has to create a project. If you want to deploy in an existing project, set this variable to false. | `bool` | `false` | no | +| [data\_disk\_size\_gb](#input\_data\_disk\_size\_gb) | Size in GB, up to 64000, of the instance's data disk (/home/jupyter). Billed in addition to the boot disk. | `number` | `100` | no | | [enable\_services](#input\_enable\_services) | Enable the necessary APIs on the project. When using an existing project, this can be set to false. | `bool` | `false` | no | | [folder\_id](#input\_folder\_id) | Folder ID where the project should be created. It can be skipped if already setting organization\_id. Leave blank if the project should be created directly underneath the Organization node. | `string` | `""` | no | | [image\_family](#input\_image\_family) | DEPRECATED: Image of the AI notebook. | `string` | `null` | no | | [image\_project](#input\_image\_project) | DEPRECATED: Google Cloud project where the image is hosted. | `string` | `null` | no | -| [instance\_image](#input\_instance\_image) | Image of the AI notebook.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted. | `map(string)` | 
{
  "family": "tf-latest-cpu",
  "project": "deeplearning-platform-release"
}
| no | | [ip\_cidr\_range](#input\_ip\_cidr\_range) | Unique IP CIDR Range for AI Notebooks subnet | `string` | `"10.142.190.0/24"` | no | | [machine\_type](#input\_machine\_type) | Type of VM you would like to spin up | `string` | `"n1-standard-1"` | no | | [network\_name](#input\_network\_name) | Name of the network to be created. | `string` | `"ai-notebook"` | no | diff --git a/community/front-end/ofe/infrastructure_files/workbench_tf/google/main.tf b/community/front-end/ofe/infrastructure_files/workbench_tf/google/main.tf index 62eb9b0bb1..b42aa81567 100644 --- a/community/front-end/ofe/infrastructure_files/workbench_tf/google/main.tf +++ b/community/front-end/ofe/infrastructure_files/workbench_tf/google/main.tf @@ -71,7 +71,7 @@ data "google_project" "existing_project" { module "project_radlab_ds_analytics" { count = var.create_project ? 1 : 0 source = "terraform-google-modules/project-factory/google" - version = "~> 11.0" + version = "~> 18.0" name = format("%s-%s", var.project_name, local.random_id) random_project_id = false @@ -110,7 +110,7 @@ data "google_compute_subnetwork" "default" { module "vpc_ai_notebook" { count = var.create_network ? 1 : 0 source = "terraform-google-modules/network/google" - version = "~> 3.0" + version = "~> 10.0" project_id = local.project.project_id network_name = var.network_name @@ -166,47 +166,55 @@ resource "google_service_account_iam_member" "sa_ai_notebook_user_iam" { member = "user:${var.trusted_user}" } -resource "google_notebooks_instance" "ai_notebook" { - project = local.project.project_id - name = "notebooks-instance-${local.random_id}" - location = var.zone - machine_type = var.machine_type - - vm_image { - image_family = try(var.instance_image.family, null) - image_name = try(var.instance_image.name, null) - project = var.instance_image.project - } +# Vertex AI Workbench Instance: Google retired user-managed notebooks +# (google_notebooks_instance), which can no longer be created. Legacy +# deep-learning image families do not apply, so the default Workbench image +# is used. +resource "google_workbench_instance" "ai_notebook" { + project = local.project.project_id + name = "notebooks-instance-${local.random_id}" + location = var.zone - instance_owners = var.owner_id + gce_setup { + machine_type = var.machine_type - install_gpu_driver = false - boot_disk_type = var.boot_disk_type - boot_disk_size_gb = var.boot_disk_size_gb + boot_disk { + # The API rejects boot disks smaller than the 150 GB Workbench image. + disk_size_gb = max(var.boot_disk_size_gb, 150) + disk_type = var.boot_disk_type + } - no_public_ip = false - no_proxy_access = false + # Workbench Instances also create a data disk, mounted at /home/jupyter. + data_disks { + disk_size_gb = var.data_disk_size_gb + disk_type = var.boot_disk_type + } - network = local.network.self_link - subnet = local.subnet.self_link + network_interfaces { + network = local.network.self_link + subnet = local.subnet.self_link + } - post_startup_script = format("gs://%s/%s", var.wb_startup_script_bucket, var.wb_startup_script_name) + disable_public_ip = false - labels = { - module = "data-science" + metadata = { + terraform = "true" + proxy-mode = "mail" + post-startup-script = format("gs://%s/%s", var.wb_startup_script_bucket, var.wb_startup_script_name) + } } - metadata = { - terraform = "true" - proxy-mode = "mail" - } + instance_owners = var.owner_id + labels = { + module = "data-science" + } } resource "google_compute_instance_iam_member" "oslogin_permissions" { - project = google_notebooks_instance.ai_notebook.project - zone = google_notebooks_instance.ai_notebook.location - instance_name = google_notebooks_instance.ai_notebook.name + project = google_workbench_instance.ai_notebook.project + zone = google_workbench_instance.ai_notebook.location + instance_name = google_workbench_instance.ai_notebook.name role = "roles/compute.osLogin" member = "user:${var.trusted_user}" } @@ -215,8 +223,8 @@ module "waitforstartup" { source = "./wait-for-startup" - instance_name = google_notebooks_instance.ai_notebook.name - zone = google_notebooks_instance.ai_notebook.location - project_id = google_notebooks_instance.ai_notebook.project + instance_name = google_workbench_instance.ai_notebook.name + zone = google_workbench_instance.ai_notebook.location + project_id = google_workbench_instance.ai_notebook.project timeout = 1200 } diff --git a/community/front-end/ofe/infrastructure_files/workbench_tf/google/outputs.tf b/community/front-end/ofe/infrastructure_files/workbench_tf/google/outputs.tf index 6cf9245f0d..15c42041f3 100644 --- a/community/front-end/ofe/infrastructure_files/workbench_tf/google/outputs.tf +++ b/community/front-end/ofe/infrastructure_files/workbench_tf/google/outputs.tf @@ -26,10 +26,10 @@ output "project_radlab_ds_analytics_id" { output "notebook_instance_name" { description = "Notebook Instance Names" - value = google_notebooks_instance.ai_notebook.name + value = google_workbench_instance.ai_notebook.name } output "notebook_proxy_uri" { description = "Notebook Proxy URIs" - value = google_notebooks_instance.ai_notebook.proxy_uri + value = google_workbench_instance.ai_notebook.proxy_uri } diff --git a/community/front-end/ofe/infrastructure_files/workbench_tf/google/provider.tf b/community/front-end/ofe/infrastructure_files/workbench_tf/google/provider.tf index f825178b66..c4b8e00ba6 100644 --- a/community/front-end/ofe/infrastructure_files/workbench_tf/google/provider.tf +++ b/community/front-end/ofe/infrastructure_files/workbench_tf/google/provider.tf @@ -15,7 +15,9 @@ */ provider "google" { - region = var.region - zone = var.zone - project = var.project + region = var.region + zone = var.zone + # Provider 6.x rejects an empty project, and var.project is optional, so + # fall back to the existing-project name that OFE always supplies. + project = var.project != "" ? var.project : var.project_name } diff --git a/community/front-end/ofe/infrastructure_files/workbench_tf/google/variables.tf b/community/front-end/ofe/infrastructure_files/workbench_tf/google/variables.tf index 7c14f0857c..2bab8eb5d1 100644 --- a/community/front-end/ofe/infrastructure_files/workbench_tf/google/variables.tf +++ b/community/front-end/ofe/infrastructure_files/workbench_tf/google/variables.tf @@ -44,6 +44,17 @@ variable "create_project" { default = false } +variable "data_disk_size_gb" { + description = "Size in GB, up to 64000, of the instance's data disk (/home/jupyter). Billed in addition to the boot disk." + type = number + default = 100 + + validation { + condition = var.data_disk_size_gb > 0 && var.data_disk_size_gb <= 64000 + error_message = "The 'data_disk_size_gb' setting must be greater than 0 and at most 64000." + } +} + variable "enable_services" { description = "Enable the necessary APIs on the project. When using an existing project, this can be set to false." type = bool @@ -56,7 +67,7 @@ variable "folder_id" { default = "" } -# Deprecated, replaced by instance_image +# Deprecated; kept so an old tfvars setting fails clearly # tflint-ignore: terraform_unused_declarations variable "image_family" { description = "DEPRECATED: Image of the AI notebook." @@ -65,11 +76,11 @@ variable "image_family" { validation { condition = var.image_family == null - error_message = "The 'image_family' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + error_message = "The 'image_family' setting is no longer supported: Workbench Instances use the default Workbench image." } } -# Deprecated, replaced by instance_image +# Deprecated; kept so an old tfvars setting fails clearly # tflint-ignore: terraform_unused_declarations variable "image_project" { description = "DEPRECATED: Google Cloud project where the image is hosted." @@ -78,35 +89,12 @@ variable "image_project" { validation { condition = var.image_project == null - error_message = "The 'image_project' setting is deprecated, please use 'var.instance_image' with the fields 'project' and 'family' or 'name'." + error_message = "The 'image_project' setting is no longer supported: Workbench Instances use the default Workbench image." } } -variable "instance_image" { - description = <<-EOD - Image of the AI notebook. - - Expected Fields: - name: The name of the image. Mutually exclusive with family. - family: The image family to use. Mutually exclusive with name. - project: The project where the image is hosted. - EOD - type = map(string) - default = { - project = "deeplearning-platform-release" - family = "tf-latest-cpu" - } - - validation { - condition = can(coalesce(var.instance_image.project)) - error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID." - } - - validation { - condition = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family)) - error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name." - } -} +# No instance_image variable: Workbench Instances always use the default +# Workbench image (see main.tf). variable "ip_cidr_range" { description = "Unique IP CIDR Range for AI Notebooks subnet" diff --git a/community/front-end/ofe/infrastructure_files/workbench_tf/google/versions.tf b/community/front-end/ofe/infrastructure_files/workbench_tf/google/versions.tf index 4ee45d2c4a..c788e903fe 100644 --- a/community/front-end/ofe/infrastructure_files/workbench_tf/google/versions.tf +++ b/community/front-end/ofe/infrastructure_files/workbench_tf/google/versions.tf @@ -20,7 +20,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 3.87.0" + version = ">= 6.4.0" } random = { source = "hashicorp/random" diff --git a/community/front-end/ofe/infrastructure_files/workbench_tf/google/wait-for-startup/scripts/wait-for-startup-status.sh b/community/front-end/ofe/infrastructure_files/workbench_tf/google/wait-for-startup/scripts/wait-for-startup-status.sh index 8f34c4a53b..c67e1b40ae 100755 --- a/community/front-end/ofe/infrastructure_files/workbench_tf/google/wait-for-startup/scripts/wait-for-startup-status.sh +++ b/community/front-end/ofe/infrastructure_files/workbench_tf/google/wait-for-startup/scripts/wait-for-startup-status.sh @@ -13,6 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Readiness detection for Vertex AI Workbench Instances. +# +# Polls the Workbench API for `state == ACTIVE` (JupyterLab up, proxy URI +# populated) rather than a serial-console marker, which Workbench images +# never emit. This is independent of the OFE post-startup script, so a +# slow or failing post-startup step does not block or fail deployment. + if [ -z "${INSTANCE_NAME}" ]; then echo "INSTANCE_NAME is unset" exit 1 @@ -27,26 +34,30 @@ if [ -z "${PROJECT_ID}" ]; then fi CLOUDSDK_CONFIG=$(mktemp -d) +export CLOUDSDK_CONFIG GOOGLE_APPLICATION_CREDENTIALS="../../cloud_credentials" -gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} -GCLOUD="gcloud compute instances get-serial-port-output ${INSTANCE_NAME} --port 1 --zone ${ZONE} --project ${PROJECT_ID}" -FINISH_LINE="BRINGUP COMPLETE" -ERROR_LINE="BRINGUP FAILED" +gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" +# TIMEOUT: 0 = ready, 1 = timed out, 2 = instance entered a terminal +# non-ready state (stopped/suspended/deleted) during bring-up. TIMEOUT=1 tries=0 -until [ $tries -ge "${RETRIES}" ]; do - STATUS_LINE=$(${GCLOUD} 2>/dev/null | grep -oE "${FINISH_LINE}|${ERROR_LINE}") - echo "STATUS_LINE: '${STATUS_LINE}'" - if [ "${STATUS_LINE}" == "${FINISH_LINE}" ]; then +until [ "$tries" -ge "${RETRIES}" ]; do + STATE=$(gcloud workbench instances describe "${INSTANCE_NAME}" \ + --location "${ZONE}" --project "${PROJECT_ID}" \ + --format="value(state)" 2>/dev/null) + echo "workbench state: '${STATE}'" + if [ "${STATE}" == "ACTIVE" ]; then TIMEOUT=0 break fi - if [ "${STATUS_LINE}" == "${ERROR_LINE}" ]; then + case "${STATE}" in + STOPPED | SUSPENDED | DELETED) TIMEOUT=2 break - fi - echo "could not detect end of startup script. Sleeping." + ;; + esac + echo "workbench not ACTIVE yet. Sleeping." sleep 30 ((tries++)) done @@ -54,11 +65,11 @@ done rm -r "${CLOUDSDK_CONFIG}" if [ "${TIMEOUT}" -eq "0" ]; then - echo "Startup script completed" + echo "Workbench instance is ACTIVE" elif [ "${TIMEOUT}" -eq "1" ]; then - echo "Startup script timed out" >&2 + echo "Timed out waiting for workbench instance to become ACTIVE" >&2 else - echo "Node bringup failed - check virtual serial console logs" >&2 + echo "Workbench instance entered a terminal non-ready state: '${STATE}'" >&2 fi exit "${TIMEOUT}" diff --git a/community/front-end/ofe/script/migrate_tf_state_to_gcs.sh b/community/front-end/ofe/script/migrate_tf_state_to_gcs.sh new file mode 100755 index 0000000000..b6ac418528 --- /dev/null +++ b/community/front-end/ofe/script/migrate_tf_state_to_gcs.sh @@ -0,0 +1,440 @@ +#! /bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################################ +# +# One-off migration of Terraform state from local disk into the TKFE state +# bucket. +# +# Resources TKFE provisioned before state moved to a versioned GCS bucket +# still have a terraform.tfstate in their run-time directory on the service +# machine. This script moves that state into the bucket without recreating or +# interrupting the resource. +# +# Run it ON THE SERVICE MACHINE, as the user that owns the run-time +# directories, after upgrading TKFE: +# +# ./script/migrate_tf_state_to_gcs.sh --dry-run +# ./script/migrate_tf_state_to_gcs.sh --yes +# +# Safety: +# - The state is copied to terraform.tfstate.premigration BEFORE migrating, +# because 'terraform init -migrate-state' empties the original. The copy +# is never deleted or overwritten. +# - The copy is kept in the resource's run-time directory, beside its +# cloud_credentials (e.g. clusters/cluster_/), not in the Terraform +# working directory: for clusters, filesystems and image builders that is +# a Cluster Toolkit deployment group, which the Toolkit moves into .ghpc/ +# and later deletes on every reconfigure. +# - Nothing is changed without --yes. +# - The bucket is versioned, so a bad upload can be rolled back. +# - Success is reported only once 'terraform plan' shows the migrated state +# still matches reality. +# +# Recovery from a half-finished directory: copy terraform.tfstate.premigration +# from the resource's run-time directory to terraform.tfstate in the Terraform +# working directory, delete .terraform/, remove backend.tf if this script added +# it, delete terraform.tfstate.migration-incomplete if present, and run +# 'terraform init'. +# +################################################################################ + +set -o pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +BASE_DIR=$(cd -- "${SCRIPT_DIR}/.." &>/dev/null && pwd) +# The application's own interpreter, with Django and OFE's dependencies. +# Every call that imports OFE code must use it. +OFE_PYTHON=${OFE_PYTHON:-/opt/gcluster/django-env/bin/python3} + +dry_run=0 +confirmed=0 +failures=0 +migrated=0 +skipped=0 + +error() { + echo "$*" >&2 +} + +usage() { + cat <<'USAGE' +Usage: migrate_tf_state_to_gcs.sh [--dry-run | --yes] [--dir ] + + --dry-run Report what would be migrated; change nothing. + --yes Perform the migration. + --dir Migrate only this one working directory (repeatable). + +Exactly one of --dry-run or --yes is required. +USAGE +} + +# +# Print the state bucket from the application configuration, the same source +# the application uses. +# +get_state_bucket() { + PYTHONPATH="${BASE_DIR}/website" "${OFE_PYTHON}" -c ' +import sys +from ghpcfe.cluster_manager import utils +try: + print(utils.get_terraform_state_bucket()) +except (OSError, RuntimeError) as err: + sys.exit(f"Error: {err}") +' +} + +# +# Print this server's state namespace, read from its database. +# +get_state_namespace() { + + local ns + ns=$(cd "${BASE_DIR}/website" && "${OFE_PYTHON}" manage.py tfstate_namespace | tail -n 1) || return 1 + if [[ ! ${ns} =~ ^[0-9]{8}-[0-9a-f]{8}$ ]]; then + error "Error: could not read the Terraform state namespace (got '${ns}')" + return 1 + fi + echo "${ns}" +} + +# +# Print the state prefix for working directory ${1}'s resource. +# - Only the directory-to-resource-kind mapping lives here; the prefix itself +# comes from utils.terraform_state_prefix(), as in the application, so the +# two cannot drift apart. +# +prefix_for_dir() { + + local dir=${1} + local rel=${dir#"${BASE_DIR}/"} + + local kind id + case ${rel} in + clusters/cluster_*) kind=cluster id=${rel#clusters/cluster_} ;; + fs/fs_*) kind=filesystem id=${rel#fs/fs_} ;; + vpcs/vpc_*) kind=vpc id=${rel#vpcs/vpc_} ;; + workbenches/workbench_*) kind=workbench id=${rel#workbenches/workbench_} ;; + images/image_*) kind=image id=${rel#images/image_} ;; + *) return 1 ;; + esac + id=${id%%/*} + + PYTHONPATH="${BASE_DIR}/website" "${OFE_PYTHON}" -c ' +import sys +from ghpcfe.cluster_manager import utils +print(utils.terraform_state_prefix(sys.argv[1], sys.argv[2], namespace=sys.argv[3])) +' "${kind}" "${id}" "${state_namespace}" +} + +# +# Print the credential Terraform was run with for working directory ${1}. +# - It is at the top of the resource's run-time directory, not in the +# Terraform working directory, which may be nested below it. +# +credentials_for_dir() { + + local dir=${1} + local probe=${dir} + + while [[ ${probe} != "${BASE_DIR}" && ${probe} != "/" ]]; do + if [[ -f ${probe}/cloud_credentials ]]; then + echo "${probe}/cloud_credentials" + return 0 + fi + probe=$(dirname "${probe}") + done + return 1 +} + +# +# Print the path of the pre-migration copy of working directory ${1}'s state. +# - It goes in the resource's run-time directory, /_, +# which holds cloud_credentials. For clusters, filesystems and image +# builders the working directory is a Cluster Toolkit deployment group +# below that; every 'gcluster create -w' moves the groups into .ghpc/ and +# deletes the previous set, so a copy kept there would not survive two +# reconfigures. For networks the two directories are the same. +# +backup_for_dir() { + + local dir=${1} + local rel=${dir#"${BASE_DIR}/"} + + case ${rel} in + clusters/cluster_* | fs/fs_* | vpcs/vpc_* | workbenches/workbench_* | images/image_*) ;; + *) return 1 ;; + esac + local kind_dir=${rel%%/*} + local resource=${rel#*/} + resource=${resource%%/*} + + echo "${BASE_DIR}/${kind_dir}/${resource}/terraform.tfstate.premigration" +} + +# +# Migrate Terraform working directory ${1}. +# +migrate_dir() { + + local dir=${1} + local state="${dir}/terraform.tfstate" + local incomplete="${state}.migration-incomplete" + local prefix creds bucket backup + + if [[ -f ${incomplete} ]]; then + backup=$(backup_for_dir "${dir}") || backup="not known" + error " FAIL: an earlier migration of ${dir} did not finish: its state is already" + error " in the bucket, but terraform plan did not come back clean. Investigate," + error " then delete ${incomplete} (the original is ${backup})." + ((failures += 1)) + return 1 + fi + + if [[ ! -f ${state} ]]; then + echo " skip: no local state in ${dir}" + ((skipped += 1)) + return 0 + fi + + if ! python3 -c "import json,sys; json.load(open(sys.argv[1]))" "${state}" 2>/dev/null; then + error " FAIL: ${state} is not valid JSON; refusing to migrate" + ((failures += 1)) + return 1 + fi + + # An empty state has nothing to move and would silently "succeed". + local nresources + nresources=$(python3 -c \ + "import json,sys; print(len(json.load(open(sys.argv[1])).get('resources', [])))" \ + "${state}") + if [[ ${nresources} -eq 0 ]]; then + echo " skip: ${dir} has no resources in state" + ((skipped += 1)) + return 0 + fi + + if ! prefix=$(prefix_for_dir "${dir}"); then + error " FAIL: cannot derive a state prefix for ${dir}" + ((failures += 1)) + return 1 + fi + + if ! backup=$(backup_for_dir "${dir}"); then + error " FAIL: cannot derive a backup location for ${dir}" + ((failures += 1)) + return 1 + fi + # Never overwrite an existing copy: it may be the only record of another + # working directory's state. A copy identical to the current state is one + # this script took on an earlier run that stopped before migrating. + if [[ -e ${backup} ]] && ! cmp -s "${state}" "${backup}"; then + error " FAIL: ${backup} already exists and differs from" + error " ${state}; refusing to overwrite it. Find out which" + error " state it holds, move it elsewhere if it is no longer needed, and rerun." + ((failures += 1)) + return 1 + fi + + bucket=${state_bucket} + + if [[ ${dry_run} -eq 1 ]]; then + echo " would migrate: ${dir}" + echo " ${nresources} resources -> gs://${bucket}/${prefix}/" + echo " original to be kept as ${backup}" + return 0 + fi + + if ! creds=$(credentials_for_dir "${dir}"); then + error " FAIL: no cloud_credentials found for ${dir}" + ((failures += 1)) + return 1 + fi + + # Blueprint-generated directories get their backend from the regenerated + # blueprint; raw-template directories need one written here. + if ! grep -rqs 'backend "gcs"' "${dir}"/*.tf; then + echo " writing backend.tf for ${dir}" + PYTHONPATH="${BASE_DIR}/website" "${OFE_PYTHON}" -c ' +import sys +from ghpcfe.cluster_manager import utils +utils.write_backend_tf(sys.argv[1], sys.argv[2]) +' "${dir}" "${prefix}" || { + error " FAIL: could not write backend.tf for ${dir}" + ((failures += 1)) + return 1 + } + fi + + echo " migrating ${dir} -> gs://${bucket}/${prefix}/" + + # Copy BEFORE migrating: 'terraform init -migrate-state' empties the local + # state file. An existing copy was checked above to be identical, so it + # is kept as it is. + if [[ ! -e ${backup} ]] && ! cp -p "${state}" "${backup}"; then + error " FAIL: could not back up ${state} to ${backup}; not migrating" + ((failures += 1)) + return 1 + fi + if [[ ! -s ${backup} ]]; then + error " FAIL: backup ${backup} is empty; not migrating" + ((failures += 1)) + return 1 + fi + + if ! GOOGLE_APPLICATION_CREDENTIALS="${creds}" \ + terraform -chdir="${dir}" init -migrate-state -force-copy -no-color; then + error " FAIL: terraform init -migrate-state failed for ${dir}" + ((failures += 1)) + return 1 + fi + + # Prove the migrated state still matches reality before touching the + # local copy. An empty plan is the only acceptable outcome. + local rc=0 + GOOGLE_APPLICATION_CREDENTIALS="${creds}" \ + terraform -chdir="${dir}" plan -detailed-exitcode -no-color >/dev/null || rc=$? + if [[ ${rc} -ne 0 ]]; then + # init -migrate-state has already moved the state and emptied the local + # file. Rename it so a rerun reports this directory rather than + # skipping it as "no resources". + mv -f "${state}" "${incomplete}" + if [[ ${rc} -eq 2 ]]; then + error " FAIL: ${dir}: state moved to gs://${bucket}/${prefix}/, but terraform plan" + error " reports changes. The original is ${backup}." + else + error " FAIL: ${dir}: state moved to gs://${bucket}/${prefix}/, but terraform plan" + error " errored. The original is ${backup}." + fi + ((failures += 1)) + return 1 + fi + + # Remove the state file the migration emptied and Terraform's own backup; + # only the copy taken above is kept. + rm -f "${state}" "${state}.backup" + + echo " OK: ${dir} (pre-migration state kept as ${backup})" + ((migrated += 1)) + return 0 +} + +################################################################################ +# +declare -a explicit_dirs + +while [[ ${#} -gt 0 ]]; do + case ${1} in + --dry-run) + dry_run=1 + shift + ;; + --yes) + confirmed=1 + shift + ;; + --dir) + if ! d=$(cd -- "${2}" 2>/dev/null && pwd); then + error "Error: --dir: no such directory: ${2}" + exit 1 + fi + explicit_dirs+=("${d}") + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + *) + error "Unknown argument: ${1}" + usage + exit 1 + ;; + esac +done + +if [[ ${dry_run} -eq 0 && ${confirmed} -eq 0 ]]; then + error "Refusing to run: pass --dry-run to preview, or --yes to migrate." + usage + exit 1 +fi +if [[ ${dry_run} -eq 1 && ${confirmed} -eq 1 ]]; then + error "Pass only one of --dry-run and --yes." + exit 1 +fi + +if ! command -v terraform &>/dev/null; then + error "Error: terraform is not in \$PATH" + exit 1 +fi + +if ! state_bucket=$(get_state_bucket); then + exit 1 +fi +if ! state_namespace=$(get_state_namespace); then + exit 1 +fi + +echo "" +echo "Terraform state bucket: gs://${state_bucket}" +echo "State namespace: ${state_namespace}" +if [[ ${dry_run} -eq 1 ]]; then + echo "Mode: dry run (nothing will be changed)" +else + echo "Mode: MIGRATE" +fi +echo "" + +# -- Collect the Terraform working directories to consider. A working directory +# is one holding a local terraform.tfstate. +# +declare -a dirs +if [[ ${#explicit_dirs[@]} -gt 0 ]]; then + dirs=("${explicit_dirs[@]}") +else + # Prune .ghpc/ (the Toolkit's backups of earlier deployment groups, + # whose stale state would map to the live prefix) and .terraform/ + # (Terraform's backend settings, not a state file). Also collect + # directories a previous run left half-migrated, so they are reported. + while IFS= read -r state_file; do + d=$(dirname "${state_file}") + [[ " ${dirs[*]} " == *" ${d} "* ]] || dirs+=("${d}") + done < <(find "${BASE_DIR}"/clusters "${BASE_DIR}"/fs "${BASE_DIR}"/vpcs \ + "${BASE_DIR}"/images "${BASE_DIR}"/workbenches \ + \( -name .ghpc -o -name .terraform \) -prune -o \ + -type f \( -name terraform.tfstate -o -name terraform.tfstate.migration-incomplete \) \ + -print 2>/dev/null | sort -u) +fi + +if [[ ${#dirs[@]} -eq 0 ]]; then + echo "No local Terraform state found. Nothing to migrate." + exit 0 +fi + +for dir in "${dirs[@]}"; do + migrate_dir "${dir}" || true +done + +echo "" +echo "Migrated: ${migrated} Skipped: ${skipped} Failed: ${failures}" +echo "" + +if [[ ${failures} -gt 0 ]]; then + exit 1 +fi +exit 0 + +# +# eof diff --git a/community/front-end/ofe/teardown.sh b/community/front-end/ofe/teardown.sh index ee94760c76..074b5504a3 100755 --- a/community/front-end/ofe/teardown.sh +++ b/community/front-end/ofe/teardown.sh @@ -84,6 +84,224 @@ cleanup_server_service_account() { return 0 } +# +# Print the state bucket working directory of deployment ${1}: its own +# directory under tf/state_bucket/deployments/, or tf/state_bucket/ itself if +# an earlier release left that deployment's files there and deploy.sh has not +# moved them yet. Prints nothing if there is neither. +# +state_bucket_dir() { + + local name=${1} + + [[ ${name} == "" || ${name} == . || ${name} == .. || ${name} == */* ]] && return 0 + if [[ -f tf/state_bucket/deployments/${name}/terraform.tfstate ]]; then + echo "tf/state_bucket/deployments/${name}" + elif [[ -f tf/state_bucket/terraform.tfstate ]] && + grep -qxF "deployment_name = \"${name}\"" tf/state_bucket/terraform.tfvars 2>/dev/null; then + echo "tf/state_bucket" + fi +} + +# +# Return 0 only if no FrontEnd is provably deployed from tf/: no lock file, +# and tf/'s state can be read and records no FrontEnd. Fails closed, so a +# bucket is never deleted from under a FrontEnd that may still be running. +# +front_end_gone() { + + local shown + + [[ -f tf/.tkfe.lock ]] && return 1 + shown=$(cd tf && terraform show 2>/dev/null) || return 1 + ! grep -q ghpcfe_id <<<"${shown}" +} + +# +# How to get past a front_end_gone() refusal. +# +front_end_gone_help() { + + echo " If the FrontEnd is still running, tear it down and delete its" + echo " bucket with: ./teardown.sh --delete-state-bucket" + echo " If it is already gone (for example after a failed teardown), run" + echo " 'terraform -chdir=tf init' if needed, check that" + echo " 'terraform -chdir=tf state list' lists nothing, then remove the" + echo " leftover lock file tf/.tkfe.lock and rerun." +} + +# +# Check deployment ${1}'s state bucket can be deleted, and confirm. +# - Sets ${sb_dir} and ${state_bucket}. +# - Refuses unless every state in the bucket is provably empty, before +# anything is destroyed, so a refusal leaves everything as it was. +# +prepare_state_bucket_deletion() { + + local name=${1} + local reply + + sb_dir=$(state_bucket_dir "${name}") + if [[ -z ${sb_dir} ]]; then + echo " Error: no state bucket for deployment '${name}' is recorded in this" + echo " directory (tf/state_bucket/deployments/${name}/)." + return 1 + fi + state_bucket=$(terraform -chdir="${sb_dir}" output -raw bucket_name 2>/dev/null) || + state_bucket="" + if [[ -z ${state_bucket} ]]; then + echo " Error: ${sb_dir} records no state bucket; it may already be deleted." + return 1 + fi + + check_state_bucket_empty "${state_bucket}" || return 1 + + echo " This will permanently delete the Terraform state bucket" + echo " gs://${state_bucket}" + echo " and every object in it, including earlier versions of state files." + if [[ ${assume_yes} -eq 1 ]]; then + echo " -y flag passed. Proceeding." + else + read -r -p " Type the bucket name to confirm: " reply || reply="" + if [[ ${reply} != "${state_bucket}" ]]; then + echo " Not confirmed; nothing has been changed." + echo "Exiting." + exit 0 + fi + fi + echo "" +} + +# +# Check a state bucket holds no resources. +# - Reads every Terraform state in the bucket, under every namespace and +# prefix. Lists each that records resources or cannot be read, and each +# lock (an operation may still be running). +# - Returns 1 unless every state is provably empty. +# +check_state_bucket_empty() { + + local bucket=${1} + local objects object noncurrent count + local -a busy=() + + # The bucket is versioned and the listing includes noncurrent versions, + # which every released lock and superseded state leaves; only live + # objects count. + if ! objects=$(gcloud storage objects list "gs://${bucket}/**" \ + --format='value(name,noncurrent_time)' 2>&1); then + echo " Error: could not list gs://${bucket}:" + echo " ${objects}" + return 1 + fi + while IFS=$'\t' read -r object noncurrent; do + [[ -z ${object} || -n ${noncurrent} ]] && continue + case ${object} in + *.tflock) + busy+=("${object} (locked: an operation may be running)") + ;; + *.tfstate) + if ! count=$(gcloud storage cat "gs://${bucket}/${object}" 2>/dev/null | + python3 -c 'import json, sys; print(len(json.load(sys.stdin)["resources"]))' 2>/dev/null); then + busy+=("${object} (could not be read)") + elif [[ ${count} -ne 0 ]]; then + busy+=("${object} (${count} resources)") + fi + ;; + esac + done <<<"${objects}" + + if [[ ${#busy[@]} -gt 0 ]]; then + echo " Error: gs://${bucket} is not provably empty, so it will not be" + echo " deleted:" + printf ' %s\n' "${busy[@]}" + echo " Delete those resources in TKFE first, or destroy them from" + echo " this state (see 'Terraform State' in docs/admin_guide.md)." + return 1 + fi +} + +# +# Delete the state bucket in ${sb_dir}. +# - The bucket is protected by prevent_destroy and force_destroy = false in +# tf/state_bucket/main.tf. Those are lifted in a temporary copy, so the +# tracked files are never edited. force_destroy is read from state, so it +# is applied to the bucket before the destroy. +# - Refuses if Terraform would change the bucket in any other way, for +# example replace it after a change of region. +# - The copy's state, which records what is left, replaces ${sb_dir}'s +# whether or not the destroy succeeds, and also on an interrupt. +# +delete_state_bucket() { + + local tmp rc=0 started=0 actions + + tmp=$(mktemp -d) || return 1 + # Before Terraform starts nothing has changed, so the copy is not needed. + trap 'if [[ ${started} -eq 1 ]]; then + cp "${tmp}/terraform.tfstate" "${sb_dir}/terraform.tfstate" || true + fi + rm -rf "${tmp:?}" + echo " Interrupted. To finish, rerun teardown.sh with --delete-state-bucket." + exit 130' INT TERM + cp tf/state_bucket/*.tf "${sb_dir}"/terraform.tfstate "${sb_dir}"/*.tfvars "${tmp}/" || + rc=1 + sed -i -e '/^ lifecycle {$/,/^ }$/d' \ + -e 's/^ force_destroy = false$/ force_destroy = true/' "${tmp}/main.tf" || rc=1 + if grep -q '^ *prevent_destroy *= *true' "${tmp}/main.tf" || + ! grep -q '^ force_destroy = true$' "${tmp}/main.tf"; then + echo " Error: could not lift the bucket's deletion protection in a copy of" + echo " tf/state_bucket/main.tf." + rc=1 + fi + + # Checked again immediately before the destroy, which deletes every + # object, in case anything was provisioned since the first check. + # The protection change is planned first and exactly that plan applied, + # so nothing but an in-place update of the bucket is ever applied. + if [[ ${rc} -eq 0 ]]; then + started=1 + if ! terraform -chdir="${tmp}" init -input=false || + ! terraform -chdir="${tmp}" plan -input=false \ + -target=google_storage_bucket.tfstate -out=target.tfplan || + ! actions=$(terraform -chdir="${tmp}" show -json target.tfplan | + python3 -c ' +import json, sys +for c in json.load(sys.stdin).get("resource_changes", []): + if c["address"] == "google_storage_bucket.tfstate": + print(",".join(c["change"]["actions"])) +'); then + rc=1 + elif [[ ${actions} != update && ${actions} != no-op ]]; then + echo " Error: Terraform would change gs://${state_bucket} (${actions:-not in state})," + echo " not only lift its protection, probably because a setting such" + echo " as the region differs from the bucket's. Nothing was deleted." + rc=1 + elif ! terraform -chdir="${tmp}" apply -input=false target.tfplan; then + rc=1 + elif ! check_state_bucket_empty "${state_bucket}"; then + # The bucket stays, so restore force_destroy = false from the + # tracked main.tf (prevent_destroy does not block an update). + cp tf/state_bucket/main.tf "${tmp}/main.tf" && + terraform -chdir="${tmp}" apply -input=false -auto-approve \ + -target=google_storage_bucket.tfstate || + echo " Warning: could not restore force_destroy = false; the next deploy.sh does." + rc=1 + elif ! terraform -chdir="${tmp}" destroy -input=false -auto-approve; then + rc=1 + fi + trap - INT TERM + if ! cp "${tmp}/terraform.tfstate" "${sb_dir}/terraform.tfstate"; then + echo " Error: could not copy the state back to ${sb_dir}; it is kept in" + echo " ${tmp}/terraform.tfstate" + return 1 + fi + fi + trap - INT TERM + rm -rf "${tmp:?}" + return ${rc} +} + # # tfdestroy() { @@ -105,6 +323,32 @@ tfdestroy() { # -- Exit on any errors set -e +# -- Options +# +assume_yes=0 +delete_bucket=0 +bucket_deployment="" +while [[ ${#} -gt 0 ]]; do + case "${1}" in + -y) + assume_yes=1 + ;; + --delete-state-bucket) + delete_bucket=1 + if [[ ${2} && ${2} != -* ]]; then + bucket_deployment=${2} + shift + fi + ;; + *) + echo " Error: Unknown option: ${1}" + echo " Usage: ${0} [-y] [--delete-state-bucket []]" + exit 1 + ;; + esac + shift +done + # -- Splash screen # cat <<'HEADER' @@ -135,6 +379,29 @@ fi # TODO: Check default authorisation has been set up # 'gcloud auth list' or similar could be used +# -- A state bucket named explicitly: only the bucket is touched, so a +# FrontEnd that may still be deployed from here under that name is refused +# rather than torn down. tf/'s deployment name is read leniently; if +# tf/terraform.tfvars is missing or names none, it is treated as possibly +# this one. +# +if [[ ${bucket_deployment} ]]; then + tf_name=$(sed -n 's/^[[:space:]]*deployment_name[[:space:]]*=[[:space:]]*"\([^"]*\)".*$/\1/p' \ + tf/terraform.tfvars 2>/dev/null | head -n 1) + if [[ ! -f tf/terraform.tfvars || -z ${tf_name} || ${tf_name} == "${bucket_deployment}" ]] && + ! front_end_gone; then + echo " Error: the FrontEnd '${bucket_deployment}' may still be deployed from here" + echo " (tf/.tkfe.lock exists, or tf/'s state shows it or cannot be read)," + echo " and --delete-state-bucket deletes only a bucket." + front_end_gone_help + exit 1 + fi + prepare_state_bucket_deletion "${bucket_deployment}" || exit 1 + delete_state_bucket || exit 1 + echo " The Terraform state bucket gs://${state_bucket} has been deleted." + exit 0 +fi + # -- Check terraform file for a TKFE deployment exits # @@ -151,7 +418,7 @@ fi if [[ ! -f tf/.tkfe.lock ]]; then echo " Warning: No lock file found" echo " It is likely there is no FrontEnd currently deployed" - if [[ "$1" == "-y" ]]; then + if [[ ${assume_yes} -eq 1 ]]; then echo " -y flag passed. Proceeding anyway." else read -r -p " Proceed anyway? [y/N]: " ready @@ -174,6 +441,21 @@ project=$(awk '$1 ~ /project_id/ {v=substr($3,2,length($3)-2); print v}' \ dname=$(awk '$1 ~ /deployment_name/ {v=substr($3,2,length($3)-2); print v}' \ ${tfvars}) +# -- Check before the confirmation prompts or any change, so a refusal leaves +# everything as it was: a FrontEnd that tf/ does not show must be provably +# gone, and the state bucket must be empty. +# +if [[ ${delete_bucket} -eq 1 ]]; then + if ! (cd tf && terraform show 2>/dev/null) | grep -q ghpcfe_id && ! front_end_gone; then + echo " Error: cannot confirm that no FrontEnd is deployed from here" + echo " (tf/.tkfe.lock exists, or tf/'s state cannot be read); nothing" + echo " has been changed." + front_end_gone_help + exit 1 + fi + prepare_state_bucket_deletion "${dname}" || exit 1 +fi + # -- The tf directory can exist, but TKFE deployment was aborted # If this happened, the service account may still need removing. # @@ -189,6 +471,19 @@ tname=$( # -- If no name returned from terraform, there's nothing running # +if [[ ! ${tname} && ${delete_bucket} -eq 1 ]]; then + if ! front_end_gone; then + echo " Error: cannot confirm that no FrontEnd is deployed from here" + echo " (tf/.tkfe.lock exists, or tf/'s state cannot be read), so the" + echo " state bucket is kept." + front_end_gone_help + exit 1 + fi + echo " No FrontEnd is deployed from here; deleting only its state bucket." + delete_state_bucket || exit 1 + echo " The Terraform state bucket gs://${state_bucket} has been deleted." + exit 0 +fi if [[ ! ${tname} ]]; then echo " Error: No terraform deployment found" echo "Exiting." @@ -198,7 +493,7 @@ fi echo " This will destroy the running FrontEnd: ${dname}" echo " Please ensure all resources deployed by the FrontEnd have been deleted." echo "" -if [[ "$1" == "-y" ]]; then +if [[ ${assume_yes} -eq 1 ]]; then echo " -y flag passed. Proceeding anyway." else read -r -p " Proceed? [y/N]: " ready @@ -231,9 +526,43 @@ cleanup_server_service_account "${project}" "${dname}" # rm -f tf/.tkfe.lock 2>/dev/null +# -- The Terraform state bucket is NOT destroyed here unless +# --delete-state-bucket was given and it was checked empty above. It holds +# the state of the clusters, filesystems, networks, images and workbenches +# this FrontEnd provisioned; destroying it would leave any that are still +# running impossible to remove with Terraform. +# +if [[ ${delete_bucket} -eq 1 ]]; then + echo "" + if ! delete_state_bucket; then + echo " Error: the FrontEnd was torn down, but deleting its state bucket" + echo " gs://${state_bucket} failed (see above). Rerun:" + echo " ./teardown.sh --delete-state-bucket ${dname}" + exit 1 + fi +else + state_bucket="" + sb_dir=$(state_bucket_dir "${dname}") + if [[ ${sb_dir} ]]; then + state_bucket=$(terraform -chdir="${sb_dir}" output -raw bucket_name 2>/dev/null || true) + fi +fi + echo "" echo " Completed teardown" echo "" +if [[ ${delete_bucket} -eq 1 ]]; then + echo " The Terraform state bucket gs://${state_bucket} has been deleted." + echo "" +elif [[ ${state_bucket} ]]; then + echo " The Terraform state bucket has been left in place:" + echo " gs://${state_bucket}" + echo " It is versioned and protected against accidental deletion. Once you" + echo " are certain every resource this FrontEnd provisioned is gone, remove" + echo " it with:" + echo " ./teardown.sh --delete-state-bucket ${dname}" + echo "" +fi wait exit 0 diff --git a/community/front-end/ofe/tf/README.md b/community/front-end/ofe/tf/README.md index eb30140e03..008bcb9330 100644 --- a/community/front-end/ofe/tf/README.md +++ b/community/front-end/ofe/tf/README.md @@ -69,6 +69,7 @@ limitations under the License. | [server\_instance\_type](#input\_server\_instance\_type) | Instance size to use from HPC Frontend webserver | `string` | `"n2d-standard-2"` | no | | [static\_ip](#input\_static\_ip) | Optional pre-configured static IP for HPC Frontend. | `string` | `""` | no | | [subnet](#input\_subnet) | Subnet in which to deploy HPC Frontend. | `string` | `""` | no | +| [terraform\_state\_bucket](#input\_terraform\_state\_bucket) | Name of the GCS bucket holding Terraform state for resources the front end
provisions, created by the separate tf/state\_bucket root and passed in by
deploy.sh. No default: silently falling back to local state would bring
back the risk this bucket exists to remove. | `string` | n/a | yes | | [webserver\_hostname](#input\_webserver\_hostname) | DNS Hostname for the webserver | `string` | `""` | no | | [zone](#input\_zone) | GCP Zone for HPC Frontend deployment. | `string` | n/a | yes | diff --git a/community/front-end/ofe/tf/main.tf b/community/front-end/ofe/tf/main.tf index 0e86c2627b..5cc4e18ac9 100644 --- a/community/front-end/ofe/tf/main.tf +++ b/community/front-end/ofe/tf/main.tf @@ -165,6 +165,7 @@ resource "google_compute_instance" "server_vm" { metadata = { startup-script-url = "${module.control_bucket.bucket.url}/webserver/startup.sh", webserver-config-bucket = module.control_bucket.bucket.name, + terraform-state-bucket = var.terraform_state_bucket, ghpcfe-c2-topic = module.pubsub.topic, hostname = var.webserver_hostname deploy_mode = var.deployment_mode diff --git a/community/front-end/ofe/tf/state_bucket/.gitignore b/community/front-end/ofe/tf/state_bucket/.gitignore new file mode 100644 index 0000000000..599f9c77f7 --- /dev/null +++ b/community/front-end/ofe/tf/state_bucket/.gitignore @@ -0,0 +1,3 @@ +# Per-deployment working directories created by deploy.sh. They hold each +# deployment's local state and settings, and links to the .tf files here. +/deployments/ diff --git a/community/front-end/ofe/tf/state_bucket/README.md b/community/front-end/ofe/tf/state_bucket/README.md new file mode 100644 index 0000000000..2db066338f --- /dev/null +++ b/community/front-end/ofe/tf/state_bucket/README.md @@ -0,0 +1,73 @@ +## Description + +Creates the versioned, deletion-protected Cloud Storage bucket that holds the +Terraform state of everything the Open Front End provisions; `deploy.sh` +applies it before the front end (see "Terraform State" in +`docs/admin_guide.md`). + +The Toolkit's `modules/file-system/cloud-storage-bucket` is not used because it +cannot carry `prevent_destroy` (Terraform allows no `lifecycle` block on a +module call), grants `roles/storage.objectUser` only per member rather than as an +authoritative binding for the writer list, and always creates an authoritative +`roles/storage.objectViewer` binding. + +## License + + +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +| ---- | ------- | +| [terraform](#requirement\_terraform) | >= 1.12.2 | +| [google](#requirement\_google) | >= 6.4.0, < 9.0.0 | + +## Providers + +| Name | Version | +| ---- | ------- | +| [google](#provider\_google) | >= 6.4.0, < 9.0.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +| ---- | ---- | +| [google_storage_bucket.tfstate](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket) | resource | +| [google_storage_bucket_iam_binding.state_writers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam_binding) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +| ---- | ----------- | ---- | ------- | :------: | +| [default\_writers](#input\_default\_writers) | Writers managed by deploy.sh: its provisioning service account, when that account exists. Regenerated on every deploy; do not set by hand. | `list(string)` | `[]` | no | +| [deployment\_name](#input\_deployment\_name) | Base "name" for the deployment. | `string` | n/a | yes | +| [extra\_labels](#input\_extra\_labels) | Extra labels to apply to created GCP resources. | `map(any)` | `{}` | no | +| [noncurrent\_version\_retention\_days](#input\_noncurrent\_version\_retention\_days) | Days to retain non-current (superseded) state file versions before deletion. | `number` | `90` | no | +| [project\_id](#input\_project\_id) | GCP Project in which to create the Terraform state bucket. | `string` | n/a | yes | +| [region](#input\_region) | GCP Region (bucket location) for the Terraform state bucket. | `string` | n/a | yes | +| [state\_bucket\_writers](#input\_state\_bucket\_writers) | IAM members, in full "serviceAccount:name@project.iam.gserviceaccount.com"
form, granted roles/storage.objectUser on the state bucket: the identities
that run Terraform for the front end. Set in admin.auto.tfvars, which
deploy.sh creates once and never overwrites. Add any provisioning
credential from another project here, or it cannot read or write its
resources' state. | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +| ---- | ----------- | +| [bucket\_name](#output\_bucket\_name) | Name of the Terraform state bucket. deploy.sh passes this to the ../ root as terraform\_state\_bucket. | +| [bucket\_url](#output\_bucket\_url) | gs:// URL of the Terraform state bucket. | + diff --git a/community/front-end/ofe/tf/state_bucket/main.tf b/community/front-end/ofe/tf/state_bucket/main.tf new file mode 100644 index 0000000000..9fa304f291 --- /dev/null +++ b/community/front-end/ofe/tf/state_bucket/main.tf @@ -0,0 +1,115 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# Terraform state bucket for resources provisioned by the Open Front End. +# +# A SEPARATE root from ../, applied before it and never referenced by it. +# teardown.sh removes it only with --delete-state-bucket, so tearing the front +# end down cannot destroy the state of the resources it provisioned. A resource +# whose state is gone cannot be destroyed by Terraform: it keeps running and +# keeps billing. +# +# This is NOT the control bucket (module "control_bucket" in ../main.tf), which +# stages bootstrap files and startup scripts and is force-destroy and +# unversioned so that teardown can remove it. + +locals { + default_labels = { + ghpcfe_id = var.deployment_name, + } + labels = merge(var.extra_labels, local.default_labels) + + # The name depends only on project and deployment name, so deploy.sh can + # compute it without this root's local state and adopt an existing bucket + # instead of creating a second one. Region is excluded: a bucket cannot + # change location. The hash of project and full deployment name keeps names + # distinct across projects and between names that clean to the same text. + # + # Bucket names allow 3-63 characters of a-z, 0-9, '-' and '_', starting and + # ending with a letter or digit ('.' needs domain verification). The + # deployment name is lowercased, other characters become '-', it is cut to + # fit, and leading/trailing '-' and '_' are dropped ("ofe" if nothing is + # left). A name that is already valid and at most 46 characters is used + # unchanged; keep it that way, or existing deployments get a new bucket. + bucket_suffix = "-tfstate-${substr(sha256("${var.project_id}/${var.deployment_name}"), 0, 8)}" + bucket_name_clean = replace(substr(replace(lower(var.deployment_name), "/[^a-z0-9_-]/", "-"), 0, 63 - length(local.bucket_suffix)), "/^[-_]+|[-_]+$/", "") + bucket_name_prefix = local.bucket_name_clean == "" ? "ofe" : local.bucket_name_clean + bucket_name = "${local.bucket_name_prefix}${local.bucket_suffix}" + + writers = distinct(concat(var.state_bucket_writers, var.default_writers)) +} + +resource "google_storage_bucket" "tfstate" { + name = local.bucket_name + project = var.project_id + location = var.region + + storage_class = "STANDARD" + + # The reason for a dedicated bucket: a state file truncated by a crash + # mid-write can be rolled back to a prior generation. + versioning { + enabled = true + } + + # A routine teardown must never be able to empty this bucket. + force_destroy = false + + # State contains full resource detail and may contain generated secrets. + uniform_bucket_level_access = true + public_access_prevention = "enforced" + + # Bounds versioned storage. Retention is deliberately generous, to allow + # recovery from a mistake noticed late. + lifecycle_rule { + condition { + days_since_noncurrent_time = var.noncurrent_version_retention_days + with_state = "ARCHIVED" + } + action { + type = "Delete" + } + } + + labels = local.labels + + # Removing this bucket must be deliberate, never a side effect. + # teardown.sh --delete-state-bucket lifts this and force_destroy in a + # temporary copy, once every state in the bucket is empty; it edits these + # exact lines with sed, so keep their form. + lifecycle { + prevent_destroy = true + } +} + +# Grants access to the identity that runs Terraform: NOT the front end's own +# service account, but the one whose key is registered in the portal as a +# Credential (passed per resource as GOOGLE_APPLICATION_CREDENTIALS). It may +# belong to another project (docs/AdvancedAdmin.md), so it must be listed +# explicitly: deploy.sh supplies the provisioning service account it creates +# (default_writers); administrators add others to state_bucket_writers in +# admin.auto.tfvars. +# +# Authoritative, so the listed members are exactly the principals holding the +# role. objectUser on this bucket alone covers Terraform's init, plan, apply, +# destroy and state locking; no project-level role is needed. +resource "google_storage_bucket_iam_binding" "state_writers" { + count = length(local.writers) > 0 ? 1 : 0 + + bucket = google_storage_bucket.tfstate.name + role = "roles/storage.objectUser" + members = local.writers +} diff --git a/community/front-end/ofe/tf/state_bucket/outputs.tf b/community/front-end/ofe/tf/state_bucket/outputs.tf new file mode 100644 index 0000000000..74d72e1a1a --- /dev/null +++ b/community/front-end/ofe/tf/state_bucket/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "bucket_name" { + description = "Name of the Terraform state bucket. deploy.sh passes this to the ../ root as terraform_state_bucket." + value = google_storage_bucket.tfstate.name +} + +output "bucket_url" { + description = "gs:// URL of the Terraform state bucket." + value = google_storage_bucket.tfstate.url +} diff --git a/community/front-end/ofe/tf/state_bucket/provider.tf b/community/front-end/ofe/tf/state_bucket/provider.tf new file mode 100644 index 0000000000..774e1bab33 --- /dev/null +++ b/community/front-end/ofe/tf/state_bucket/provider.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google" { + project = var.project_id + region = var.region +} diff --git a/community/front-end/ofe/tf/state_bucket/variables.tf b/community/front-end/ofe/tf/state_bucket/variables.tf new file mode 100644 index 0000000000..8c436e200e --- /dev/null +++ b/community/front-end/ofe/tf/state_bucket/variables.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "GCP Project in which to create the Terraform state bucket." +} + +variable "region" { + type = string + description = "GCP Region (bucket location) for the Terraform state bucket." +} + +variable "deployment_name" { + description = "Base \"name\" for the deployment." + type = string +} + +variable "state_bucket_writers" { + type = list(string) + default = [] + description = <<-EOT + IAM members, in full "serviceAccount:name@project.iam.gserviceaccount.com" + form, granted roles/storage.objectUser on the state bucket: the identities + that run Terraform for the front end. Set in admin.auto.tfvars, which + deploy.sh creates once and never overwrites. Add any provisioning + credential from another project here, or it cannot read or write its + resources' state. + EOT +} + +variable "default_writers" { + type = list(string) + default = [] + description = "Writers managed by deploy.sh: its provisioning service account, when that account exists. Regenerated on every deploy; do not set by hand." +} + +variable "noncurrent_version_retention_days" { + type = number + default = 90 + description = "Days to retain non-current (superseded) state file versions before deletion." +} + +variable "extra_labels" { + type = map(any) + default = {} + description = "Extra labels to apply to created GCP resources." +} diff --git a/community/front-end/ofe/tf/state_bucket/versions.tf b/community/front-end/ofe/tf/state_bucket/versions.tf new file mode 100644 index 0000000000..736b301e96 --- /dev/null +++ b/community/front-end/ofe/tf/state_bucket/versions.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.12.2" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.4.0, < 9.0.0" + } + } +} diff --git a/community/front-end/ofe/tf/variables.tf b/community/front-end/ofe/tf/variables.tf index b40bf6228e..09bd1ceb39 100644 --- a/community/front-end/ofe/tf/variables.tf +++ b/community/front-end/ofe/tf/variables.tf @@ -109,3 +109,17 @@ variable "extra_labels" { default = {} description = "Extra labels to apply to created GCP resources." } + +variable "terraform_state_bucket" { + type = string + description = <<-EOT + Name of the GCS bucket holding Terraform state for resources the front end + provisions, created by the separate tf/state_bucket root and passed in by + deploy.sh. No default: silently falling back to local state would bring + back the risk this bucket exists to remove. + EOT + validation { + condition = length(trimspace(var.terraform_state_bucket)) > 0 + error_message = "The variable 'terraform_state_bucket' must not be empty. Apply tf/state_bucket first and pass its bucket_name output." + } +} diff --git a/community/front-end/ofe/website/ghpcfe/cluster_manager/clusterinfo.py b/community/front-end/ofe/website/ghpcfe/cluster_manager/clusterinfo.py index 500662c520..84d8db8ac8 100644 --- a/community/front-end/ofe/website/ghpcfe/cluster_manager/clusterinfo.py +++ b/community/front-end/ofe/website/ghpcfe/cluster_manager/clusterinfo.py @@ -95,15 +95,17 @@ def update(self): self._prepare_bootstrap_gcs() def start_cluster(self, credentials): - self.cluster.cloud_state = "nm" + # A Retry of a cluster whose earlier apply succeeded must not forget + # that its resources exist. + if self.cluster.cloud_state != "m": + self.cluster.cloud_state = "nm" self.cluster.status = "c" self.cluster.save() - self._create_cluster_dir() - self._set_credentials(credentials) - self.update() - try: + self._create_cluster_dir() + self._set_credentials(credentials) + self.update() self._run_ghpc() self._initialize_terraform() self._apply_terraform() @@ -113,15 +115,17 @@ def start_cluster(self, credentials): self.cluster.save() # Not a lot we can do if terraform fails, it's on the admin user to - # investigate and fix the errors shown in the log + # investigate and fix the errors shown in the log. + # cloud_state is left alone: it becomes "m" as soon as the apply + # succeeds, so a later failure keeps the cluster managed. except Exception: # pylint: disable=broad-except self.cluster.status = "e" - self.cluster.cloud_state = "nm" self.cluster.save() raise def reconfigure_cluster(self): try: + self.update() self._run_ghpc() self._initialize_terraform() self._apply_terraform() @@ -130,10 +134,11 @@ def reconfigure_cluster(self): self.cluster.save() # Not a lot we can do if terraform fails, it's on the admin user to - # investigate and fix the errors shown in the log + # investigate and fix the errors shown in the log. + # The cluster was running before this reconfigure and still exists, + # so it stays managed. except Exception: # pylint: disable=broad-except self.cluster.status = "e" - self.cluster.cloud_state = "nm" self.cluster.save() raise @@ -333,6 +338,10 @@ def _prepare_ghpc_yaml(self): "login_uses": self._yaml_refs_to_uses(filesystems_refs, indent_level=2), "controller_sa": "sa", "startup_bucket": self.config["server"]["gcs_bucket"], + "state_bucket": utils.get_terraform_state_bucket(), + "state_prefix": utils.terraform_state_prefix( + "cluster", self.cluster.id + ), } rendered_yaml = template.render(context) @@ -357,6 +366,7 @@ def _prepare_ghpc_yaml(self): except Exception as e: logger.exception(f"Exception happened creating blueprint for cluster {self.cluster.name} - {e}") + raise def _prepare_bootstrap_gcs(self): template_dir = ( @@ -397,7 +407,7 @@ def _initialize_terraform(self): } try: logger.info("Invoking Terraform Init") - utils.run_terraform(terraform_dir, "init") + utils.run_terraform(terraform_dir, "init", extra_env=extra_env) utils.run_terraform(terraform_dir, "validate", extra_env=extra_env) logger.info("Invoking Terraform Plan") utils.run_terraform(terraform_dir, "plan", extra_env=extra_env) @@ -658,88 +668,89 @@ def _apply_terraform(self): logger.info("Invoking Terraform Apply") utils.run_terraform(terraform_dir, "apply", extra_env=extra_env) - # Look for Management and Login Nodes in TF state file - tf_state_file = terraform_dir / "terraform.tfstate" - with tf_state_file.open("r") as statefp: - state = json.load(statefp) - - # Extract and save Artifact Registry repository and secret information (if any) - self.extract_and_update_registry_info(state) - - # Apply Perms to the service accounts - try: - service_accounts = self._get_service_accounts(state) - self._apply_service_account_permissions(service_accounts) - except Exception as e: - # Be nicer to the user and continue creating cluster - logger.warning(f"An error occurred while applying permissions to service accounts: {e}") - - - # Cluster is now being initialized - self.cluster.internal_name = self.cluster.name - self.cluster.cloud_state = "m" - - # Cluster initialization is now running. Stamp the start time - # so a bootstrap that stalls (controller never reports ready) - # can be surfaced instead of spinning on "initialising". - self.cluster.status = "i" - self.cluster.initialization_started = timezone.now() - self.cluster.save() - - # Filters for Management Nodes (Controller) - mgmt_filters = { - "module": "module.slurm_controller", - "type": "google_compute_instance_from_template", - "name": "controller", - } - mgmt_nodes = self._create_model_instances_from_tf_state( - state, - mgmt_filters, + # The apply succeeded, so the cluster's resources exist. Record + # that before anything below can fail (reading the state back, + # registry or node bookkeeping), so the cluster stays managed. + self.cluster.internal_name = self.cluster.name + self.cluster.cloud_state = "m" + self.cluster.save() + + # Look for Management and Login Nodes in the Terraform state + state = utils.get_terraform_state(terraform_dir, extra_env) + + # Extract and save Artifact Registry repository and secret information (if any) + self.extract_and_update_registry_info(state) + + # Apply Perms to the service accounts + try: + service_accounts = self._get_service_accounts(state) + self._apply_service_account_permissions(service_accounts) + except Exception as e: + # Be nicer to the user and continue creating cluster + logger.warning(f"An error occurred while applying permissions to service accounts: {e}") + + + # Cluster initialization is now running. Stamp the start time + # so a bootstrap that stalls (controller never reports ready) + # can be surfaced instead of spinning on "initialising". + self.cluster.status = "i" + self.cluster.initialization_started = timezone.now() + self.cluster.save() + + # Filters for Management Nodes (Controller) + mgmt_filters = { + "module": "module.slurm_controller", + "type": "google_compute_instance_from_template", + "name": "controller", + } + mgmt_nodes = self._create_model_instances_from_tf_state( + state, + mgmt_filters, + ) + if len(mgmt_nodes) != 1: + logger.warning( + "Found %d controller nodes, there should be only 1", + len(mgmt_nodes), + ) + if len(mgmt_nodes): + node = mgmt_nodes[0] + node.save() + self.cluster.controller_node = node + logger.info( + "Created cluster controller node with IP address %s", + node.public_ip if node.public_ip else node.internal_ip, ) - if len(mgmt_nodes) != 1: - logger.warning( - "Found %d controller nodes, there should be only 1", - len(mgmt_nodes), - ) - if len(mgmt_nodes): - node = mgmt_nodes[0] - node.save() - self.cluster.controller_node = node - logger.info( - "Created cluster controller node with IP address %s", - node.public_ip if node.public_ip else node.internal_ip, - ) - # Filters for Login Nodes - login_filters = { - "module": 'module.slurm_controller.module.login["slurm-login"].module.instance', - "type": "google_compute_instance_from_template", - "name": "slurm_instance", - } - login_nodes = self._create_model_instances_from_tf_state( - state, - login_filters, + # Filters for Login Nodes + login_filters = { + "module": 'module.slurm_controller.module.login["slurm-login"].module.instance', + "type": "google_compute_instance_from_template", + "name": "slurm_instance", + } + login_nodes = self._create_model_instances_from_tf_state( + state, + login_filters, + ) + if len(login_nodes) != self.cluster.num_login_nodes: + logger.warning( + "Found %d login nodes, expected %d from config", + len(login_nodes), + self.cluster.num_login_nodes, + ) + for lnode in login_nodes: + lnode.cluster_login = self.cluster + lnode.save() + logger.info( + "Created login node with IP address %s", + lnode.public_ip + if lnode.public_ip + else lnode.internal_ip, ) - if len(login_nodes) != self.cluster.num_login_nodes: - logger.warning( - "Found %d login nodes, expected %d from config", - len(login_nodes), - self.cluster.num_login_nodes, - ) - for lnode in login_nodes: - lnode.cluster_login = self.cluster - lnode.save() - logger.info( - "Created login node with IP address %s", - lnode.public_ip - if lnode.public_ip - else lnode.internal_ip, - ) - # Set up Spack Install location - self._configure_spack_install_loc() + # Set up Spack Install location + self._configure_spack_install_loc() - self.cluster.save() + self.cluster.save() except subprocess.CalledProcessError as err: # We can error during provisioning, in which case Terraform diff --git a/community/front-end/ofe/website/ghpcfe/cluster_manager/filesystem.py b/community/front-end/ofe/website/ghpcfe/cluster_manager/filesystem.py index e4624cf272..d52e42d066 100644 --- a/community/front-end/ofe/website/ghpcfe/cluster_manager/filesystem.py +++ b/community/front-end/ofe/website/ghpcfe/cluster_manager/filesystem.py @@ -17,6 +17,7 @@ import json import logging +import os import subprocess from pathlib import Path @@ -40,6 +41,12 @@ def write_filestore_yaml(fs: GCPFilestoreFilesystem, target_dir: Path) -> None: f""" blueprint_name: {fs.name} +terraform_backend_defaults: + type: gcs + configuration: + bucket: {utils.get_terraform_state_bucket()} + prefix: {utils.terraform_state_prefix("filesystem", fs.id)} + vars: project_id: {project_id} deployment_name: {fs.name} @@ -92,6 +99,11 @@ def create_filesystem(fs: Filesystem) -> None: def _run_ghpc(target_dir: Path, cred_env: dict) -> None: ghpc_path = "/opt/gcluster/cluster-toolkit/ghpc" + # env= replaces the whole environment, so add the credential to the real + # one, as clusterinfo.py's _run_ghpc does; without HOME, ghpc fails. + env = os.environ.copy() + env.update(cred_env) + try: logger.info("Invoking ghpc create") log_out_fn = target_dir / "ghpc_create_log.stdout" @@ -104,7 +116,7 @@ def _run_ghpc(target_dir: Path, cred_env: dict) -> None: stdout=log_out, stderr=log_err, check=True, - env=cred_env + env=env, ) except subprocess.CalledProcessError as cpe: logger.error("ghpc exec failed", exc_info=cpe) @@ -123,7 +135,7 @@ def start_filesystem(fs: Filesystem) -> None: _run_ghpc(_base_dir_for_fs(fs), extra_env) target_dir = _tf_dir_for_fs(fs) - utils.run_terraform(target_dir, "init") + utils.run_terraform(target_dir, "init", extra_env=extra_env) utils.run_terraform(target_dir, "plan", extra_env=extra_env) logger.info("Invoking terraform apply for fs %s:%s", fs.id, fs.name) @@ -135,7 +147,7 @@ def start_filesystem(fs: Filesystem) -> None: fs.name, ) (out_fn, _) = utils.run_terraform( - target_dir, "output", arguments=["-json"] + target_dir, "output", arguments=["-json"], extra_env=extra_env ) with out_fn.open("r") as outputfp: results = json.load(outputfp) diff --git a/community/front-end/ofe/website/ghpcfe/cluster_manager/image.py b/community/front-end/ofe/website/ghpcfe/cluster_manager/image.py index 311e38b372..15c81a4d10 100644 --- a/community/front-end/ofe/website/ghpcfe/cluster_manager/image.py +++ b/community/front-end/ofe/website/ghpcfe/cluster_manager/image.py @@ -19,6 +19,7 @@ import logging from . import utils +from . import image_startup import json import subprocess import os @@ -112,14 +113,25 @@ def _create_blueprint(self): runners = "" for script in scripts: script_path = os.path.join(settings.MEDIA_ROOT, script.content.name) - runners+=f""" + runners+=f""" - type: {script.type} destination: {script.name} source: {script_path}""" + if not runners: + # The startup-script module rejects null runners, so render + # an explicit empty list for a no-op builder script. + runners = " []" with blueprint_file.open("w") as f: f.write( f"""blueprint_name: {self.blueprint_name} + +terraform_backend_defaults: + type: gcs + configuration: + bucket: {utils.get_terraform_state_bucket()} + prefix: {utils.terraform_state_prefix("image", self.image.id)} + vars: project_id: {project_id} deployment_name: {self.blueprint_name} @@ -150,6 +162,7 @@ def _create_blueprint(self): source: modules/packer/custom-image kind: packer use: + - network1 - scripts_for_image settings: source_image_project_id: [{self.image.source_image_project}] @@ -158,7 +171,6 @@ def _create_blueprint(self): image_family: $(vars.image_family) state_timeout: 30m zone: $(vars.zone) - subnetwork_name: $(vars.subnetwork_name) image_storage_locations: ["{self.image.cloud_region}"] metadata: enable-oslogin: {self.image.enable_os_login} @@ -168,6 +180,7 @@ def _create_blueprint(self): except Exception as e: self.update_image_status("e") logger.error(f"Error occurred while creating blueprint: {e}") + raise def _run_ghpc(self): target_dir = self.image_dir @@ -182,7 +195,8 @@ def _run_ghpc(self): with log_out_fn.open("wb") as log_out: with log_err_fn.open("wb") as log_err: subprocess.run( - [self.ghpc_path, "create", "image.yaml"], + [self.ghpc_path, "create", "image.yaml", "-w", + "--validation-level", "WARNING"], cwd=target_dir, stdout=log_out, stderr=log_err, @@ -195,6 +209,46 @@ def _run_ghpc(self): # No logs from stdout/err - get dumped to files raise + def _run_ghpc_import_inputs(self): + """Bridge builder-env outputs into the packer group. + + Mirrors the deployment instructions ghpc generates for + cross-group `use`: export-outputs from the applied builder + group, then import-inputs into the packer group. + """ + target_dir = self.image_dir + log_out_fn = target_dir / "ghpc_import_log.stdout" + log_err_fn = target_dir / "ghpc_import_log.stderr" + env = os.environ.copy() + env["GOOGLE_APPLICATION_CREDENTIALS"] = self._get_credentials_file() + try: + with log_out_fn.open("ab") as log_out: + with log_err_fn.open("ab") as log_err: + subprocess.run( + [self.ghpc_path, "export-outputs", + f"{self.blueprint_name}/builder-env"], + cwd=target_dir, + stdout=log_out, + stderr=log_err, + check=True, + env=env, + ) + subprocess.run( + [self.ghpc_path, "import-inputs", + f"{self.blueprint_name}/packer-image"], + cwd=target_dir, + stdout=log_out, + stderr=log_err, + check=True, + env=env, + ) + except subprocess.CalledProcessError as cpe: + self.update_image_status("e") + logger.error( + f"gcluster export/import bridge failed for image " + f"{self.image.id}", exc_info=cpe) + raise + def _create_builder_env(self): """Setup builder environment on GCP.""" extra_env = { @@ -208,7 +262,7 @@ def _create_builder_env(self): except OSError as e: self.update_image_status("e") logger.error(f"Error occurred while constructing terraform_dir: {e}") - utils.run_terraform(terraform_dir, "init") + utils.run_terraform(terraform_dir, "init", extra_env=extra_env) utils.run_terraform(terraform_dir, "validate", extra_env=extra_env) logger.info("Invoking Terraform Plan for builder env.") utils.run_terraform(terraform_dir, "plan", extra_env=extra_env) @@ -220,7 +274,18 @@ def _create_builder_env(self): "-raw", "startup_script_scripts_for_image"], ) - utils.copy_file(f"{terraform_dir}/terraform_output_log.stdout",f"{packer_dir}/custom-image/startup_script.sh") + # Wrapped so that the build cannot hang on the Packer plugin's + # own completion report; see image_startup.py. + with open(f"{terraform_dir}/terraform_output_log.stdout", encoding="utf-8") as src: + startup_script = image_startup.wrap_startup_script(src.read()) + with open(f"{packer_dir}/custom-image/startup_script.sh", "w", encoding="utf-8") as dst: + dst.write(startup_script) + # The packer module uses the builder VPC (ghpc create rejects + # unused modules), so its subnetwork input arrives through the + # toolkit's cross-group bridge. startup_script arrives that way + # too, but the explicit startup_script_file -var below overrides it. + logger.info("Importing packer group inputs from builder env.") + self._run_ghpc_import_inputs() except subprocess.CalledProcessError as cpe: self.update_image_status("e") logger.error(f"Terraform exec failed for builder env, image: {self.image.id}", exc_info=cpe) diff --git a/community/front-end/ofe/website/ghpcfe/cluster_manager/image_startup.py b/community/front-end/ofe/website/ghpcfe/cluster_manager/image_startup.py new file mode 100644 index 0000000000..a800ef933a --- /dev/null +++ b/community/front-end/ofe/website/ghpcfe/cluster_manager/image_startup.py @@ -0,0 +1,70 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Startup script for the Packer builder VM of a custom image. + +Packer's googlecompute builder waits for its startup-script wrapper to set the +instance metadata key `startup-script-status` to `done` or `error`. In +packer-plugin-googlecompute v1.2.7 the wrapper names the instance with bash's +$HOSTNAME, which on some boots of the Slurm base images is still the FQDN; the +API rejects that name, the status is never set, and the build waits forever. + +wrap_startup_script() runs the image's startup script inside an outer script +that reports its result, success or failure, under the instance name from the +metadata server. + +Temporary: the upstream fix (hashicorp/packer-plugin-googlecompute#355) is +merged but not released. The `~> 1.2.5` constraint in +modules/packer/custom-image/versions.pkr.hcl picks up a fixed release, after +which this wrapper can be removed. + +Standard library only, so tests can load it without Django. +""" + +_DELIMITER = "OFE_IMAGE_STARTUP_SCRIPT_EOF" + +_TEMPLATE = """#!/bin/bash +# Runs the image's startup script, then sets startup-script-status for Packer +# under the instance's real name (see ghpcfe/cluster_manager/image_startup.py). +md() {{ + curl -sSf -H "Metadata-Flavor: Google" \\ + "http://metadata.google.internal/computeMetadata/v1/instance/$1" +}} + +script=$(mktemp) || exit 1 +cat >"$script" <<'{delimiter}' +{script} +{delimiter} +chmod +x "$script" +"$script" +rc=$? +rm -f "$script" + +if [ "$rc" -eq 0 ]; then status="done"; else status="error"; fi +if name=$(md name) && zone=$(md zone); then + gcloud compute instances add-metadata "$name" --zone "${{zone##*/}}" \\ + --metadata "startup-script-status=$status" || + echo "OFE: could not set startup-script-status=$status" >&2 +else + echo "OFE: could not read instance name or zone from metadata" >&2 +fi +exit "$rc" +""" + + +def wrap_startup_script(script): + """Return `script` wrapped so that its result reaches Packer.""" + if _DELIMITER in script.splitlines(): + raise ValueError(f"startup script contains the line {_DELIMITER}") + return _TEMPLATE.format(delimiter=_DELIMITER, script=script.rstrip("\n")) diff --git a/community/front-end/ofe/website/ghpcfe/cluster_manager/utils.py b/community/front-end/ofe/website/ghpcfe/cluster_manager/utils.py index 32bfa12a1b..08c9ae7a53 100644 --- a/community/front-end/ofe/website/ghpcfe/cluster_manager/utils.py +++ b/community/front-end/ofe/website/ghpcfe/cluster_manager/utils.py @@ -15,9 +15,15 @@ """Commonly used utility routines""" import copy +import datetime import json import logging import os +import posixpath +import re +import secrets +import shlex +import socket import subprocess from pathlib import Path import shutil @@ -65,6 +71,105 @@ def _pathify(var): return g_config +# Terraform state prefixes by resource kind, placed under this database's +# state namespace (see get_state_namespace). They mirror the on-disk working +# directory names (clusters/cluster_, fs/fs_, ...), so an operator can +# map a prefix to its working directory by inspection. +TERRAFORM_STATE_PREFIXES = { + "cluster": "clusters/cluster_{id}", + "filesystem": "fs/fs_{id}", + "vpc": "vpcs/vpc_{id}", + "workbench": "workbenches/workbench_{id}", + "image": "images/image_{id}", +} + + +def new_state_namespace(): + """Return a fresh state namespace: UTC creation date plus 8 random hex digits.""" + return ( + f"{datetime.datetime.now(datetime.timezone.utc):%Y%m%d}-" + f"{secrets.token_hex(4)}" + ) + + +def get_state_namespace(): + """Return this database's state namespace, creating it on first use. + + Primary keys are unique only within one database, but the state bucket + outlives it; the namespace stops a fresh database reusing an earlier + database's prefixes. + """ + # Imported here so this module stays importable without Django. + from ..models import StateNamespace # pylint: disable=import-outside-toplevel + + return StateNamespace.current() + + +def terraform_state_prefix(kind, resource_id, namespace=None): + """Return the state-bucket prefix for one provisioned resource. + + `kind` is a key of TERRAFORM_STATE_PREFIXES; `resource_id` is the model's + primary key. `namespace` defaults to this database's state namespace. + """ + try: + template = TERRAFORM_STATE_PREFIXES[kind] + except KeyError as err: + raise ValueError( + f"No Terraform state prefix defined for resource kind {kind!r}. " + f"Known kinds: {sorted(TERRAFORM_STATE_PREFIXES)}" + ) from err + if namespace is None: + namespace = get_state_namespace() + return f"{namespace}/{template.format(id=resource_id)}" + + +def get_terraform_state_bucket(): + """Return the GCS bucket holding Terraform state for provisioned resources. + + Set from the `terraform-state-bucket` instance metadata key by + infrastructure_files/gcs_bucket/webserver/startup.sh. Missing configuration + is fatal, not a fallback to local state: local state is lost with the + instance, and Terraform cannot destroy a resource whose state is lost. + """ + bucket = load_config()["server"].get("terraform_state_bucket") + if not bucket: + raise RuntimeError( + "No 'terraform_state_bucket' in the server configuration. " + "Terraform state must be stored in GCS; refusing to provision with " + "local state. Check that the deployment set the " + "'terraform-state-bucket' instance metadata key and that " + "configuration.yaml was regenerated." + ) + return bucket + + +def write_backend_tf(target_dir, prefix): + """Write a GCS backend configuration into a Terraform working directory. + + For the resources copied from infrastructure_files/ (networks, + workbenches); Cluster Toolkit deployments get their backend from + `terraform_backend_defaults` in the blueprint. + + A file, unlike `-backend-config` at init, is found by every later + terraform invocation in the directory, including ones that re-enter a + directory created earlier (see vpc.create_subnet). Idempotent: an existing + file is overwritten with the same content. + """ + target_dir = Path(target_dir) + backend_file = target_dir / "backend.tf" + backend_file.write_text( + "terraform {\n" + ' backend "gcs" {\n' + f' bucket = "{get_terraform_state_bucket()}"\n' + f' prefix = "{prefix}"\n' + " }\n" + "}\n", + encoding="utf-8", + ) + logger.info("Wrote GCS backend for prefix %s to %s", prefix, backend_file) + return backend_file + + def _parse_tfvars(filename): res = {} with open(filename, "r", encoding="utf-8") as fp: @@ -122,14 +227,14 @@ def load_cluster_info(args): f"{args.cluster_dir.as_posix()}" ) - tf_state_file = ( - args.cluster_dir / "terraform" / args.cloud / "terraform.tfstate" - ) - with tf_state_file.open("r") as statefp: - state = json.load(statefp) - args.cluster_ip = state["outputs"]["ManagementPublicIP"]["value"] - args.cluster_name = state["outputs"]["cluster_id"]["value"] - args.tf_state = state + # NOTE: this function has no caller, and the layout it expects + # (/terraform/) is not the one ClusterInfo produces. + # It reads state from the backend only because the local state file no + # longer exists; the layout mismatch is deliberately left alone. + state = get_terraform_state(args.cluster_dir / "terraform" / args.cloud) + args.cluster_ip = state["outputs"]["ManagementPublicIP"]["value"] + args.cluster_name = state["outputs"]["cluster_id"]["value"] + args.tf_state = state args.cluster_vars = _parse_tfvars( args.cluster_dir / "terraform" / args.cloud / "terraform.tfvars" @@ -191,12 +296,262 @@ def rsync_dir( raise +def _terraform_env(extra_env=None): + """Build the environment for a terraform invocation.""" + new_env = os.environ.copy() + # Don't have terraform try to reuse any existing SSH agent + # It has its own keys + if "SSH_AUTH_SOCK" in new_env: + del new_env["SSH_AUTH_SOCK"] + new_env.update(extra_env if extra_env else {}) + return new_env + + +def get_terraform_state(target_dir, extra_env=None): + """Return the Terraform state document for a working directory. + + Uses `terraform state pull`, because with a GCS backend there is no local + terraform.tfstate; the output is the same raw state document. + + This contacts the backend, so `extra_env` must carry + GOOGLE_APPLICATION_CREDENTIALS for the resource's provisioning credential. + + Not run through run_terraform(): it inserts -no-color directly after the + command word, which `terraform state pull` rejects. + """ + result = subprocess.run( + ["terraform", "state", "pull"], + cwd=target_dir, + env=_terraform_env(extra_env), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + try: + return json.loads(result.stdout) + except json.JSONDecodeError as err: + raise RuntimeError( + f"Could not parse Terraform state pulled from {target_dir}: {err}" + ) from err + + +# Terraform commands that take the state lock. With the GCS backend a second +# operation on one resource finds the lock held, and Terraform's default +# -lock-timeout of 0 makes it fail at once instead of waiting for the first. +TERRAFORM_LOCKING_COMMANDS = ("plan", "apply", "destroy", "import", "refresh") +# A cluster create or destroy can hold the lock for 20-40 minutes, so a +# shorter wait would still fail behind one. +TERRAFORM_LOCK_TIMEOUT = "45m" + + +class TerraformStateLockError(subprocess.CalledProcessError): + """A resource's Terraform state lock could not be taken. + + Raised before Terraform runs when the lock is provably stale, and after + Terraform gives up waiting for a lock. It is a CalledProcessError so that + callers handle it exactly as any other failed Terraform run; the message + is also carried in `stderr`, which those callers log. + """ + + def __init__(self, cmd, message, returncode=1): + super().__init__(returncode, cmd, stderr=message.encode("utf-8")) + self.message = message + + def __str__(self): + return self.message + + +def _gcs_backend_config(target_dir): + """Return (bucket, prefix) of an initialised directory's GCS backend. + + `terraform init` records the backend in .terraform/terraform.tfstate, + whether it came from a written backend.tf or from the main.tf the Cluster + Toolkit generates. Returns None if there is no such file or the backend + is not GCS. + """ + marker = Path(target_dir) / ".terraform" / "terraform.tfstate" + try: + backend = json.loads(marker.read_text()).get("backend") or {} + except FileNotFoundError: + return None + if backend.get("type") != "gcs": + return None + config = backend.get("config") or {} + return config["bucket"], config.get("prefix") or "" + + +def _gcs_client(credentials_file): + # Imported here so this module stays importable without the Cloud + # client libraries, which the Django-free tests rely on. + from google.cloud import storage # pylint: disable=import-outside-toplevel + + return storage.Client.from_service_account_json(credentials_file) + + +def _read_gcs_state_lock(bucket, prefix, credentials_file): + """Return the lock info of the default workspace's state, or None. + + The GCS backend uses the lock object's generation as the lock ID, not the + ID written inside the object, and force-unlock expects the generation, so + "ID" is replaced with it. + """ + name = posixpath.join(prefix.strip("/"), "default.tflock") + blob = _gcs_client(credentials_file).bucket(bucket).get_blob(name) + if blob is None: + return None + info = json.loads(blob.download_as_bytes()) + info["ID"] = str(blob.generation) + return info + + +def _terraform_running_in(target_dir, proc_root="/proc"): + """True if any terraform process on this machine runs in target_dir.""" + target = Path(target_dir).resolve() + for proc in Path(proc_root).iterdir(): + if not proc.name.isdigit(): + continue + try: + if (proc / "comm").read_text().strip() != "terraform": + continue + if (proc / "cwd").resolve(strict=True) == target: + return True + except PermissionError: + # Another user's terraform: its directory can't be read, so it + # may hold this lock. Count it as running; the operation then + # waits for the lock instead of calling it stale. + return True + except OSError: + # Exited since the listing + continue + return False + + +def _short_hostname(name): + return name.split(".")[0] + + +def _lock_age(created): + try: + then = datetime.datetime.fromisoformat(created) + except (TypeError, ValueError): + return "unknown" + if then.tzinfo is None: + then = then.replace(tzinfo=datetime.timezone.utc) + age = datetime.datetime.now(datetime.timezone.utc) - then + return str(datetime.timedelta(seconds=max(0, int(age.total_seconds())))) + + +def _state_lock_message(target_dir, info, credentials_file, reason): + lock_id = info.get("ID") or "" + credentials_file = credentials_file or "" + unlock = ( + f"cd {shlex.quote(str(target_dir))} && " + f"GOOGLE_APPLICATION_CREDENTIALS={shlex.quote(str(credentials_file))} " + f"terraform force-unlock -force {lock_id}" + ) + return "\n".join([ + f"Terraform state for {target_dir} is locked. {reason}", + f" Lock ID: {lock_id}", + f" Operation: {info.get('Operation', 'unknown')}", + f" Holder: {info.get('Who', 'unknown')}", + f" Age: {_lock_age(info.get('Created'))}" + f" (created {info.get('Created', 'unknown')})", + "Only if no Terraform operation is running for this resource, clear" + " the lock with:", + f" {unlock}", + "Never force-unlock while an operation is running. See \"A resource" + " reports a stale state lock\" in docs/admin_guide.md.", + ]) + + +def _check_stale_state_lock(target_dir, cmdline, credentials_file): + """Raise TerraformStateLockError if the state lock is provably stale. + + A lock goes stale only when Terraform is killed hard (server reboot, OOM, + SIGKILL) while holding it. It is provably stale when it was taken on this + machine and no terraform process here runs in this directory. A lock + taken elsewhere, a live process, or any failure of this check is left to + Terraform, which waits up to TERRAFORM_LOCK_TIMEOUT for it. + + This never removes the lock: removing a lock that is in use corrupts the + state it protects. + """ + if not credentials_file: + return + try: + backend = _gcs_backend_config(target_dir) + if backend is None: + return + info = _read_gcs_state_lock(*backend, credentials_file) + if info is None: + return + _, sep, host = str(info.get("Who", "")).rpartition("@") + if not sep or ( + _short_hostname(host) != _short_hostname(socket.gethostname()) + ): + return + if _terraform_running_in(target_dir): + return + # The holder may have finished and released the lock between the + # read and the process scan; only the same lock still present counts. + again = _read_gcs_state_lock(*backend, credentials_file) + if again is None or again.get("ID") != info.get("ID"): + return + except Exception as err: # pylint: disable=broad-except + # Diagnostic only: it must never block a legitimate operation. + logger.warning( + "Could not check the Terraform state lock for %s; running" + " Terraform anyway: %s", target_dir, err, + ) + return + + message = _state_lock_message( + target_dir, info, credentials_file, + "The lock was taken on this server and no Terraform process on this" + " server holds it, so it is stale: left by a Terraform run that was" + " killed.", + ) + logger.error(message) + raise TerraformStateLockError(cmdline, message) + + +_LOCK_INFO_LINE = re.compile( + r"^\s*(ID|Path|Operation|Who|Version|Created):\s*(.*?)\s*$", re.MULTILINE +) + + +def _raise_if_lock_timeout(err, target_dir, log_err_fn, credentials_file): + """Turn a run that timed out waiting for the state lock into a clear error.""" + try: + stderr = Path(log_err_fn).read_text(errors="replace") + except OSError: + return + if "Error acquiring the state lock" not in stderr: + return + info = {} + for key, value in _LOCK_INFO_LINE.findall(stderr): + info.setdefault(key, value) + message = _state_lock_message( + target_dir, info, credentials_file, + f"Terraform waited {TERRAFORM_LOCK_TIMEOUT} for the lock and gave up.", + ) + logger.error(message) + raise TerraformStateLockError(err.cmd, message, err.returncode) from err + + def run_terraform(target_dir, command, arguments=None, extra_env=None): arguments = arguments if arguments else [] extra_env = extra_env if extra_env else {} cmdline = ["terraform", command, "-no-color"] + # Never prompt: these run unattended, and a prompt (e.g. init asking to + # migrate state after a backend change) would block forever. With + # -input=false, Terraform fails with its message instead. + if command in ("init", "plan", "apply", "destroy", "import", "refresh"): + cmdline.append("-input=false") + if command in TERRAFORM_LOCKING_COMMANDS: + cmdline.append(f"-lock-timeout={TERRAFORM_LOCK_TIMEOUT}") cmdline.extend(arguments) if command in ["apply", "destroy"]: cmdline.append("-auto-approve") @@ -204,12 +559,12 @@ def run_terraform(target_dir, command, arguments=None, extra_env=None): log_out_fn = Path(target_dir) / f"terraform_{command}_log.stdout" log_err_fn = Path(target_dir) / f"terraform_{command}_log.stderr" - new_env = os.environ.copy() - # Don't have terraform try to reuse any existing SSH agent - # It has its own keys - if "SSH_AUTH_SOCK" in new_env: - del new_env["SSH_AUTH_SOCK"] - new_env.update(extra_env) + new_env = _terraform_env(extra_env) + + if command in TERRAFORM_LOCKING_COMMANDS: + _check_stale_state_lock( + target_dir, cmdline, new_env.get("GOOGLE_APPLICATION_CREDENTIALS") + ) # Some Cluster Toolkit modules (e.g. the Slurm controller's compute-node # cleanup) run `gcloud` directly from an apply/destroy-time local-exec @@ -250,14 +605,23 @@ def run_terraform(target_dir, command, arguments=None, extra_env=None): with log_out_fn.open("wb") as log_out: with log_err_fn.open("wb") as log_err: - subprocess.run( - cmdline, - cwd=target_dir, - env=new_env, - stdout=log_out, - stderr=log_err, - check=True, - ) + try: + subprocess.run( + cmdline, + cwd=target_dir, + env=new_env, + stdout=log_out, + stderr=log_err, + check=True, + ) + except subprocess.CalledProcessError as err: + if command in TERRAFORM_LOCKING_COMMANDS: + log_err.flush() + _raise_if_lock_timeout( + err, target_dir, log_err_fn, + new_env.get("GOOGLE_APPLICATION_CREDENTIALS"), + ) + raise return (log_out_fn, log_err_fn) diff --git a/community/front-end/ofe/website/ghpcfe/cluster_manager/validate_credential.py b/community/front-end/ofe/website/ghpcfe/cluster_manager/validate_credential.py index 2e83823a82..1c8511e906 100644 --- a/community/front-end/ofe/website/ghpcfe/cluster_manager/validate_credential.py +++ b/community/front-end/ofe/website/ghpcfe/cluster_manager/validate_credential.py @@ -19,10 +19,24 @@ import json import warnings +from google.api_core import exceptions as google_exceptions +from google.auth import exceptions as auth_exceptions +from google.cloud import storage from google.oauth2 import service_account +from . import utils + logger = logging.getLogger(__name__) +# What Terraform's GCS backend needs to read, write and lock a resource's +# state in the state bucket. +STATE_BUCKET_PERMISSIONS = ( + "storage.objects.create", + "storage.objects.delete", + "storage.objects.get", + "storage.objects.list", +) + def validate_credential(cloud_provider, credential_detail): """communicate with a cloud provider to validate a credential""" @@ -56,3 +70,60 @@ def _validate_credential_gcp(credential_detail): return False return True + + +def check_state_bucket_access(credential_detail): + """Return a warning if a GCP credential cannot use the state bucket. + + Asks the bucket (testIamPermissions) which of STATE_BUCKET_PERMISSIONS + the credential's service account holds: this writes nothing, needs no + permission and includes project-level roles. The front end never grants + access itself; the warning tells an administrator how. Returns None if + every permission is held. + """ + info = json.loads(credential_detail) + email = info.get("client_email") + server = utils.load_config()["server"] + bucket = server.get("terraform_state_bucket") + if not bucket: + return ( + f"Could not check whether service account {email} can use the " + "Terraform state bucket: the server configuration names no " + "terraform_state_bucket. Provisioning will fail until it does." + ) + + creds = service_account.Credentials.from_service_account_info(info) + client = storage.Client(project=info.get("project_id"), credentials=creds) + # One try with a short timeout: this runs inside the registration request. + try: + granted = client.bucket(bucket).test_iam_permissions( + list(STATE_BUCKET_PERMISSIONS), timeout=30, retry=None + ) + except google_exceptions.NotFound: + return ( + f"Could not check whether service account {email} can use the " + f"Terraform state bucket gs://{bucket}: the bucket was not found. " + "Provisioning will fail until it exists and this account can use it." + ) + except (google_exceptions.GoogleAPICallError, + auth_exceptions.GoogleAuthError) as err: + logger.warning("State bucket access check for %s failed: %s", email, err) + return ( + f"Could not check whether service account {email} can use the " + f"Terraform state bucket gs://{bucket}: {err}" + ) + + missing = sorted(set(STATE_BUCKET_PERMISSIONS) - set(granted)) + if not missing: + return None + deployment = server.get("deployment_name") or "" + sb_dir = f"tf/state_bucket/deployments/{deployment}" + return ( + f"Service account {email} cannot read and write the Terraform state " + f"bucket gs://{bucket} (missing {', '.join(missing)}), so provisioning " + "with this credential will fail until it can. On the machine TKFE was " + "deployed from, add this line to state_bucket_writers in " + f"{sb_dir}/admin.auto.tfvars: " + f'"serviceAccount:{email}", ' + f"then run: terraform -chdir={sb_dir} apply" + ) diff --git a/community/front-end/ofe/website/ghpcfe/cluster_manager/vpc.py b/community/front-end/ofe/website/ghpcfe/cluster_manager/vpc.py index 873343b93a..f156a9f5bd 100644 --- a/community/front-end/ofe/website/ghpcfe/cluster_manager/vpc.py +++ b/community/front-end/ofe/website/ghpcfe/cluster_manager/vpc.py @@ -35,6 +35,11 @@ def create_vpc(vpc: VirtualNetwork) -> None: target_dir.parent.mkdir(parents=True) shutil.copytree(_tf_source_dir_for_vpc("GCP"), target_dir) + # point Terraform at the shared state bucket before any init runs + utils.write_backend_tf( + target_dir, utils.terraform_state_prefix("vpc", vpc.id) + ) + # create credential file with open(target_dir / "cloud_credentials", "w", encoding="utf-8") as fp: fp.write(vpc.cloud_credential.detail) @@ -64,7 +69,7 @@ def create_vpc(vpc: VirtualNetwork) -> None: generate_vpc_tf_datablock(vpc, target_dir) try: - utils.run_terraform(target_dir, "init") + utils.run_terraform(target_dir, "init", extra_env=extra_env) utils.run_terraform(target_dir, "validate", extra_env=extra_env) utils.run_terraform(target_dir, "plan", extra_env=extra_env) @@ -80,36 +85,49 @@ def create_vpc(vpc: VirtualNetwork) -> None: def start_vpc(vpc: VirtualNetwork) -> None: """Effectively, just 'terraform apply'""" target_dir = _tf_dir_for_vpc(vpc.id) + extra_env = { + "GOOGLE_APPLICATION_CREDENTIALS": Path( + target_dir / "cloud_credentials" + ).as_posix() + } try: - extra_env = { - "GOOGLE_APPLICATION_CREDENTIALS": Path( - target_dir / "cloud_credentials" - ).as_posix() - } utils.run_terraform(target_dir, "apply", extra_env=extra_env) - tf_state_file = target_dir / "terraform.tfstate" - with tf_state_file.open("r") as statefp: - state = json.load(statefp) - if vpc.is_managed: - vpc.cloud_id = state["outputs"]["vpc_id"]["value"] - vpc.save() - for subnet in vpc.subnets.all(): - if subnet.is_managed: - subnet.cloud_id = state["outputs"][f"subnet-{subnet.id}"][ - "value" - ] - subnet.cloud_state = "m" - subnet.save() except subprocess.CalledProcessError as err: logger.error("Terraform apply failed", exc_info=err) if err.stdout: logger.info("TF stdout:\n%s\n", err.stdout.decode("utf-8")) if err.stderr: logger.info("TF stderr:\n%s\n", err.stderr.decode("utf-8")) - vpc.status = "e" + # A failed apply can leave some resources behind. "Unknown" keeps + # Edit, Apply Cloud Changes and Destroy available instead of leaving + # the network stuck in "Creating". + vpc.cloud_state = "um" vpc.save() raise + # The apply succeeded, so the network exists. Mark it managed before + # reading the state back, so a failure there does not leave it stuck in + # "Creating". + vpc.cloud_state = "m" + vpc.save() + try: + state = utils.get_terraform_state(target_dir, extra_env) + vpc.cloud_id = state["outputs"]["vpc_id"]["value"] + vpc.save() + for subnet in vpc.subnets.all(): + if subnet.is_managed: + subnet.cloud_id = state["outputs"][f"subnet-{subnet.id}"][ + "value" + ] + subnet.cloud_state = "m" + subnet.save() + except (subprocess.CalledProcessError, RuntimeError, KeyError) as err: + logger.error( + "Terraform apply succeeded but reading its state failed", + exc_info=err, + ) + raise + def destroy_vpc(vpc: VirtualNetwork) -> None: target_dir = _tf_dir_for_vpc(vpc.id) diff --git a/community/front-end/ofe/website/ghpcfe/cluster_manager/workbenchinfo.py b/community/front-end/ofe/website/ghpcfe/cluster_manager/workbenchinfo.py index c282ebb010..8bb4036a95 100644 --- a/community/front-end/ofe/website/ghpcfe/cluster_manager/workbenchinfo.py +++ b/community/front-end/ofe/website/ghpcfe/cluster_manager/workbenchinfo.py @@ -106,17 +106,30 @@ def copy_terraform(self): / self.cloud_dir, terraform_dir / self.cloud_dir, ) + # point Terraform at the shared state bucket before any init runs + utils.write_backend_tf( + terraform_dir / self.cloud_dir, + utils.terraform_state_prefix("workbench", self.workbench.id), + ) return terraform_dir def copy_startup_script(self): + # Runs as the Vertex AI Workbench Instances post-startup-script. The + # managed image owns JupyterLab and its systemd unit, so this only + # mounts attached filesystems into the notebook home and optionally + # installs the Slurm client. It writes no completion marker: the + # wait-for-startup module polls the instance state instead. user = self.workbench.trusted_user + social_account = user.socialaccount_set.first() + uid = social_account.uid if social_account else "" + home_base = "/home/jupyter" # pylint: disable=line-too-long startup_script_vars = f""" -USER=$(curl -s http://metadata.google.internal/computeMetadata/v1/oslogin/users?pagesize=1024 \ +OSLOGIN_USER=$(curl -s http://metadata.google.internal/computeMetadata/v1/oslogin/users?pagesize=1024 \ -H 'Metadata-Flavor: Google' | \ jq '.[][] | \ - select ( .name == "{user.socialaccount_set.first().uid}") | \ + select ( .name == "{uid}") | \ .posixAccounts | \ .[].username' 2>&- | tr -d '"') """ @@ -230,24 +243,17 @@ def copy_startup_script(self): f.write( f"""#!/bin/bash echo "starting startup script at `date`" | tee -a /tmp/startup.log -echo "Getting username..." | tee -a /tmp/startup.log {startup_script_vars} +echo "oslogin user resolved to: ${{OSLOGIN_USER}}" | tee -a /tmp/startup.log echo "Setting up storage" | tee -a /tmp/startup.log -sudo apt-get -y update && sudo apt-get install -y nfs-common +apt-get -y update && apt-get install -y nfs-common {slurm_config_segment} -mkdir /tmp/jupyterhome -chown $USER:$USER /tmp/jupyterhome - -mkdir /home/$USER -chown $USER:$USER /home/$USER +mkdir -p {home_base} -cp /home/jupyter/.jupyter /tmp/jupyterhome/.jupyter -R -chown $USER:$USER /tmp/jupyterhome/.jupyter -R - -cat << EOF > /tmp/jupyterhome/DATA_LOSS_WARNING.txt +cat << EOF > {home_base}/DATA_LOSS_WARNING.txt DATA LOSS WARNING: The data on this workbench instance is not automatically saved unless it is @@ -270,12 +276,13 @@ def copy_startup_script(self): self.workbench.id == mp.workbench.id and mp.export.filesystem.hostname_or_ip ): - f.write("mkdir -p " + mp.mount_path + "\n") - f.write( - "mkdir -p /tmp/jupyterhome`dirname " - + mp.mount_path - + "`\n" + # Symlink the mount into the notebook home so it is + # visible in the managed JupyterLab file browser. + link_name = ( + f"{home_base}/" + + mp.mount_path.strip("/").replace("/", "_") ) + f.write("mkdir -p " + mp.mount_path + "\n") f.write( "mount " + mp.export.filesystem.hostname_or_ip @@ -286,13 +293,7 @@ def copy_startup_script(self): + "\n" ) f.write("chmod 777 " + mp.mount_path + "\n") - f.write( - "ln -s " - + mp.mount_path - + " /tmp/jupyterhome`dirname " - + mp.mount_path - + "` \n" - ) + f.write(f"ln -sfn {mp.mount_path} {link_name}\n") f.write( 'echo "' + mp.export.filesystem.hostname_or_ip @@ -300,7 +301,9 @@ def copy_startup_script(self): + mp.export.export_name + " is mounted at " + mp.mount_path - + '" >> /tmp/jupyterhome/DATA_LOSS_WARNING.txt\n' + + '" >> ' + + home_base + + "/DATA_LOSS_WARNING.txt\n" ) logger.debug("Writing workbench startup script") @@ -331,15 +334,12 @@ def prepare_terraform_vars(self): region = "{region}" zone = "{zone}" project_name = "{project}" +network_name = "{self.workbench.subnet.vpc.cloud_id}" subnet_name = "{subnet}" machine_type = "{self.workbench.machine_type}" boot_disk_type = "{self.workbench.boot_disk_type}" boot_disk_size_gb = "{self.workbench.boot_disk_capacity}" trusted_user = "{self.workbench.trusted_user.email}" -instance_image = {{ - project = "deeplearning-platform-release" - family = "{self.workbench.image_family}" -}} owner_id = ["{self.workbench.trusted_user.email}"] wb_startup_script_name = "workbench/workbench_{self.workbench.id}_startup_script" @@ -370,7 +370,9 @@ def _terraform_init(self): try: self.workbench.cloud_state = "cm" self.workbench.save() - utils.run_terraform(terraform_dir / self.cloud_dir, "init") + utils.run_terraform( + terraform_dir / self.cloud_dir, "init", extra_env=extra_env + ) utils.run_terraform( terraform_dir / self.cloud_dir, "validate", extra_env=extra_env ) @@ -395,32 +397,6 @@ def _terraform_create(self): utils.run_terraform( terraform_dir / self.cloud_dir, "apply", extra_env=extra_env ) - # Look for Management Public IP in terraform.tfstate - tf_state_file = terraform_dir / self.cloud_dir / "terraform.tfstate" - with tf_state_file.open("r") as statefp: - outputs = json.load(statefp)["outputs"] - wb_name = "UNKNOWN" - try: - wb_name = outputs["notebook_instance_name"]["value"] - except KeyError: - logger.error( - "Failed to parse workbench instance name from TF state" - ) - try: - deployment_id = outputs["deployment_id"]["value"] - wb_name = f"notebooks-instance-{deployment_id}-0" - except KeyError: - logger.error( - "Failed to parse deployment ID from TF state" - ) - # workbench is now being initialized - self.workbench.internal_name = wb_name - self.workbench.cloud_state = "m" - self.workbench.status = "i" - - self.workbench.save() - # Ansible is now running... Probably 15-30 minutes or so - except subprocess.CalledProcessError as err: # We can error during provisioning, in which case Terraform # doesn't tear things down. Run a `destroy`, just in case @@ -437,6 +413,43 @@ def _terraform_create(self): self._terraform_destroy() raise + # The apply succeeded, so its resources exist. A failure reading the + # state back must not reach the destroy above. + try: + outputs = utils.get_terraform_state( + terraform_dir / self.cloud_dir, extra_env + )["outputs"] + except (subprocess.CalledProcessError, RuntimeError) as err: + logger.error( + "Terraform apply succeeded but reading its state failed", + exc_info=err, + ) + self.workbench.status = "e" + self.workbench.cloud_state = "m" + self.workbench.save() + raise + wb_name = "UNKNOWN" + try: + wb_name = outputs["notebook_instance_name"]["value"] + except KeyError: + logger.error( + "Failed to parse workbench instance name from TF state" + ) + try: + deployment_id = outputs["deployment_id"]["value"] + wb_name = f"notebooks-instance-{deployment_id}-0" + except KeyError: + logger.error( + "Failed to parse deployment ID from TF state" + ) + # workbench is now being initialized + self.workbench.internal_name = wb_name + self.workbench.cloud_state = "m" + self.workbench.status = "i" + + self.workbench.save() + # Ansible is now running... Probably 15-30 minutes or so + def _terraform_destroy(self): terraform_dir = self.workbench_dir / "terraform" @@ -490,16 +503,25 @@ def _get_proxy_uri(self): logger.info("TF stderr:\n%s\n", err.stderr.decode("utf-8")) raise - tf_state_file = terraform_dir / self.cloud_dir / "terraform.tfstate" - with tf_state_file.open("r") as statefp: - outputs = json.load(statefp)["outputs"] - try: - wb_uri = outputs["notebook_proxy_uri"]["value"] - except KeyError: - logger.error( - "Failed to get workbench uri from TF output" - ) - raise + try: + outputs = utils.get_terraform_state( + terraform_dir / self.cloud_dir, extra_env + )["outputs"] + except (subprocess.CalledProcessError, RuntimeError) as err: + # Not retried: the backend client already retries transient + # errors, so a failure here is not the "URI not ready yet" + # condition @retry(ValueError) exists for. + logger.error("Failed to read workbench state", exc_info=err) + self.workbench.status = "e" + self.workbench.save() + raise + try: + wb_uri = outputs["notebook_proxy_uri"]["value"] + except KeyError: + logger.error( + "Failed to get workbench uri from TF output" + ) + raise if not wb_uri: logger.info("Awaiting workbench uri update (got \"%s\")", wb_uri) diff --git a/community/front-end/ofe/website/ghpcfe/forms.py b/community/front-end/ofe/website/ghpcfe/forms.py index c5d31ffecd..3a2dd90214 100644 --- a/community/front-end/ofe/website/ghpcfe/forms.py +++ b/community/front-end/ofe/website/ghpcfe/forms.py @@ -557,7 +557,6 @@ class Meta: "machine_type", "boot_disk_type", "boot_disk_capacity", - "image_family", "attached_cluster", ) diff --git a/community/front-end/ofe/website/ghpcfe/management/commands/tfstate_namespace.py b/community/front-end/ofe/website/ghpcfe/management/commands/tfstate_namespace.py new file mode 100644 index 0000000000..d30ac20cdb --- /dev/null +++ b/community/front-end/ofe/website/ghpcfe/management/commands/tfstate_namespace.py @@ -0,0 +1,26 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Print this server's Terraform state namespace.""" + +from django.core.management.base import BaseCommand + +from ghpcfe.cluster_manager import utils + + +class Command(BaseCommand): + help = "Print the namespace this server's Terraform state is filed under." + + def handle(self, *args, **options): + self.stdout.write(utils.get_state_namespace()) diff --git a/community/front-end/ofe/website/ghpcfe/models.py b/community/front-end/ofe/website/ghpcfe/models.py index dde314dd1b..a0900c2c78 100644 --- a/community/front-end/ofe/website/ghpcfe/models.py +++ b/community/front-end/ofe/website/ghpcfe/models.py @@ -33,6 +33,8 @@ from rest_framework.authtoken.models import Token from django import template +from .cluster_manager.utils import new_state_namespace + register = template.Library() logger = logging.getLogger(__name__) @@ -1950,6 +1952,8 @@ class Workbench(CloudResource): ("r-latest-cpu-experimental-ubuntu-2004", "R (Experimental)"), ) # pylint: enable=line-too-long + # Unused: Workbench Instances use the default image, so this is neither + # shown in the UI nor passed to Terraform. Kept to avoid a schema migration. image_family = models.CharField( max_length=64, choices=WORKBENCH_IMAGEFAMILIES, @@ -2004,3 +2008,21 @@ def mount_source(self): def __str__(self): return f"{self.mount_path} on {self.workbench}" + + +class StateNamespace(models.Model): + """This database's namespace in the Terraform state bucket (one row). + + Created once per database so a fresh database cannot reuse an earlier + database's state prefixes. + """ + + namespace = models.CharField(max_length=32, unique=True, editable=False) + created = models.DateTimeField(auto_now_add=True) + + @classmethod + def current(cls): + obj, _ = cls.objects.get_or_create( + pk=1, defaults={"namespace": new_state_namespace()} + ) + return obj.namespace diff --git a/community/front-end/ofe/website/ghpcfe/templates/blueprint/cluster_config.yaml.j2 b/community/front-end/ofe/website/ghpcfe/templates/blueprint/cluster_config.yaml.j2 index 8fa71907e0..c8caf402a3 100644 --- a/community/front-end/ofe/website/ghpcfe/templates/blueprint/cluster_config.yaml.j2 +++ b/community/front-end/ofe/website/ghpcfe/templates/blueprint/cluster_config.yaml.j2 @@ -16,6 +16,12 @@ limitations under the License. blueprint_name: {{ cluster.cloud_id }} +terraform_backend_defaults: + type: gcs + configuration: + bucket: {{ state_bucket }} + prefix: {{ state_prefix }} + vars: project_id: {{ project_id }} deployment_name: {{ cluster.cloud_id }} diff --git a/community/front-end/ofe/website/ghpcfe/templates/workbench/create_form.html b/community/front-end/ofe/website/ghpcfe/templates/workbench/create_form.html index cd5277b065..d5ce8dd4a8 100644 --- a/community/front-end/ofe/website/ghpcfe/templates/workbench/create_form.html +++ b/community/front-end/ofe/website/ghpcfe/templates/workbench/create_form.html @@ -143,12 +143,6 @@

Create a new workbench

{{ form.boot_disk_capacity }} {{ form.boot_disk_capacity.help_text }} -
- {{ form.image_family.errors }} - {{ form.image_family.label_tag }} - {{ form.image_family }} - {{ form.image_family.help_text }} -
{{ form.attached_cluster.errors }} diff --git a/community/front-end/ofe/website/ghpcfe/templates/workbench/update.html b/community/front-end/ofe/website/ghpcfe/templates/workbench/update.html index 340135fc85..39ef2d2121 100644 --- a/community/front-end/ofe/website/ghpcfe/templates/workbench/update.html +++ b/community/front-end/ofe/website/ghpcfe/templates/workbench/update.html @@ -69,12 +69,6 @@

Workbench Detail

{{ form.boot_disk_capacity }} {{ form.boot_disk_capacity.help_text }}
-
- {{ form.image_family.errors }} - {{ form.image_family.label_tag }} - {{ form.image_family }} - {{ form.image_family.help_text }} -
{{ form.attached_cluster.errors }} {{ form.attached_cluster.label_tag }} diff --git a/community/front-end/ofe/website/ghpcfe/tests/__init__.py b/community/front-end/ofe/website/ghpcfe/tests/__init__.py new file mode 100644 index 0000000000..f8cb6d9d20 --- /dev/null +++ b/community/front-end/ofe/website/ghpcfe/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/community/front-end/ofe/website/ghpcfe/tests/test_credential_state_bucket.py b/community/front-end/ofe/website/ghpcfe/tests/test_credential_state_bucket.py new file mode 100644 index 0000000000..87926224ad --- /dev/null +++ b/community/front-end/ofe/website/ghpcfe/tests/test_credential_state_bucket.py @@ -0,0 +1,203 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the state bucket check made when a credential is registered. + +A provisioning credential whose service account cannot use the Terraform +state bucket fails only later, when Terraform first reads or writes state. +check_state_bucket_access() asks the bucket which permissions that account +holds and returns a warning saying how to grant them. + +validate_credential.py is loaded by path into a throwaway package, with the +Google client libraries and `utils` stubbed in sys.modules, so these tests +need no Django settings, credentials or network. +""" + +import importlib.util +import json +import sys +import types +import unittest +from pathlib import Path +from unittest import mock + +CLUSTER_MANAGER = Path(__file__).resolve().parents[1] / "cluster_manager" + +EMAIL = "prov@other-project.iam.gserviceaccount.com" +DETAIL = json.dumps( + {"type": "service_account", "project_id": "other-project", "client_email": EMAIL} +) + + +# Stand-ins for google.api_core.exceptions, raised by the tests' fake bucket. +class NotFound(Exception): + pass + + +class GoogleAPICallError(Exception): + pass + + +class GoogleAuthError(Exception): + pass + + +def _load_validate_credential(): + google = types.ModuleType("google") + google.__path__ = [] + cloud = types.ModuleType("google.cloud") + cloud.__path__ = [] + storage = types.ModuleType("google.cloud.storage") + storage.Client = mock.MagicMock() + cloud.storage = storage + api_core = types.ModuleType("google.api_core") + api_core.__path__ = [] + exceptions = types.ModuleType("google.api_core.exceptions") + exceptions.NotFound = NotFound + exceptions.GoogleAPICallError = GoogleAPICallError + api_core.exceptions = exceptions + auth = types.ModuleType("google.auth") + auth.__path__ = [] + auth_exceptions = types.ModuleType("google.auth.exceptions") + auth_exceptions.GoogleAuthError = GoogleAuthError + auth.exceptions = auth_exceptions + oauth2 = types.ModuleType("google.oauth2") + oauth2.__path__ = [] + service_account = types.ModuleType("google.oauth2.service_account") + service_account.Credentials = mock.MagicMock() + oauth2.service_account = service_account + pkg = types.ModuleType("ofe_cred_stub") + pkg.__path__ = [] + utils = types.ModuleType("ofe_cred_stub.utils") + utils.load_config = mock.MagicMock() + pkg.utils = utils + stubs = { + "google": google, + "google.cloud": cloud, + "google.cloud.storage": storage, + "google.api_core": api_core, + "google.api_core.exceptions": exceptions, + "google.auth": auth, + "google.auth.exceptions": auth_exceptions, + "google.oauth2": oauth2, + "google.oauth2.service_account": service_account, + "ofe_cred_stub": pkg, + "ofe_cred_stub.utils": utils, + } + with mock.patch.dict(sys.modules, stubs): + spec = importlib.util.spec_from_file_location( + "ofe_cred_stub.validate_credential", CLUSTER_MANAGER / "validate_credential.py" + ) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +validate_credential = _load_validate_credential() + + +class CheckStateBucketAccessTests(unittest.TestCase): + """Each case is answered with a warning, or None when nothing is missing.""" + + BUCKET = "dep1-tfstate-abcd1234" + + def _check(self, granted=None, error=None, server=None): + if server is None: + server = {"terraform_state_bucket": self.BUCKET, "deployment_name": "dep1"} + bucket = mock.MagicMock() + if error is not None: + bucket.test_iam_permissions.side_effect = error + else: + bucket.test_iam_permissions.return_value = granted + client = mock.MagicMock() + client.bucket.return_value = bucket + with mock.patch.object( + validate_credential.utils, "load_config", return_value={"server": server} + ), mock.patch.object( + validate_credential.storage, "Client", return_value=client + ) as client_cls, mock.patch.object( + validate_credential.service_account.Credentials, + "from_service_account_info", + return_value="creds-of-the-credential", + ): + warning = validate_credential.check_state_bucket_access(DETAIL) + return warning, client_cls, client, bucket + + def test_full_access_gives_no_warning(self): + warning, *_ = self._check(granted=list(validate_credential.STATE_BUCKET_PERMISSIONS)) + self.assertIsNone(warning) + + def test_checks_with_the_registered_credential_not_the_server_account(self): + _, client_cls, client, bucket = self._check( + granted=list(validate_credential.STATE_BUCKET_PERMISSIONS) + ) + client_cls.assert_called_once_with( + project="other-project", credentials="creds-of-the-credential" + ) + client.bucket.assert_called_once_with(self.BUCKET) + asked = bucket.test_iam_permissions.call_args.args[0] + self.assertEqual(sorted(asked), sorted(validate_credential.STATE_BUCKET_PERMISSIONS)) + # Only asks; never reads or changes the bucket's IAM policy. + bucket.get_iam_policy.assert_not_called() + bucket.set_iam_policy.assert_not_called() + + def test_missing_permissions_give_the_exact_line_and_command(self): + warning, *_ = self._check(granted=["storage.objects.get", "storage.objects.list"]) + self.assertIn(f"Service account {EMAIL}", warning) + self.assertIn(f"gs://{self.BUCKET}", warning) + self.assertIn("storage.objects.create, storage.objects.delete", warning) + self.assertIn( + "state_bucket_writers in tf/state_bucket/deployments/dep1/admin.auto.tfvars", + warning, + ) + self.assertIn(f'"serviceAccount:{EMAIL}",', warning) + self.assertIn("terraform -chdir=tf/state_bucket/deployments/dep1 apply", warning) + self.assertIn("will fail", warning) + + def test_no_access_at_all(self): + warning, *_ = self._check(granted=[]) + self.assertIn(f'"serviceAccount:{EMAIL}",', warning) + + def test_missing_bucket_is_reported_not_raised(self): + warning, *_ = self._check(error=NotFound("404 bucket does not exist")) + self.assertIn(f"gs://{self.BUCKET}", warning) + self.assertIn("was not found", warning) + self.assertIn(EMAIL, warning) + + def test_unreadable_bucket_is_reported_not_raised(self): + warning, *_ = self._check(error=GoogleAPICallError("503 backend error")) + self.assertIn("Could not check", warning) + self.assertIn("503 backend error", warning) + + def test_no_configured_bucket_is_reported_without_calling_the_api(self): + warning, client_cls, *_ = self._check(granted=[], server={}) + self.assertIn("terraform_state_bucket", warning) + client_cls.assert_not_called() + + def test_auth_failure_is_reported_not_raised(self): + warning, *_ = self._check(error=GoogleAuthError("invalid_grant: account not found")) + self.assertIn("Could not check", warning) + self.assertIn("invalid_grant", warning) + + def test_single_short_attempt_inside_the_request(self): + _, _, _, bucket = self._check(granted=list(validate_credential.STATE_BUCKET_PERMISSIONS)) + kwargs = bucket.test_iam_permissions.call_args.kwargs + self.assertIsNone(kwargs["retry"]) + self.assertLessEqual(kwargs["timeout"], 30) + + def test_unknown_deployment_name_gives_a_placeholder_path(self): + warning, *_ = self._check(granted=[], server={"terraform_state_bucket": self.BUCKET}) + self.assertIn("tf/state_bucket/deployments//admin.auto.tfvars", warning) + self.assertNotIn("None", warning) diff --git a/community/front-end/ofe/website/ghpcfe/tests/test_deploy_script.py b/community/front-end/ofe/website/ghpcfe/tests/test_deploy_script.py new file mode 100644 index 0000000000..9d0b793e8a --- /dev/null +++ b/community/front-end/ofe/website/ghpcfe/tests/test_deploy_script.py @@ -0,0 +1,734 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the state-bucket logic in deploy.sh. + +deploy_state_bucket() decides whether to create or adopt the shared Terraform +state bucket and which service accounts may write to it. A mistake there can +orphan the state of every resource the front end provisions, so these tests +execute the real function rather than matching on the script's text. + +How: the function is extracted from deploy.sh by brace-matching (never +retyped), then run under bash inside a temporary directory whose PATH holds +stub `gcloud` and `terraform` executables and a stub script/service_account.sh. +The stubs record their calls and answer from canned data, so the tests need no +network, credentials, gcloud or terraform install, and are deterministic. + +Standard library only (no Django; see test_tf_state_backend.py), so it runs +under both `python3 -m pytest` and `python3 -m unittest`. +""" + +import json +import re +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path + +OFE_DIR = Path(__file__).resolve().parents[3] +DEPLOY_SH = OFE_DIR / "deploy.sh" + + +def _extract_bash_function(path: Path, name: str) -> str: + """Return a bash function's full source, found by brace-matching. + + Looks for "() {" or " () {" and returns everything up to + the matching closing brace, so a test runs exactly what the script + ships rather than a copy that could drift from it. + """ + text = path.read_text() + match = re.search(rf"^{re.escape(name)}\s*\(\)\s*{{", text, re.MULTILINE) + if not match: + raise AssertionError(f"no function named {name!r} found in {path}") + depth = 0 + start = match.end() - 1 # position of the opening brace + for i in range(start, len(text)): + if text[i] == "{": + depth += 1 + elif text[i] == "}": + depth -= 1 + if depth == 0: + return text[match.start() : i + 1] + raise AssertionError(f"unbalanced braces extracting {name!r} from {path}") + + +class RegionDerivationTests(unittest.TestCase): + """A --config deployment must derive region from zone, as the + interactive path does. + + Without this, deploy_from_config() leaves `region` unset, so the + generated tf/state_bucket/terraform.tfvars and tf/terraform.tfvars both + get `region = ""`, and `terraform apply` fails outright. + """ + + def test_config_deployment_derives_region_from_zone(self): + body = _extract_bash_function(DEPLOY_SH, "deploy_from_config") + self.assertIn( + "region=${zone%-*}", + body, + "deploy_from_config() must derive region from zone, the same " + "way the interactive deploy() path does", + ) + + +class StateBucketSandbox(unittest.TestCase): + """Runs the real deploy_state_bucket() against stubbed external tools. + + Not a test case itself (it defines no tests); the classes below share it. + """ + + # deploy_state_bucket() calls prepare_state_bucket_dir(), which uses + # STATE_BUCKET_ROOT, and validate_bucket_name(), which uses error() and + # BUCKET_NAME_MAX_LENGTH, so all of them are taken from deploy.sh. + FUNCTION_BODY = "\n".join( + [ + _extract_bash_function(DEPLOY_SH, "error"), + re.search( + r"^BUCKET_NAME_MAX_LENGTH=\d+$", DEPLOY_SH.read_text(), re.MULTILINE + ).group(0), + _extract_bash_function(DEPLOY_SH, "validate_bucket_name"), + re.search( + r"^STATE_BUCKET_ROOT=\S+$", DEPLOY_SH.read_text(), re.MULTILINE + ).group(0), + _extract_bash_function(DEPLOY_SH, "prepare_state_bucket_dir"), + _extract_bash_function(DEPLOY_SH, "deploy_state_bucket"), + ] + ) + SB_DIR = "tf/state_bucket/deployments/testdeploy" + + BUCKET = "testdeploy-tfstate-abcd1234" + PROJECT_NUMBER = "111111111111" + DEFAULT_SA = "serviceAccount:testdeploy-tkfe-sa@test-project.iam.gserviceaccount.com" + FOREIGN = "serviceAccount:other@other-project.iam.gserviceaccount.com" + + def _sandbox_run( + self, + *, + sa_exists: bool, + preexisting_tfvars: str | None = None, + preexisting_admin: bytes | None = None, + bucket: str = "absent", + in_local_state: bool = False, + policy: str = '{"bindings": []}', + local_output: str | None = None, + apply_fails: bool = False, + state_unreadable: bool = False, + computed_name: str | None = None, + state_show_name: str | None = None, + owner: str | None = None, + legacy: dict[str, str] | None = None, + other_deployments: dict[str, dict[str, str]] | None = None, + deployment_name: str = "testdeploy", + ) -> dict: + """Run deploy_state_bucket() in an isolated sandbox. + + `bucket` controls the stub gcloud's answer to `storage buckets list` + for this project: "absent" (success, nothing listed), "exists" + (success, the bucket listed), "error" (non-zero exit, with a message + that deliberately includes "not found: 404" to show the message text + is not what decides). `local_output` is what `terraform output -raw + bucket_name` reports from local state: None -> the computed bucket, + "" -> no outputs. `apply_fails` makes `terraform apply` exit non-zero + with a 409 message, as when the name is already taken. + `in_local_state` and `state_unreadable` both create a local + terraform.tfstate; with `state_unreadable`, `terraform state list` + fails on it. `computed_name` overrides the name `terraform console` + derives. `owner` is what `gcloud storage buckets describe --raw` + reports as the bucket's project number after a failed create: None + -> describe fails (bucket not visible), "this" -> this project's + number, anything else -> that number. `legacy` maps file names to + contents left in tf/state_bucket/ itself by an earlier release; + `other_deployments` maps deployment names to the files already in + their own directories. + """ + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + sb_root = root / "tf" / "state_bucket" + sb = root / self.SB_DIR + sb.mkdir(parents=True) + for tf_file in ("main.tf", "variables.tf"): + (sb_root / tf_file).write_text("# tracked\n") + for name, content in (legacy or {}).items(): + (sb_root / name).write_text(content) + for deployment, files in (other_deployments or {}).items(): + (sb_root / "deployments" / deployment).mkdir(parents=True, exist_ok=True) + for name, content in files.items(): + path = sb_root / "deployments" / deployment / name + if content.startswith("-> "): + path.symlink_to(content[3:]) + else: + path.write_text(content) + (root / "script").mkdir() + (root / "bin").mkdir() + + if preexisting_tfvars is not None: + (sb / "terraform.tfvars").write_text(preexisting_tfvars) + if preexisting_admin is not None: + (sb / "admin.auto.tfvars").write_bytes(preexisting_admin) + if in_local_state or state_unreadable: + (sb / "terraform.tfstate").write_text("{}") + (root / "policy.json").write_text(policy) + + def stub(path: Path, body: str) -> None: + path.write_text("#!/bin/bash\n" + body) + path.chmod(path.stat().st_mode | stat.S_IEXEC) + + stub(root / "script" / "service_account.sh", f"exit {0 if sa_exists else 1}\n") + + if local_output is None: + output_action = f"echo {self.BUCKET}" + elif local_output: + output_action = f"echo {local_output}" + else: + output_action = 'echo "No outputs found" >&2; exit 1' + if state_unreadable: + state_list_action = ( + 'echo "Error: Failed to load state: unsupported state version 9" >&2; exit 1' + ) + elif in_local_state: + state_list_action = "echo google_storage_bucket.tfstate" + else: + state_list_action = ":" + apply_action = ( + 'echo "Error 409: bucket name already in use" >&2; exit 1' + if apply_fails + else ":" + ) + name = computed_name if computed_name is not None else self.BUCKET + stub( + root / "bin" / "terraform", + f'echo "$*" >> "{root}/tf_calls.log"\n' + 'case "$*" in\n' + f' *" console"*) cat >/dev/null; echo \'"{name}"\' ;;\n' + f' *"state list"*) {state_list_action} ;;\n' + + ( + f' *"state show"*) echo \' name = "{state_show_name}"\' ;;\n' + if state_show_name + else "" + ) + + + f' *"output -raw bucket_name"*) {output_action} ;;\n' + f' *" apply"*) {apply_action} ;;\n' + "esac\n" + "exit 0\n", + ) + + listing = { + "absent": "exit 0", + "exists": f"echo {self.BUCKET}; exit 0", + "error": ( + 'echo "ERROR: HTTPError 403: denied; not found: 404" >&2; exit 1' + ), + }[bucket] + if owner is None: + describe = 'echo "ERROR: not visible" >&2; exit 1' + else: + describe = f"echo {self.PROJECT_NUMBER if owner == 'this' else owner}; exit 0" + stub( + root / "bin" / "gcloud", + f'echo "$*" >> "{root}/gcloud_calls.log"\n' + 'case "$1 $2 $3" in\n' + f' "storage buckets list") {listing} ;;\n' + f' "storage buckets get-iam-policy") cat "{root}/policy.json" ;;\n' + f' "storage buckets describe") {describe} ;;\n' + f' "projects describe "*) echo {self.PROJECT_NUMBER} ;;\n' + "esac\n", + ) + + # Call it exactly as deploy() does: under the script's top-level + # `set -e`, but as an `if !` condition, which suspends errexit for + # the whole function. Calling it bare under `set -e` would test a + # stricter context than production runs. + script = ( + f"{self.FUNCTION_BODY}\n" + "set -e\n" + "if ! deploy_state_bucket; then exit 1; fi\n" + ) + env = { + "PATH": f"{root / 'bin'}:/usr/bin:/bin", + "SCRIPT_DIR": str(root), + "project_id": "test-project", + "deployment_name": deployment_name, + "region": "europe-west4", + "USER": "test-user", + } + result = subprocess.run( + ["bash", "-c", script], + cwd=root, + env=env, + capture_output=True, + text=True, + timeout=30, + ) + tfvars = sb / "terraform.tfvars" + admin = sb / "admin.auto.tfvars" + calls = root / "tf_calls.log" + gcalls = root / "gcloud_calls.log" + + def snapshot(path: Path) -> dict: + return { + str(f.relative_to(path)): ( + "-> " + str(f.readlink()) if f.is_symlink() else f.read_text() + ) + for f in sorted(path.rglob("*")) + if f.is_file() or f.is_symlink() + } + + return { + "rc": result.returncode, + "files": snapshot(sb_root), + "out": result.stdout + result.stderr, + "tfvars": tfvars.read_text() if tfvars.exists() else None, + "admin": admin.read_bytes() if admin.exists() else None, + "tf_calls": calls.read_text().splitlines() if calls.exists() else [], + "gcloud_calls": gcalls.read_text().splitlines() if gcalls.exists() else [], + } + + def _ok(self, **kw) -> dict: + res = self._sandbox_run(**kw) + self.assertEqual(res["rc"], 0, f"deploy_state_bucket() failed: {res['out']!r}") + return res + + @staticmethod + def _list(text: str, name: str) -> list[str]: + match = re.search(rf"^{name}\s*=\s*\[(.*?)\]", text, re.MULTILINE | re.DOTALL) + assert match, f"no {name} in: {text!r}" + return re.findall(r'"([^"]+)"', match.group(1)) + + @staticmethod + def _imports(res): + return [c for c in res["tf_calls"] if c.split()[1:2] == ["import"]] + + @staticmethod + def _applied(res): + return any("apply" in c.split()[:2] for c in res["tf_calls"]) + + +class DeployStateBucketWritersTests(StateBucketSandbox): + """The state bucket's writer list is split by owner and never rebuilt + from a hand-edited file. + + terraform.tfvars belongs to deploy.sh and is regenerated on every deploy + (default_writers: the provisioning SA, when it exists). admin.auto.tfvars + belongs to the administrator (state_bucket_writers): deploy.sh creates it + once and never reads or rewrites it. The binding is authoritative, so the + regression guarded against is a hand-added writer being silently revoked + by the next deploy. + """ + + def test_generated_tfvars_never_carries_state_bucket_writers(self): + res = self._ok(sa_exists=True) + self.assertNotIn("state_bucket_writers", res["tfvars"]) + self.assertEqual(self._list(res["tfvars"], "default_writers"), [self.DEFAULT_SA]) + + def test_default_writers_empty_when_default_sa_absent(self): + res = self._ok(sa_exists=False) + self.assertEqual(self._list(res["tfvars"], "default_writers"), []) + + def test_first_deploy_creates_empty_admin_file(self): + res = self._ok(sa_exists=True) + self.assertEqual(self._list(res["admin"].decode(), "state_bucket_writers"), []) + + def test_hand_edited_admin_file_is_left_byte_for_byte(self): + # Multi-line, indented, commented, CRLF: layouts a line-based parser + # would misread, silently dropping writers. + hand_edited = ( + b"# my writers\r\n" + b"state_bucket_writers = [\r\n" + b' "' + self.FOREIGN.encode() + b'", # cross-project\r\n' + b' "serviceAccount:third@p.iam.gserviceaccount.com"\r\n' + b"]\r\n" + b"noncurrent_version_retention_days = 365\r\n" + ) + res = self._ok(sa_exists=True, preexisting_admin=hand_edited) + self.assertEqual(res["admin"], hand_edited) + self.assertTrue(self._applied(res)) + + def test_admin_file_untouched_on_redeploy_without_default_sa(self): + admin = b'state_bucket_writers = ["' + self.FOREIGN.encode() + b'"]\n' + res = self._ok(sa_exists=False, preexisting_admin=admin) + self.assertEqual(res["admin"], admin) + + def test_old_format_tfvars_without_admin_file_stops_before_apply(self): + res = self._sandbox_run( + sa_exists=True, + preexisting_tfvars=f'state_bucket_writers = ["{self.FOREIGN}"]\n', + ) + self.assertNotEqual(res["rc"], 0) + self.assertIn("Move that line into", res["out"]) + self.assertFalse(self._applied(res)) + self.assertIsNone(res["admin"]) + + def test_old_format_tfvars_with_admin_file_present_proceeds(self): + res = self._ok( + sa_exists=True, + preexisting_tfvars=f'state_bucket_writers = ["{self.FOREIGN}"]\n', + preexisting_admin=b"state_bucket_writers = []\n", + ) + self.assertNotIn("state_bucket_writers", res["tfvars"]) + + +class DeployStateBucketAdoptionTests(StateBucketSandbox): + """A deploy from a machine without tf/state_bucket's local state must + adopt the existing bucket, never create a second one, and must stop + rather than act on local state for a different bucket. + + Whether the bucket exists is decided from gcloud's exit status and output, + never from its error wording: exists -> import, listed nothing -> create, + failure -> stop before apply. + """ + + def test_absent_bucket_is_created_not_imported(self): + res = self._ok(sa_exists=True, bucket="absent") + self.assertEqual(self._imports(res), []) + self.assertTrue(self._applied(res)) + + def test_existing_bucket_is_imported_and_seeds_admin_file(self): + policy = ( + '{"bindings": [{"role": "roles/storage.objectUser", "members": [' + f'"{self.FOREIGN}", "{self.DEFAULT_SA}", ' + '"deleted:serviceAccount:gone@p.iam.gserviceaccount.com?uid=1"]},' + ' {"role": "roles/storage.legacyBucketOwner", "members": ["projectOwner:p"]}]}' + ) + res = self._ok(sa_exists=True, bucket="exists", policy=policy) + imports = self._imports(res) + self.assertEqual(len(imports), 1, res["tf_calls"]) + self.assertTrue( + imports[0].endswith(f"google_storage_bucket.tfstate test-project/{self.BUCKET}"), + imports[0], + ) + # Live holders seed the admin file, minus deleted principals, other + # roles, and the default SA (which deploy.sh supplies itself). + self.assertEqual(self._list(res["admin"].decode(), "state_bucket_writers"), [self.FOREIGN]) + self.assertEqual(self._list(res["tfvars"], "default_writers"), [self.DEFAULT_SA]) + self.assertTrue(self._applied(res)) + + def test_adoption_seeds_writers_from_an_earlier_object_admin_binding(self): + other = "serviceAccount:other@p.iam.gserviceaccount.com" + policy = ( + '{"bindings": [{"role": "roles/storage.objectAdmin", "members": [' + f'"{self.FOREIGN}", "{other}"]}}, ' + '{"role": "roles/storage.objectUser",' + f' "members": ["{self.FOREIGN}"]}}]}}' + ) + res = self._ok(sa_exists=False, bucket="exists", policy=policy) + # Buckets created before the move to objectUser hold objectAdmin; + # each holder is listed once. + self.assertEqual( + self._list(res["admin"].decode(), "state_bucket_writers"), [self.FOREIGN, other] + ) + + def test_adoption_does_not_overwrite_an_existing_admin_file(self): + policy = ( + '{"bindings": [{"role": "roles/storage.objectUser",' + f' "members": ["{self.FOREIGN}"]}}]}}' + ) + admin = b"state_bucket_writers = [] # deliberately empty\n" + res = self._ok(sa_exists=False, bucket="exists", policy=policy, preexisting_admin=admin) + self.assertEqual(res["admin"], admin) + + def test_bucket_already_in_local_state_is_not_reimported(self): + res = self._ok(sa_exists=True, bucket="exists", in_local_state=True) + self.assertEqual(self._imports(res), []) + + def test_local_state_for_a_different_bucket_stops_before_apply(self): + # A bucket named under an earlier, non-deterministic naming scheme. + res = self._sandbox_run( + sa_exists=True, in_local_state=True, local_output="testdeploy-tfstate-c1d6" + ) + self.assertNotEqual(res["rc"], 0) + self.assertIn("gs://testdeploy-tfstate-c1d6", res["out"]) + self.assertIn(f"gs://{self.BUCKET}", res["out"]) + self.assertIn(f"mv {self.SB_DIR}/admin.auto.tfvars", res["out"]) + self.assertFalse(self._applied(res)) + self.assertEqual(self._imports(res), []) + + def test_local_state_without_bucket_output_stops_before_apply(self): + res = self._sandbox_run(sa_exists=True, in_local_state=True, local_output="") + self.assertNotEqual(res["rc"], 0) + self.assertIn("", res["out"]) + self.assertFalse(self._applied(res)) + + def test_local_state_without_output_uses_the_name_in_state(self): + # An adoption whose apply failed: the bucket is in state, no outputs. + res = self._sandbox_run( + sa_exists=True, in_local_state=True, local_output="", state_show_name=self.BUCKET + ) + # The stub writes no outputs on apply, so only the guard is checked: + # no wrong-bucket stop, and the apply runs. + self.assertNotIn("belongs to", res["out"]) + self.assertNotIn("", res["out"]) + self.assertTrue(self._applied(res)) + + def test_bucket_is_looked_up_by_exact_name_within_the_project(self): + res = self._ok(sa_exists=True, bucket="absent") + lists = [c for c in res["gcloud_calls"] if c.startswith("storage buckets list")] + self.assertEqual(len(lists), 1, res["gcloud_calls"]) + self.assertIn("--project=test-project", lists[0]) + self.assertIn(f"--filter=name={self.BUCKET}", lists[0]) + + def test_lookup_error_stops_before_apply(self): + # The stub's message contains "not found: 404"; it must still stop. + res = self._sandbox_run(sa_exists=True, bucket="error") + self.assertNotEqual(res["rc"], 0) + self.assertIn("could not determine whether", res["out"]) + self.assertIn("403", res["out"]) + self.assertFalse(self._applied(res)) + self.assertEqual(self._imports(res), []) + + def test_unparseable_policy_stops_before_apply(self): + res = self._sandbox_run(sa_exists=True, bucket="exists", policy="not json") + self.assertNotEqual(res["rc"], 0) + self.assertFalse(self._applied(res)) + + def test_name_owned_by_another_project_fails_at_create_with_explanation(self): + # Bucket names are global. The project's listing cannot show a bucket + # owned elsewhere, so the create is attempted, fails with a 409, and + # the owner is then named from its project number. + res = self._sandbox_run(sa_exists=True, bucket="absent", apply_fails=True, owner="999") + self.assertNotEqual(res["rc"], 0) + self.assertIn("already exists in another project (number", res["out"]) + self.assertIn("999", res["out"]) + self.assertEqual(self._imports(res), []) + + def test_failed_create_of_own_bucket_says_to_rerun_and_adopt(self): + # A 409 can also mean the listing missed this project's own bucket. + res = self._sandbox_run(sa_exists=True, bucket="absent", apply_fails=True, owner="this") + self.assertNotEqual(res["rc"], 0) + self.assertIn("already exists in test-project", res["out"]) + self.assertIn("Rerun deploy.sh to adopt it", res["out"]) + self.assertNotIn("another project", res["out"]) + + def test_failed_update_of_bucket_in_local_state_is_not_called_a_name_clash(self): + # An ordinary redeploy whose apply fails for another reason (IAM, + # quota) must not be told to rerun and adopt a bucket it manages. + res = self._sandbox_run( + sa_exists=True, bucket="exists", in_local_state=True, apply_fails=True, owner="this" + ) + self.assertNotEqual(res["rc"], 0) + self.assertIn("already managed by this deployment", res["out"]) + self.assertNotIn("Rerun deploy.sh to adopt it", res["out"]) + self.assertFalse( + any(c.startswith("storage buckets describe") for c in res["gcloud_calls"]), + res["gcloud_calls"], + ) + + def test_failed_apply_after_adopting_the_bucket_is_not_called_a_name_clash(self): + res = self._sandbox_run(sa_exists=True, bucket="exists", apply_fails=True, owner="this") + self.assertNotEqual(res["rc"], 0) + self.assertEqual(len(self._imports(res)), 1, res["tf_calls"]) + self.assertIn("already managed by this deployment", res["out"]) + self.assertNotIn("Rerun deploy.sh to adopt it", res["out"]) + + def test_failed_create_of_invisible_bucket_says_so(self): + res = self._sandbox_run(sa_exists=True, bucket="absent", apply_fails=True, owner=None) + self.assertNotEqual(res["rc"], 0) + self.assertIn("a project", res["out"]) + self.assertIn("this account cannot see", res["out"]) + + +class DeployStateBucketDirectoryTests(StateBucketSandbox): + """Each deployment name keeps its state bucket files in its own + directory, tf/state_bucket/deployments//, so a second deployment + from the same checkout neither collides with an earlier one nor needs + its files moved by hand. Files an earlier release left in + tf/state_bucket/ move to their owner's directory only when the owner is + certain; otherwise the deploy stops before anything runs. + """ + + LEGACY_TFVARS = 'project_id = "test-project"\ndeployment_name = "olddeploy"\n' + + def test_files_are_written_in_the_deployments_own_directory(self): + res = self._ok(sa_exists=True) + self.assertIn("deployments/testdeploy/terraform.tfvars", res["files"]) + self.assertIn("deployments/testdeploy/admin.auto.tfvars", res["files"]) + self.assertEqual(res["files"]["deployments/testdeploy/main.tf"], "-> ../../main.tf") + self.assertEqual( + res["files"]["deployments/testdeploy/variables.tf"], "-> ../../variables.tf" + ) + for call in res["tf_calls"]: + self.assertTrue(call.startswith(f"-chdir={self.SB_DIR} "), call) + + def test_links_are_refreshed_on_every_deploy(self): + # A link to a .tf file since removed from the tracked root must not + # linger, and existing links are replaced, not duplicated. + links = {"main.tf": "-> ../../main.tf", "removed.tf": "-> ../../removed.tf"} + res = self._ok(sa_exists=True, other_deployments={"testdeploy": links}) + self.assertEqual( + sorted( + k + for k in res["files"] + if k.startswith("deployments/testdeploy/") and k.endswith(".tf") + ), + ["deployments/testdeploy/main.tf", "deployments/testdeploy/variables.tf"], + ) + + def test_another_deployments_files_are_left_untouched(self): + other = { + "terraform.tfstate": '{"other": true}', + "terraform.tfvars": 'deployment_name = "other"\n', + "admin.auto.tfvars": "state_bucket_writers = []\n", + } + res = self._ok(sa_exists=True, other_deployments={"other": other}) + for name, content in other.items(): + self.assertEqual(res["files"][f"deployments/other/{name}"], content) + self.assertNotIn("mv ", res["out"]) + + def test_earlier_release_files_move_to_their_owner(self): + legacy = { + "terraform.tfstate": '{"legacy": true}', + "terraform.tfstate.backup": "{}", + "terraform.tfvars": self.LEGACY_TFVARS, + "admin.auto.tfvars": "state_bucket_writers = []\n", + } + res = self._ok(sa_exists=True, legacy=legacy) + for name, content in legacy.items(): + self.assertNotIn(name, res["files"]) + self.assertEqual(res["files"][f"deployments/olddeploy/{name}"], content) + self.assertIn("Moving deployment olddeploy's state bucket files", res["out"]) + + def test_earlier_release_state_for_another_deployments_bucket_stops(self): + # An earlier release rewrote terraform.tfvars before applying, so a + # failed deploy under another name left it naming the wrong owner. + state = json.dumps( + { + "resources": [ + { + "type": "google_storage_bucket", + "name": "tfstate", + "instances": [{"attributes": {"labels": {"ghpcfe_id": "first"}}}], + } + ] + } + ) + legacy = {"terraform.tfstate": state, "terraform.tfvars": self.LEGACY_TFVARS} + res = self._sandbox_run(sa_exists=True, legacy=legacy) + self.assertNotEqual(res["rc"], 0) + self.assertIn("cannot tell which deployment", res["out"]) + self.assertEqual(res["files"]["terraform.tfstate"], state) + self.assertEqual(res["tf_calls"], []) + + def test_earlier_release_state_whose_label_matches_moves(self): + state = json.dumps( + { + "resources": [ + { + "type": "google_storage_bucket", + "name": "tfstate", + "instances": [{"attributes": {"labels": {"ghpcfe_id": "olddeploy"}}}], + } + ] + } + ) + legacy = {"terraform.tfstate": state, "terraform.tfvars": self.LEGACY_TFVARS} + res = self._ok(sa_exists=True, legacy=legacy) + self.assertEqual(res["files"]["deployments/olddeploy/terraform.tfstate"], state) + + def test_dot_deployment_names_are_rejected(self): + for name in (".", ".."): + with self.subTest(name=name): + res = self._sandbox_run(sa_exists=True, deployment_name=name) + self.assertNotEqual(res["rc"], 0) + self.assertIn("invalid deployment name", res["out"]) + self.assertEqual(res["tf_calls"], []) + + def test_earlier_release_files_without_an_owner_stop_before_anything(self): + legacy = {"admin.auto.tfvars": "state_bucket_writers = []\n"} + res = self._sandbox_run(sa_exists=True, legacy=legacy) + self.assertNotEqual(res["rc"], 0) + self.assertIn("cannot tell which deployment", res["out"]) + self.assertEqual(res["files"]["admin.auto.tfvars"], legacy["admin.auto.tfvars"]) + self.assertEqual(res["tf_calls"], []) + self.assertEqual(res["gcloud_calls"], []) + + def test_earlier_release_files_stop_if_the_owner_already_has_files(self): + legacy = {"terraform.tfstate": "{}", "terraform.tfvars": self.LEGACY_TFVARS} + res = self._sandbox_run( + sa_exists=True, + legacy=legacy, + other_deployments={"olddeploy": {"terraform.tfstate": '{"newer": true}'}}, + ) + self.assertNotEqual(res["rc"], 0) + self.assertIn("cannot tell which deployment", res["out"]) + self.assertEqual(res["files"]["terraform.tfstate"], "{}") + self.assertEqual( + res["files"]["deployments/olddeploy/terraform.tfstate"], '{"newer": true}' + ) + self.assertEqual(res["tf_calls"], []) + + +class DeployStateBucketLocalStateTests(StateBucketSandbox): + """Local tf/state_bucket state is read only when it exists, and a file + that exists but cannot be read stops the deploy with Terraform's own + error instead of being mistaken for "no state". + """ + + def test_init_upgrades_a_lock_file_from_an_older_provider_pin(self): + # A lock file written under an earlier provider constraint makes a + # plain `terraform init` fail; -upgrade re-resolves it in range. + res = self._ok(sa_exists=True, bucket="absent") + inits = [c for c in res["tf_calls"] if c.split()[1:2] == ["init"]] + self.assertEqual(len(inits), 1, res["tf_calls"]) + self.assertIn("-upgrade", inits[0].split()) + + def test_no_local_state_file_skips_state_list(self): + res = self._ok(sa_exists=True, bucket="absent") + self.assertFalse(any("state list" in c for c in res["tf_calls"]), res["tf_calls"]) + + def test_unreadable_local_state_stops_with_terraform_error(self): + res = self._sandbox_run(sa_exists=True, bucket="exists", state_unreadable=True) + self.assertNotEqual(res["rc"], 0) + self.assertIn(f"could not read {self.SB_DIR}/terraform.tfstate", res["out"]) + self.assertIn("unsupported state version 9", res["out"]) + self.assertIn("terraform.tfstate.unreadable", res["out"]) + self.assertEqual(self._imports(res), []) + self.assertFalse(self._applied(res)) + self.assertFalse( + any(c.startswith("storage buckets list") for c in res["gcloud_calls"]), + res["gcloud_calls"], + ) + + +class DeployStateBucketNameTests(StateBucketSandbox): + """An invalid computed bucket name stops the deploy before any lookup, + import or create. tf/state_bucket/main.tf derives a valid name, so this + is a safety net. + """ + + def _assert_stopped_before_anything(self, name): + res = self._sandbox_run(sa_exists=True, bucket="absent", computed_name=name) + self.assertNotEqual(res["rc"], 0, res["out"]) + self.assertIn(f"State bucket name '{name}' is invalid", res["out"]) + self.assertFalse(self._applied(res)) + self.assertEqual(self._imports(res), []) + self.assertFalse( + any(c.startswith("storage buckets list") for c in res["gcloud_calls"]), + res["gcloud_calls"], + ) + + def test_name_longer_than_63_characters_stops(self): + self._assert_stopped_before_anything("a" * 47 + "-tfstate-abcd1234") + + def test_name_with_a_dot_stops(self): + self._assert_stopped_before_anything("my.deploy-tfstate-abcd1234") + + def test_name_ending_in_a_dash_stops(self): + self._assert_stopped_before_anything("testdeploy-tfstate-") + + def test_63_character_name_is_accepted(self): + name = "a" * 46 + "-tfstate-abcd1234" + self.assertEqual(len(name), 63) + res = self._sandbox_run(sa_exists=True, bucket="absent", computed_name=name) + self.assertEqual(res["rc"], 0, res["out"]) diff --git a/community/front-end/ofe/website/ghpcfe/tests/test_image_startup.py b/community/front-end/ofe/website/ghpcfe/tests/test_image_startup.py new file mode 100644 index 0000000000..49512f33c4 --- /dev/null +++ b/community/front-end/ofe/website/ghpcfe/tests/test_image_startup.py @@ -0,0 +1,132 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the custom-image builder's startup script wrapper. + +The wrapper is run with bash against fake `curl` and `gcloud` commands, so the +tests check what it would report to Packer without a VM. The module is loaded +by path, without Django, for the same reason as in test_tf_state_backend.py. +""" + +import importlib.util +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +CLUSTER_MANAGER = Path(__file__).resolve().parents[1] / "cluster_manager" + + +def _load_image_startup(): + spec = importlib.util.spec_from_file_location( + "ofe_cluster_manager_image_startup", CLUSTER_MANAGER / "image_startup.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +image_startup = _load_image_startup() + +# Fake metadata server. Its instance name differs from $HOSTNAME (an FQDN, as +# on some builder VM boots), so reporting under $HOSTNAME fails these tests. +FAKE_CURL = """#!/bin/bash +case "${@: -1}" in + */instance/name) echo packer-abc123 ;; + */instance/zone) echo projects/123456/zones/europe-west4-a ;; + *) exit 22 ;; +esac +""" + +FAKE_GCLOUD = """#!/bin/bash +echo "$@" >>"$GCLOUD_LOG" +""" + + +class WrapStartupScriptTests(unittest.TestCase): + """The wrapped script runs unchanged and reports done or error to Packer + under the instance name from the metadata server. + """ + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + tmp = Path(self.tmp.name) + bin_dir = tmp / "bin" + bin_dir.mkdir() + for name, body in (("curl", FAKE_CURL), ("gcloud", FAKE_GCLOUD)): + path = bin_dir / name + path.write_text(body) + path.chmod(0o755) + self.gcloud_log = tmp / "gcloud.log" + self.env = dict( + os.environ, + PATH=f"{bin_dir}:{os.environ['PATH']}", + TMPDIR=str(tmp), + GCLOUD_LOG=str(self.gcloud_log), + HOSTNAME="packer-abc123.c.example-project.internal", + ) + + def _run(self, script): + wrapper = Path(self.tmp.name) / "wrapper.sh" + wrapper.write_text(image_startup.wrap_startup_script(script)) + result = subprocess.run( + ["bash", str(wrapper)], env=self.env, capture_output=True, + text=True, check=False, + ) + calls = (self.gcloud_log.read_text().splitlines() + if self.gcloud_log.exists() else []) + return result, calls + + def test_success_reports_done_under_instance_name(self): + result, calls = self._run("#!/bin/bash\necho ran\n") + self.assertEqual(result.returncode, 0) + self.assertIn("ran", result.stdout) + self.assertEqual(calls, [ + "compute instances add-metadata packer-abc123 --zone " + "europe-west4-a --metadata startup-script-status=done" + ]) + + def test_failure_reports_error_and_keeps_exit_code(self): + result, calls = self._run("#!/bin/bash\nexit 3\n") + self.assertEqual(result.returncode, 3) + self.assertEqual(calls, [ + "compute instances add-metadata packer-abc123 --zone " + "europe-west4-a --metadata startup-script-status=error" + ]) + + def test_script_text_is_passed_through_unchanged(self): + script = ('#!/bin/bash\nx=\'$HOME `id` $(id)\'\n' + 'cat <<"EOF"\n$x \\n\nEOF\necho "$x"\n') + result, _ = self._run(script) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, + "$x \\n\n$HOME `id` $(id)\n") + + def test_temporary_copy_is_removed(self): + self._run("#!/bin/bash\ntrue\n") + self.assertEqual( + sorted(p.name for p in Path(self.tmp.name).iterdir()), + ["bin", "gcloud.log", "wrapper.sh"], + ) + + def test_rejects_script_containing_the_delimiter(self): + with self.assertRaises(ValueError): + image_startup.wrap_startup_script( + "#!/bin/bash\nOFE_IMAGE_STARTUP_SCRIPT_EOF\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/community/front-end/ofe/website/ghpcfe/tests/test_migrate_tf_state_script.py b/community/front-end/ofe/website/ghpcfe/tests/test_migrate_tf_state_script.py new file mode 100644 index 0000000000..fc1fd9f13a --- /dev/null +++ b/community/front-end/ofe/website/ghpcfe/tests/test_migrate_tf_state_script.py @@ -0,0 +1,301 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end tests for script/migrate_tf_state_to_gcs.sh. + +The script moves each resource's local Terraform state into the state bucket +and keeps the original as terraform.tfstate.premigration. That copy is the +operator's only record of the state as it was before migration, so it must +survive later operations on the resource. For clusters, filesystems and image +builders the Terraform working directory is a Cluster Toolkit deployment +group, which `gcluster create -w` moves into .ghpc/previous_deployment_groups/ +on every reconfigure, deleting the set it moved there the time before. The +copy therefore lives in the resource's run-time directory instead, beside its +cloud_credentials. + +Each test runs the real script in a synthetic TKFE tree in a temporary +directory: a copy of the script, the real cluster_manager/utils.py, a +configuration.yaml, and stubs for `terraform` and for the application's +Python interpreter. Nothing here needs Django, network access or cloud +credentials. +""" + +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +OFE_DIR = Path(__file__).resolve().parents[3] +CLUSTER_MANAGER = OFE_DIR / "website" / "ghpcfe" / "cluster_manager" +MIGRATE_SCRIPT = OFE_DIR / "script" / "migrate_tf_state_to_gcs.sh" + +NAMESPACE = "20261015-0a1b2c3d" +BUCKET = "test-bucket" +STATE = '{"version": 4, "resources": [{"type": "google_compute_network"}]}' +BACKUP_NAME = "terraform.tfstate.premigration" + +# A stand-in for `terraform`. Every call is appended to $TF_CALL_LOG. +# `init -migrate-state` exits with $INIT_RC; on success it empties the local +# state, as the real one does after copying it to the backend. +# `plan -detailed-exitcode` exits with $PLAN_RC. +TERRAFORM_STUB = """#!/bin/bash +echo "$*" >>"${TF_CALL_LOG}" +dir=${1#-chdir=} +case "$*" in +*"init -migrate-state"*) + [[ ${INIT_RC:-0} -eq 0 ]] || exit "${INIT_RC}" + echo '{"version": 4, "resources": []}' >"${dir}/terraform.tfstate" + ;; +*"plan -detailed-exitcode"*) exit "${PLAN_RC:-0}" ;; +esac +""" + +# A Toolkit-generated deployment group already carries its backend block. +GENERATED_MAIN_TF = 'terraform {\n backend "gcs" {\n bucket = "test-bucket"\n }\n}\n' + + +class MigrateScriptBackupTests(unittest.TestCase): + """Where the pre-migration copy of each resource's state is kept.""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name) + + self.ofe = root / "ofe" + (self.ofe / "script").mkdir(parents=True) + shutil.copy2(MIGRATE_SCRIPT, self.ofe / "script" / MIGRATE_SCRIPT.name) + cluster_manager = self.ofe / "website" / "ghpcfe" / "cluster_manager" + cluster_manager.mkdir(parents=True) + (self.ofe / "website" / "ghpcfe" / "__init__.py").write_text("") + (cluster_manager / "__init__.py").write_text("") + shutil.copy2(CLUSTER_MANAGER / "utils.py", cluster_manager / "utils.py") + (self.ofe / "configuration.yaml").write_text( + f'config:\n server:\n terraform_state_bucket: "{BUCKET}"\n' + ) + + bin_dir = root / "bin" + bin_dir.mkdir() + self._write_executable(bin_dir / "terraform", TERRAFORM_STUB) + # The application's interpreter answers the namespace management + # command and runs everything else with this test's interpreter. + ofe_python = bin_dir / "ofe-python" + self._write_executable( + ofe_python, + "#!/bin/bash\n" + f"if [[ $1 == manage.py ]]; then echo {NAMESPACE}; exit 0; fi\n" + f'exec "{sys.executable}" "$@"\n', + ) + # The script also calls plain python3 for JSON checks; make sure it + # resolves to an interpreter that exists in this environment. + self._write_executable( + bin_dir / "python3", f'#!/bin/bash\nexec "{sys.executable}" "$@"\n' + ) + + self.tf_call_log = root / "terraform-calls.log" + self.tf_call_log.write_text("") + self.env = { + "PATH": f"{bin_dir}:/usr/bin:/bin", + "OFE_PYTHON": str(ofe_python), + "TF_CALL_LOG": str(self.tf_call_log), + # Keep the tree byte-for-byte unchanged by a dry run. + "PYTHONDONTWRITEBYTECODE": "1", + } + + @staticmethod + def _write_executable(path, content): + path.write_text(content) + path.chmod(0o755) + + def _resource(self, rel_resource_dir, rel_working_dir="", generated=False): + """Create a resource run-time directory and its Terraform working + directory holding local state. Returns (resource_dir, working_dir). + """ + resource_dir = self.ofe / rel_resource_dir + working_dir = resource_dir / rel_working_dir + working_dir.mkdir(parents=True) + (resource_dir / "cloud_credentials").write_text("{}") + (working_dir / "terraform.tfstate").write_text(STATE) + if generated: + (working_dir / "main.tf").write_text(GENERATED_MAIN_TF) + return resource_dir, working_dir + + def _run(self, *args, init_rc=0, plan_rc=0): + return subprocess.run( + ["bash", str(self.ofe / "script" / MIGRATE_SCRIPT.name), *args], + env={**self.env, "INIT_RC": str(init_rc), "PLAN_RC": str(plan_rc)}, + cwd=self.ofe, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + def _snapshot(self): + """Every file under the OFE tree, with its content.""" + return { + str(path.relative_to(self.ofe)): path.read_bytes() + for path in sorted(self.ofe.rglob("*")) + if path.is_file() + } + + def _simulate_gcluster_create(self, deployment_dir): + """Do to a deployment directory what `gcluster create -w` does + before it writes the groups again (pkg/modulewriter, prepDepDir): + delete the previous backup set, then move every group directory into + .ghpc/previous_deployment_groups/. + """ + previous = deployment_dir / ".ghpc" / "previous_deployment_groups" + shutil.rmtree(previous, ignore_errors=True) + previous.mkdir(parents=True) + for entry in sorted(deployment_dir.iterdir()): + if entry.is_dir() and entry.name != ".ghpc": + entry.rename(previous / entry.name) + + def test_blueprint_resource_backup_is_kept_in_resource_dir(self): + cases = { + "cluster": ("clusters/cluster_5", "c5/primary"), + "filesystem": ("fs/fs_3", "shared/primary"), + "image": ("images/image_4", "image_4/builder-env"), + } + for kind, (rel_resource, rel_working) in cases.items(): + with self.subTest(kind=kind): + resource_dir, working_dir = self._resource( + rel_resource, rel_working, generated=True + ) + res = self._run("--yes", "--dir", str(working_dir)) + self.assertEqual(res.returncode, 0, res.stdout + res.stderr) + + backup = resource_dir / BACKUP_NAME + self.assertEqual(backup.read_text(), STATE) + self.assertFalse((working_dir / BACKUP_NAME).exists()) + self.assertFalse((working_dir / "terraform.tfstate").exists()) + self.assertIn(f"pre-migration state kept as {backup}", res.stdout) + # The generated group already has its backend block. + self.assertFalse((working_dir / "backend.tf").exists()) + + def test_template_network_backup_location_is_unchanged(self): + vpc, _ = self._resource("vpcs/vpc_1") + res = self._run("--yes") + self.assertEqual(res.returncode, 0, res.stdout + res.stderr) + self.assertEqual((vpc / BACKUP_NAME).read_text(), STATE) + self.assertIn(f'prefix = "{NAMESPACE}/vpcs/vpc_1"', (vpc / "backend.tf").read_text()) + + def test_template_workbench_backup_is_kept_in_resource_dir(self): + workbench, working_dir = self._resource("workbenches/workbench_10", "terraform/google") + res = self._run("--yes") + self.assertEqual(res.returncode, 0, res.stdout + res.stderr) + self.assertEqual((workbench / BACKUP_NAME).read_text(), STATE) + self.assertFalse((working_dir / BACKUP_NAME).exists()) + self.assertTrue((working_dir / "backend.tf").exists()) + + def test_backup_survives_toolkit_replacing_deployment_groups(self): + cluster, working_dir = self._resource( + "clusters/cluster_5", "c5/primary", generated=True + ) + res = self._run("--yes") + self.assertEqual(res.returncode, 0, res.stdout + res.stderr) + + # Two reconfigures: the first moves the migrated group aside, the + # second deletes it. + deployment_dir = working_dir.parent + for _ in range(2): + self._simulate_gcluster_create(deployment_dir) + working_dir.mkdir() + (working_dir / "main.tf").write_text(GENERATED_MAIN_TF) + + self.assertEqual((cluster / BACKUP_NAME).read_text(), STATE) + previous = deployment_dir / ".ghpc" / "previous_deployment_groups" / "primary" + self.assertEqual(sorted(p.name for p in previous.iterdir()), ["main.tf"]) + + def test_existing_different_backup_is_not_overwritten(self): + cluster, working_dir = self._resource( + "clusters/cluster_5", "c5/primary", generated=True + ) + earlier = '{"version": 4, "resources": [{"type": "earlier"}]}' + (cluster / BACKUP_NAME).write_text(earlier) + + for mode in ("--dry-run", "--yes"): + with self.subTest(mode=mode): + res = self._run(mode) + self.assertEqual(res.returncode, 1, res.stdout + res.stderr) + self.assertIn("refusing to overwrite", res.stderr) + self.assertIn(str(cluster / BACKUP_NAME), res.stderr) + self.assertIn("Failed: 1", res.stdout) + self.assertEqual((cluster / BACKUP_NAME).read_text(), earlier) + self.assertEqual((working_dir / "terraform.tfstate").read_text(), STATE) + self.assertEqual(self.tf_call_log.read_text(), "") + + def test_rerun_after_failed_init_reuses_identical_backup(self): + cluster, working_dir = self._resource( + "clusters/cluster_5", "c5/primary", generated=True + ) + res = self._run("--yes", init_rc=1) + self.assertEqual(res.returncode, 1, res.stdout + res.stderr) + self.assertIn("terraform init -migrate-state failed", res.stderr) + self.assertEqual((cluster / BACKUP_NAME).read_text(), STATE) + self.assertEqual((working_dir / "terraform.tfstate").read_text(), STATE) + + rerun = self._run("--yes") + self.assertEqual(rerun.returncode, 0, rerun.stdout + rerun.stderr) + self.assertEqual((cluster / BACKUP_NAME).read_text(), STATE) + self.assertFalse((working_dir / "terraform.tfstate").exists()) + + def test_rerun_after_success_skips_and_keeps_backup(self): + cluster, _ = self._resource( + "clusters/cluster_5", "c5/primary", generated=True + ) + self.assertEqual(self._run("--yes").returncode, 0) + rerun = self._run("--yes") + self.assertEqual(rerun.returncode, 0, rerun.stdout + rerun.stderr) + self.assertIn("No local Terraform state found", rerun.stdout) + self.assertEqual((cluster / BACKUP_NAME).read_text(), STATE) + + def test_dry_run_writes_no_backup(self): + cluster, working_dir = self._resource( + "clusters/cluster_5", "c5/primary", generated=True + ) + vpc, _ = self._resource("vpcs/vpc_1") + before = self._snapshot() + + res = self._run("--dry-run") + self.assertEqual(res.returncode, 0, res.stdout + res.stderr) + self.assertIn(f"would migrate: {working_dir}", res.stdout) + self.assertIn(f"original to be kept as {cluster / BACKUP_NAME}", res.stdout) + self.assertIn(f"original to be kept as {vpc / BACKUP_NAME}", res.stdout) + self.assertEqual(self._snapshot(), before) + self.assertEqual(self.tf_call_log.read_text(), "") + + def test_failed_plan_points_to_backup_in_resource_dir(self): + cluster, working_dir = self._resource( + "clusters/cluster_5", "c5/primary", generated=True + ) + backup = cluster / BACKUP_NAME + res = self._run("--yes", plan_rc=2) + self.assertEqual(res.returncode, 1, res.stdout + res.stderr) + self.assertIn(f"The original is {backup}.", res.stderr) + self.assertEqual(backup.read_text(), STATE) + self.assertTrue((working_dir / "terraform.tfstate.migration-incomplete").exists()) + + rerun = self._run("--yes") + self.assertEqual(rerun.returncode, 1, rerun.stdout + rerun.stderr) + self.assertIn("did not finish", rerun.stderr) + self.assertIn(f"the original is {backup}", rerun.stderr) + self.assertEqual(backup.read_text(), STATE) + + +if __name__ == "__main__": + unittest.main() diff --git a/community/front-end/ofe/website/ghpcfe/tests/test_post_apply_state.py b/community/front-end/ofe/website/ghpcfe/tests/test_post_apply_state.py new file mode 100644 index 0000000000..818edeb064 --- /dev/null +++ b/community/front-end/ofe/website/ghpcfe/tests/test_post_apply_state.py @@ -0,0 +1,278 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A failure after a successful `terraform apply` keeps the resource managed. + +Once the apply succeeds the resource exists in the cloud, so a later failure +(reading the state back from the bucket, or bookkeeping after it) must not +mark it "New" or leave it "Creating": either hides the actions that manage or +destroy it, and the resource keeps running and billing. + +Like test_tf_state_backend.py, these tests import no Django: vpc.py and +clusterinfo.py are loaded by path with their Django-dependent imports stubbed. +""" + +import importlib.util +import subprocess +import sys +import types +import unittest +from pathlib import Path +from unittest import mock + +OFE_DIR = Path(__file__).resolve().parents[3] +CLUSTER_MANAGER = OFE_DIR / "website" / "ghpcfe" / "cluster_manager" + + +def _module(name, **attrs): + module = types.ModuleType(name) + for key, value in attrs.items(): + setattr(module, key, value) + return module + + +def _package(name): + package = _module(name) + package.__path__ = [] + return package + + +def _load(name, stubs): + """Load cluster_manager/.py inside a throwaway package.""" + with mock.patch.dict(sys.modules, stubs): + spec = importlib.util.spec_from_file_location( + f"ofe_stub.cluster_manager.{name}", CLUSTER_MANAGER / f"{name}.py" + ) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _load_utils(): + spec = importlib.util.spec_from_file_location( + "ofe_post_apply_utils", CLUSTER_MANAGER / "utils.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +utils = _load_utils() + + +def _common_stubs(**models): + manager = _package("ofe_stub.cluster_manager") + manager.utils = utils + stubs = { + "ofe_stub": _package("ofe_stub"), + "ofe_stub.cluster_manager": manager, + "ofe_stub.cluster_manager.utils": utils, + "ofe_stub.models": _module("ofe_stub.models", **models), + } + return stubs + + +def _load_vpc(): + return _load( + "vpc", + _common_stubs( + VirtualNetwork=mock.MagicMock(), VirtualSubnet=mock.MagicMock() + ), + ) + + +def _load_clusterinfo(): + stubs = _common_stubs( + Cluster=mock.MagicMock(), + ApplicationInstallationLocation=mock.MagicMock(), + ComputeInstance=mock.MagicMock(), + ContainerRegistry=mock.MagicMock(), + ) + for name in ("c2", "cloud_info"): + stubs[f"ofe_stub.cluster_manager.{name}"] = mock.MagicMock() + stubs["ofe_stub.grafana"] = mock.MagicMock() + stubs.update( + { + "django": _package("django"), + "django.template": _module( + "django.template", engines=mock.MagicMock() + ), + "django.utils": _module( + "django.utils", timezone=mock.MagicMock() + ), + "google": _package("google"), + "google.api_core": _package("google.api_core"), + "google.api_core.exceptions": _module( + "google.api_core.exceptions", + PermissionDenied=type("PermissionDenied", (Exception,), {}), + ), + "website": _package("website"), + "website.settings": _module( + "website.settings", SITE_NAME="test-site" + ), + } + ) + return _load("clusterinfo", stubs) + + +vpc_module = _load_vpc() +clusterinfo = _load_clusterinfo() + +STATE_PULL_FAILED = subprocess.CalledProcessError( + 1, ["terraform", "state", "pull"] +) +APPLY_FAILED = subprocess.CalledProcessError(1, ["terraform", "apply"]) + + +def _fake_vpc(cloud_state="cm"): + vpc = mock.MagicMock() + vpc.id = 7 + vpc.cloud_state = cloud_state + subnet = mock.MagicMock() + subnet.id = 3 + subnet.is_managed = True + vpc.subnets.all.return_value = [subnet] + return vpc, subnet + + +class StartVpcTests(unittest.TestCase): + """start_vpc never leaves a network stuck in "Creating".""" + + def _start(self, vpc, state=None, state_error=None, apply_error=None): + with mock.patch.object( + vpc_module, "_tf_dir_for_vpc", return_value=Path("/x") + ), mock.patch.object( + utils, "run_terraform", side_effect=apply_error + ), mock.patch.object( + utils, + "get_terraform_state", + return_value=state, + side_effect=state_error, + ): + vpc_module.start_vpc(vpc) + + def test_state_read_failure_after_apply_keeps_vpc_managed(self): + for error in (STATE_PULL_FAILED, RuntimeError("bad state")): + with self.subTest(error=type(error).__name__): + vpc, _ = _fake_vpc() + with self.assertRaises(type(error)): + self._start(vpc, state_error=error) + self.assertEqual(vpc.cloud_state, "m") + vpc.save.assert_called() + + def test_missing_output_after_apply_keeps_vpc_managed(self): + vpc, _ = _fake_vpc() + with self.assertRaises(KeyError): + self._start(vpc, state={"outputs": {}}) + self.assertEqual(vpc.cloud_state, "m") + + def test_apply_failure_marks_vpc_unknown_not_creating(self): + vpc, _ = _fake_vpc() + with self.assertRaises(subprocess.CalledProcessError): + self._start(vpc, apply_error=APPLY_FAILED) + self.assertEqual(vpc.cloud_state, "um") + vpc.save.assert_called() + + def test_success_records_ids(self): + vpc, subnet = _fake_vpc() + state = { + "outputs": { + "vpc_id": {"value": "net-1"}, + "subnet-3": {"value": "sub-3"}, + } + } + self._start(vpc, state=state) + self.assertEqual(vpc.cloud_state, "m") + self.assertEqual(vpc.cloud_id, "net-1") + self.assertEqual(subnet.cloud_id, "sub-3") + self.assertEqual(subnet.cloud_state, "m") + + +def _cluster_info(cloud_state): + """A ClusterInfo with every step that touches disk or the cloud mocked.""" + info = object.__new__(clusterinfo.ClusterInfo) + info.cluster = mock.MagicMock() + info.cluster.cloud_state = cloud_state + info.cluster.name = "c1" + info.cluster_dir = Path("/x") + for step in ( + "_create_cluster_dir", + "_set_credentials", + "update", + "_run_ghpc", + "_initialize_terraform", + ): + setattr(info, step, mock.MagicMock()) + return info + + +class ClusterPostApplyTests(unittest.TestCase): + """A cluster stays managed once its apply has succeeded.""" + + def _start(self, info, state_error=STATE_PULL_FAILED): + with mock.patch.object(utils, "run_terraform"), mock.patch.object( + utils, "get_terraform_state", side_effect=state_error + ): + info.start_cluster(None) + + def test_state_read_failure_after_create_keeps_cluster_managed(self): + for error in (STATE_PULL_FAILED, RuntimeError("bad state")): + with self.subTest(error=type(error).__name__): + info = _cluster_info("nm") + with self.assertRaises(type(error)): + self._start(info, state_error=error) + self.assertEqual(info.cluster.cloud_state, "m") + self.assertEqual(info.cluster.status, "e") + self.assertEqual(info.cluster.internal_name, "c1") + + def test_failure_before_apply_leaves_new_cluster_unmanaged(self): + info = _cluster_info("nm") + info._run_ghpc.side_effect = RuntimeError("gcluster failed") + with self.assertRaises(RuntimeError): + info.start_cluster(None) + self.assertEqual(info.cluster.cloud_state, "nm") + self.assertEqual(info.cluster.status, "e") + + def test_retry_of_running_cluster_keeps_it_managed(self): + info = _cluster_info("m") + info._run_ghpc.side_effect = RuntimeError("gcluster failed") + with self.assertRaises(RuntimeError): + info.start_cluster(None) + self.assertEqual(info.cluster.cloud_state, "m") + self.assertEqual(info.cluster.status, "e") + + def test_reconfigure_failure_keeps_running_cluster_managed(self): + for failing_step in ("update", "_run_ghpc", "_initialize_terraform"): + with self.subTest(step=failing_step): + info = _cluster_info("m") + getattr(info, failing_step).side_effect = RuntimeError("x") + with self.assertRaises(RuntimeError): + info.reconfigure_cluster() + self.assertEqual(info.cluster.cloud_state, "m") + self.assertEqual(info.cluster.status, "e") + + def test_reconfigure_state_read_failure_keeps_cluster_managed(self): + info = _cluster_info("m") + with mock.patch.object(utils, "run_terraform"), mock.patch.object( + utils, "get_terraform_state", side_effect=STATE_PULL_FAILED + ): + with self.assertRaises(subprocess.CalledProcessError): + info.reconfigure_cluster() + self.assertEqual(info.cluster.cloud_state, "m") + self.assertEqual(info.cluster.status, "e") + + +if __name__ == "__main__": + unittest.main() diff --git a/community/front-end/ofe/website/ghpcfe/tests/test_teardown_script.py b/community/front-end/ofe/website/ghpcfe/tests/test_teardown_script.py new file mode 100644 index 0000000000..cdd5edc32f --- /dev/null +++ b/community/front-end/ofe/website/ghpcfe/tests/test_teardown_script.py @@ -0,0 +1,608 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the state-bucket logic in teardown.sh. + +Deleting the state bucket loses the state of every resource still recorded in +it, so teardown.sh deletes it only on request, and only when it is provably +empty. + +Like test_deploy_script.py, these run the real teardown.sh, or functions +extracted from it by brace-matching, under bash in a temporary directory with +stub external tools. Standard library only (no Django). +""" + +import json +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path + +from .test_deploy_script import OFE_DIR, _extract_bash_function + +TEARDOWN_SH = OFE_DIR / "teardown.sh" + + +class StateBucketDirTests(unittest.TestCase): + """teardown.sh reports the bucket of the deployment being torn down, + from that deployment's own directory, or from tf/state_bucket/ if an + earlier release left that same deployment's files there. + """ + + FUNCTION_BODY = _extract_bash_function(TEARDOWN_SH, "state_bucket_dir") + + def _run(self, files: dict[str, str], name: str = "dep1") -> str: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for path, content in files.items(): + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_text(content) + result = subprocess.run( + ["bash", "-c", f'{self.FUNCTION_BODY}\nstate_bucket_dir "{name}"'], + cwd=root, + capture_output=True, + text=True, + check=True, + timeout=30, + ) + return result.stdout.strip() + + def test_own_directory(self): + files = { + "tf/state_bucket/deployments/dep1/terraform.tfstate": "{}", + "tf/state_bucket/deployments/dep2/terraform.tfstate": "{}", + } + self.assertEqual(self._run(files), "tf/state_bucket/deployments/dep1") + + def test_earlier_release_files_of_this_deployment(self): + files = { + "tf/state_bucket/terraform.tfstate": "{}", + "tf/state_bucket/terraform.tfvars": 'deployment_name = "dep1"\n', + } + self.assertEqual(self._run(files), "tf/state_bucket") + + def test_earlier_release_files_of_another_deployment_are_not_used(self): + files = { + "tf/state_bucket/terraform.tfstate": "{}", + "tf/state_bucket/terraform.tfvars": 'deployment_name = "dep10"\n', + } + self.assertEqual(self._run(files), "") + + def test_dot_names_are_never_resolved(self): + files = {"tf/state_bucket/terraform.tfstate": "{}"} + for name in (".", ".."): + with self.subTest(name=name): + self.assertEqual(self._run(files, name=name), "") + + def test_no_state(self): + self.assertEqual(self._run({}), "") + + +class TeardownSandbox(unittest.TestCase): + """Runs the real teardown.sh, or functions from it, in a temporary + directory with stub gcloud, terraform and script/service_account.sh. + + The stub gcloud serves the bucket's objects from `objects` (name -> + content); the stub terraform records every call with its -chdir, and on + `apply` copies the main.tf it was given so a test can check what was + applied. tf/state_bucket/ holds the real tracked .tf files. + """ + + BUCKET = "dep1-tfstate-abcd1234" + EMPTY = json.dumps({"version": 4, "resources": []}) + BUSY = json.dumps( + {"version": 4, "resources": [{"mode": "managed"}, {"mode": "managed"}]} + ) + + def _run( + self, + args: str = "", + *, + objects: dict[str, str] | None = None, + list_fails: bool = False, + fe_deployed: bool = True, + lock: bool = True, + show_fails: bool = False, + tf_tfvars: str | None = 'project_id = "proj"\ndeployment_name = "dep1"\n', + stdin: str = "", + script: str | None = None, + noncurrent: list[str] | None = None, + busy_after_apply: bool = False, + plan_actions: tuple[str, ...] = ("update",), + interrupt_on_apply: bool = False, + ) -> dict: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "bin").mkdir() + (root / "script").mkdir() + (root / "objects").mkdir() + sb_root = root / "tf" / "state_bucket" + sb_dir = sb_root / "deployments" / "dep1" + sb_dir.mkdir(parents=True) + for tf_file in (OFE_DIR / "tf" / "state_bucket").glob("*.tf"): + (sb_root / tf_file.name).write_text(tf_file.read_text()) + (sb_dir / tf_file.name).symlink_to(f"../../{tf_file.name}") + (sb_dir / "terraform.tfstate").write_text('{"bucket": "in state"}') + (sb_dir / "terraform.tfvars").write_text('deployment_name = "dep1"\n') + (sb_dir / "admin.auto.tfvars").write_text("state_bucket_writers = []\n") + if tf_tfvars is not None: + (root / "tf" / "terraform.tfvars").write_text(tf_tfvars) + if lock: + (root / "tf" / ".tkfe.lock").write_text("") + for name, content in (objects or {}).items(): + path = root / "objects" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + # Noncurrent versions are listed by name with a noncurrent time. + (root / "noncurrent").write_text( + "".join(f"{name}\t2026-10-20T10:00:00+0000\n" for name in noncurrent or []) + ) + + def stub(path: Path, body: str) -> None: + path.write_text("#!/bin/bash\n" + body) + path.chmod(path.stat().st_mode | stat.S_IEXEC) + + stub(root / "script" / "service_account.sh", "exit 1\n") + # Kept out of the f-string below: Python 3.11 rejects a backslash + # inside an f-string expression. + show_action = "echo ' ghpcfe_id = \"dep1\"'" if fe_deployed else ":" + if show_fails: + # As with tf/ not initialised: no provider schemas. + show_action = 'echo "Error: Failed to load plugin schemas" >&2; exit 1' + listing = ( + 'echo "ERROR: denied" >&2; exit 1' + if list_fails + else f'(cd "{root}/objects" && find . -type f | sed "s|^./||"; cat "{root}/noncurrent")' + ) + stub( + root / "bin" / "gcloud", + f'echo "$*" >> "{root}/gcloud_calls.log"\n' + 'case "$1 $2 $3" in\n' + ' "storage --help "*) exit 0 ;;\n' + f' "storage objects list") {listing} ;;\n' + f' "storage cat "*) cat "{root}/objects/${{3#gs://{self.BUCKET}/}}" ;;\n' + "esac\n", + ) + stub( + root / "bin" / "terraform", + 'dir=.; case "$1" in -chdir=*) dir=${1#-chdir=}; shift ;; esac\n' + f'echo "$dir $*" >> "{root}/tf_calls.log"\n' + 'case "$1" in\n' + f' output) echo {self.BUCKET} ;;\n' + ' plan) : ;;\n' + "esac\n" + # Only `show -json` reads the plan; plain `show` is the front + # end's state, answered below. + + '[ "$1 $2" = "show -json" ] && echo \'' + + json.dumps( + { + "resource_changes": [ + { + "address": "google_storage_bucket.tfstate", + "change": {"actions": list(plan_actions)}, + } + ] + } + ) + + "' && exit 0\n" + 'case "$1" in\n' + f" show) {show_action} ;;\n" + f' apply) cp "$dir/main.tf" "{root}/applied_main.tf"' + + ( + # As on Ctrl-C: the run recorded something, then the + # script is interrupted. + """; echo '{"interrupted": true}' > "$dir/terraform.tfstate"; """ + "kill -INT $PPID" + if interrupt_on_apply + else "" + ) + + ( + f'; mkdir -p "{root}/objects/ns/vpcs/vpc_9"; ' + f"echo '{self.BUSY}' > \"{root}/objects/ns/vpcs/vpc_9/default.tfstate\"" + if busy_after_apply + else "" + ) + + " ;;\n" + ' destroy) [ "$dir" = . ] || ' + 'echo \'{"destroyed": true}\' > "$dir/terraform.tfstate" ;;\n' + "esac\n", + ) + (root / "tmpdir").mkdir() + env = { + "PATH": f"{root / 'bin'}:/usr/bin:/bin", + "HOME": tmp, + "TMPDIR": str(root / "tmpdir"), + } + if script is None: + (root / "teardown.sh").write_text(TEARDOWN_SH.read_text()) + command = ["bash", str(root / "teardown.sh"), *args.split()] + else: + command = ["bash", "-c", script] + result = subprocess.run( + command, + cwd=root, + env=env, + input=stdin, + capture_output=True, + text=True, + timeout=60, + ) + calls = root / "tf_calls.log" + gcalls = root / "gcloud_calls.log" + applied = root / "applied_main.tf" + return { + "rc": result.returncode, + "out": result.stdout + result.stderr, + "tf_calls": calls.read_text().splitlines() if calls.exists() else [], + "gcloud_calls": gcalls.read_text().splitlines() if gcalls.exists() else [], + "applied_main_tf": applied.read_text() if applied.exists() else None, + "main_tf": (sb_root / "main.tf").read_text(), + "state": (sb_dir / "terraform.tfstate").read_text(), + "tmp": tmp, + "tmp_left": sorted(p.name for p in (root / "tmpdir").iterdir()), + } + + @staticmethod + def _fe_destroyed(res) -> bool: + return ". destroy -auto-approve" in res["tf_calls"] + + @staticmethod + def _bucket_destroyed(res) -> bool: + return any( + c.split()[1:2] == ["destroy"] and not c.startswith(". ") + for c in res["tf_calls"] + ) + + +class CheckStateBucketEmptyTests(TeardownSandbox): + """The bucket counts as empty only if every state in it, under every + namespace and prefix, provably records no resources and none is + locked. + """ + + FUNCTION_BODY = _extract_bash_function(TEARDOWN_SH, "check_state_bucket_empty") + + def _check(self, **kw) -> dict: + script = f'{self.FUNCTION_BODY}\ncheck_state_bucket_empty "{self.BUCKET}"' + return self._run(script=script, **kw) + + def test_all_states_empty(self): + res = self._check( + objects={ + "20261020-0a1b2c3d/vpcs/vpc_1/default.tfstate": self.EMPTY, + "20261019-99999999/clusters/cluster_2/default.tfstate": self.EMPTY, + } + ) + self.assertEqual(res["rc"], 0, res["out"]) + + def test_empty_bucket(self): + self.assertEqual(self._check(objects={})["rc"], 0) + + def test_state_with_resources_in_any_namespace_is_listed(self): + res = self._check( + objects={ + "20261020-0a1b2c3d/vpcs/vpc_1/default.tfstate": self.EMPTY, + "20261019-99999999/clusters/cluster_2/default.tfstate": self.BUSY, + } + ) + self.assertNotEqual(res["rc"], 0) + self.assertIn( + "20261019-99999999/clusters/cluster_2/default.tfstate (2 resources)", + res["out"], + ) + self.assertNotIn("vpc_1", res["out"]) + + def test_unreadable_state_is_not_treated_as_empty(self): + res = self._check(objects={"ns/fs/fs_1/default.tfstate": "not json"}) + self.assertNotEqual(res["rc"], 0) + self.assertIn("ns/fs/fs_1/default.tfstate (could not be read)", res["out"]) + + def test_lock_is_not_treated_as_empty(self): + res = self._check( + objects={ + "ns/vpcs/vpc_1/default.tfstate": self.EMPTY, + "ns/vpcs/vpc_1/default.tflock": "{}", + } + ) + self.assertNotEqual(res["rc"], 0) + self.assertIn("default.tflock (locked", res["out"]) + + def test_noncurrent_versions_are_ignored(self): + # Every released lock and superseded state leaves a noncurrent + # version in the versioned bucket; only live objects count. + res = self._check( + objects={"ns/vpcs/vpc_1/default.tfstate": self.EMPTY}, + noncurrent=["ns/vpcs/vpc_1/default.tflock", "ns/vpcs/vpc_1/default.tfstate"], + ) + self.assertEqual(res["rc"], 0, res["out"]) + cats = [c for c in res["gcloud_calls"] if c.startswith("storage cat")] + self.assertEqual(len(cats), 1, res["gcloud_calls"]) + + def test_listing_failure_is_not_treated_as_empty(self): + res = self._check(list_fails=True) + self.assertNotEqual(res["rc"], 0) + self.assertIn("could not list", res["out"]) + + +class DeleteStateBucketTests(TeardownSandbox): + """teardown.sh --delete-state-bucket deletes the bucket only when it is + provably empty and confirmed, from a temporary copy of the root, and + plain teardown.sh still keeps the bucket. + """ + + def test_plain_teardown_keeps_the_bucket(self): + res = self._run("-y", objects={"ns/vpcs/vpc_1/default.tfstate": self.EMPTY}) + self.assertEqual(res["rc"], 0, res["out"]) + self.assertTrue(self._fe_destroyed(res)) + self.assertFalse(self._bucket_destroyed(res)) + self.assertFalse( + any("objects list" in c for c in res["gcloud_calls"]), + res["gcloud_calls"], + ) + self.assertIn("has been left in place", res["out"]) + self.assertIn("./teardown.sh --delete-state-bucket dep1", res["out"]) + + def test_refuses_before_destroying_anything_when_a_state_has_resources(self): + res = self._run( + "-y --delete-state-bucket", + objects={"ns/clusters/cluster_3/default.tfstate": self.BUSY}, + ) + self.assertNotEqual(res["rc"], 0) + self.assertIn("ns/clusters/cluster_3/default.tfstate (2 resources)", res["out"]) + self.assertFalse(self._fe_destroyed(res)) + self.assertFalse(self._bucket_destroyed(res)) + self.assertIsNone(res["applied_main_tf"]) + + def test_deletes_an_empty_bucket_after_the_front_end(self): + res = self._run( + "-y --delete-state-bucket", + objects={"ns/vpcs/vpc_1/default.tfstate": self.EMPTY}, + ) + self.assertEqual(res["rc"], 0, res["out"]) + bucket_calls = [c for c in res["tf_calls"] if not c.startswith(". ")] + bucket_verbs = [c.split()[1] for c in bucket_calls] + verbs = [v for v in bucket_verbs if v in ("init", "apply", "destroy")] + self.assertEqual(verbs, ["init", "apply", "destroy"]) + apply_call = bucket_calls[bucket_verbs.index("apply")] + plan_call = bucket_calls[bucket_verbs.index("plan")] + self.assertIn("-target=google_storage_bucket.tfstate", plan_call) + self.assertIn("-out=target.tfplan", plan_call) + # Exactly the planned change is applied. + self.assertTrue(apply_call.endswith(" target.tfplan"), apply_call) + tf_index = res["tf_calls"].index(". destroy -auto-approve") + self.assertLess(tf_index, res["tf_calls"].index(bucket_calls[-1])) + self.assertIn(f"gs://{self.BUCKET} has been deleted", res["out"]) + + def test_protection_is_lifted_only_in_a_temporary_copy(self): + res = self._run("-y --delete-state-bucket", objects={}) + self.assertEqual(res["rc"], 0, res["out"]) + tracked = (OFE_DIR / "tf" / "state_bucket" / "main.tf").read_text() + self.assertEqual(res["main_tf"], tracked) + applied = res["applied_main_tf"] + self.assertNotIn("prevent_destroy", applied) + self.assertIn(" force_destroy = true\n", applied) + self.assertIn("lifecycle_rule {", applied) + for call in res["tf_calls"]: + if call.split()[1] in ("init", "apply", "destroy"): + self.assertFalse(call.startswith("tf/"), call) + # The copy's state, recording what is left, replaces the local one. + self.assertEqual(res["state"], '{"destroyed": true}\n') + + def test_rechecked_immediately_before_the_destroy(self): + # A state written after the first check, here during the targeted + # apply, must stop the destroy, which would delete every object. + res = self._run("-y --delete-state-bucket", objects={}, busy_after_apply=True) + self.assertNotEqual(res["rc"], 0) + self.assertIn("ns/vpcs/vpc_9/default.tfstate (2 resources)", res["out"]) + self.assertIsNotNone(res["applied_main_tf"]) + self.assertFalse(self._bucket_destroyed(res)) + # The bucket stays, so its protection is applied back from the + # tracked main.tf. + applies = [c for c in res["tf_calls"] if c.split()[1:2] == ["apply"]] + self.assertEqual(len(applies), 2, res["tf_calls"]) + self.assertIn("-target=google_storage_bucket.tfstate", applies[1]) + self.assertIn(" force_destroy = false\n", res["applied_main_tf"]) + self.assertIn("prevent_destroy = true", res["applied_main_tf"]) + # The copy's state is still copied back. + self.assertEqual(res["state"], '{"bucket": "in state"}') + + def test_refuses_anything_but_an_in_place_update(self): + # e.g. the region in tfvars no longer matches the bucket's location, + # so lifting the protection would replace the bucket. + for actions in (("delete", "create"), ("create",)): + with self.subTest(actions=actions): + res = self._run("-y --delete-state-bucket", objects={}, plan_actions=actions) + self.assertNotEqual(res["rc"], 0) + self.assertIn(f"would change gs://{self.BUCKET} ({','.join(actions)})", res["out"]) + self.assertIsNone(res["applied_main_tf"]) + self.assertFalse(self._bucket_destroyed(res)) + self.assertEqual(res["tmp_left"], []) + + def test_interrupt_keeps_the_recorded_state_and_removes_the_copy(self): + res = self._run("-y --delete-state-bucket", objects={}, interrupt_on_apply=True) + self.assertEqual(res["rc"], 130, res["out"]) + self.assertIn("Interrupted", res["out"]) + self.assertEqual(res["state"], '{"interrupted": true}\n') + self.assertEqual(res["tmp_left"], []) + self.assertFalse(self._bucket_destroyed(res)) + + def test_typed_confirmation_is_required_without_y(self): + for reply, deleted in (("y\n", False), (f"{self.BUCKET}\n", True)): + with self.subTest(reply=reply): + # `reply` answers the bucket-name prompt and "y" the final + # Proceed prompt. The lock-file and service-account prompts + # are not reached (lock file present, no service account). + res = self._run( + "--delete-state-bucket", objects={}, stdin=reply + "y\n" + ) + self.assertEqual(res["rc"], 0, res["out"]) + self.assertEqual(self._bucket_destroyed(res), deleted, res["out"]) + self.assertEqual(self._fe_destroyed(res), deleted, res["out"]) + + def test_bucket_only_when_the_front_end_is_already_gone(self): + res = self._run("-y --delete-state-bucket", objects={}, fe_deployed=False, lock=False) + self.assertEqual(res["rc"], 0, res["out"]) + self.assertFalse(self._fe_destroyed(res)) + self.assertTrue(self._bucket_destroyed(res)) + + def test_named_earlier_deployment_touches_only_its_bucket(self): + res = self._run("--delete-state-bucket dep0", objects={}) + self.assertNotEqual(res["rc"], 0) + self.assertIn("no state bucket for deployment 'dep0'", res["out"]) + self.assertFalse(self._fe_destroyed(res)) + self.assertEqual([c for c in res["tf_calls"] if " show" in c], []) + + def test_named_deployment_still_deployed_here_is_refused(self): + # The named form deletes only a bucket; it must never fall through to + # tearing down the FrontEnd, even with -y. + res = self._run("-y --delete-state-bucket dep1", objects={}) + self.assertNotEqual(res["rc"], 0) + self.assertIn("'dep1' may still be deployed from here", res["out"]) + self.assertFalse(self._fe_destroyed(res)) + self.assertFalse(self._bucket_destroyed(res)) + self.assertFalse(any("objects list" in c for c in res["gcloud_calls"]), res["gcloud_calls"]) + + def test_named_deployment_torn_down_here_deletes_only_its_bucket(self): + res = self._run( + "-y --delete-state-bucket dep1", objects={}, fe_deployed=False, lock=False + ) + self.assertEqual(res["rc"], 0, res["out"]) + self.assertFalse(self._fe_destroyed(res)) + self.assertTrue(self._bucket_destroyed(res)) + self.assertIn(f"gs://{self.BUCKET} has been deleted", res["out"]) + + def _assert_refused_untouched(self, res, text): + self.assertNotEqual(res["rc"], 0, res["out"]) + self.assertIn(text, res["out"]) + self.assertFalse(self._fe_destroyed(res)) + self.assertFalse(self._bucket_destroyed(res)) + self.assertIsNone(res["applied_main_tf"]) + + def test_named_deployment_with_unreadable_front_end_state_is_refused(self): + # A failing `terraform show` must not read as "no FrontEnd". + res = self._run( + "-y --delete-state-bucket dep1", objects={}, show_fails=True, lock=False + ) + self._assert_refused_untouched(res, "'dep1' may still be deployed from here") + + def test_named_deployment_with_lock_file_is_refused(self): + res = self._run( + "-y --delete-state-bucket dep1", objects={}, fe_deployed=False, lock=True + ) + self._assert_refused_untouched(res, "'dep1' may still be deployed from here") + + def test_named_deployment_shown_without_lock_is_refused(self): + res = self._run( + "-y --delete-state-bucket dep1", objects={}, fe_deployed=True, lock=False + ) + self._assert_refused_untouched(res, "'dep1' may still be deployed from here") + + def test_named_deployment_with_reformatted_tfvars_is_refused(self): + # A hand-edited name line must not make the guard skip this FrontEnd. + res = self._run( + "-y --delete-state-bucket dep1", + objects={}, + fe_deployed=True, + lock=False, + tf_tfvars='project_id = "proj"\ndeployment_name = "dep1" # edited\n', + ) + self._assert_refused_untouched(res, "'dep1' may still be deployed from here") + + def test_named_deployment_with_unreadable_tfvars_name_is_refused(self): + res = self._run( + "-y --delete-state-bucket dep1", + objects={}, + fe_deployed=False, + lock=True, + tf_tfvars='project_id = "proj"\n', + ) + self._assert_refused_untouched(res, "'dep1' may still be deployed from here") + + def _assert_refused_before_prompts(self, res): + self._assert_refused_untouched(res, "cannot confirm that no FrontEnd is deployed") + self.assertIn("nothing\n has been changed", res["out"]) + self.assertNotIn("Type the bucket name", res["out"]) + self.assertNotIn("This will permanently delete", res["out"]) + self.assertFalse( + any("objects list" in c for c in res["gcloud_calls"]), res["gcloud_calls"] + ) + + def test_bare_form_refuses_before_any_prompt_or_change(self): + # stdin answers only the "No lock file found ... Proceed anyway?" prompt. + res = self._run( + "--delete-state-bucket", objects={}, show_fails=True, lock=False, stdin="y\n" + ) + self._assert_refused_before_prompts(res) + + def test_bare_form_with_lock_refuses_before_any_prompt_or_change(self): + res = self._run("-y --delete-state-bucket", objects={}, fe_deployed=False, lock=True) + self._assert_refused_before_prompts(res) + + def test_refusals_say_how_to_clear_a_leftover_lock(self): + for args in ("-y --delete-state-bucket dep1", "-y --delete-state-bucket"): + with self.subTest(args=args): + res = self._run(args, objects={}, fe_deployed=False, lock=True) + self.assertNotEqual(res["rc"], 0) + self.assertIn("leftover lock file tf/.tkfe.lock and rerun", res["out"]) + + def test_named_form_parses_a_reformatted_name_for_another_deployment(self): + # tf/ names "other", so dep1's bucket is not this FrontEnd's: deleted + # even while that FrontEnd runs. Without the parse this would refuse. + res = self._run( + "-y --delete-state-bucket dep1", + objects={}, + fe_deployed=True, + lock=True, + tf_tfvars='project_id = "proj"\ndeployment_name = "other" # edited\n', + ) + self.assertEqual(res["rc"], 0, res["out"]) + self.assertTrue(self._bucket_destroyed(res)) + self.assertFalse(self._fe_destroyed(res)) + + def test_named_form_without_tfvars_is_refused_while_front_end_may_run(self): + res = self._run( + "-y --delete-state-bucket dep1", + objects={}, + fe_deployed=False, + lock=True, + tf_tfvars=None, + ) + self._assert_refused_untouched(res, "'dep1' may still be deployed from here") + + def test_named_form_without_tfvars_deletes_when_nothing_was_deployed(self): + res = self._run( + "-y --delete-state-bucket dep1", + objects={}, + fe_deployed=False, + lock=False, + tf_tfvars=None, + ) + self.assertEqual(res["rc"], 0, res["out"]) + self.assertTrue(self._bucket_destroyed(res)) + + def test_bucket_only_with_unreadable_front_end_state_is_refused(self): + res = self._run("-y --delete-state-bucket", objects={}, show_fails=True, lock=False) + self._assert_refused_untouched(res, "cannot confirm that no FrontEnd is deployed") + + def test_bucket_only_with_lock_file_is_refused(self): + res = self._run("-y --delete-state-bucket", objects={}, fe_deployed=False, lock=True) + self._assert_refused_untouched(res, "cannot confirm that no FrontEnd is deployed") + + def test_unknown_option_is_rejected(self): + res = self._run("--delete-state-buckets") + self.assertNotEqual(res["rc"], 0) + self.assertIn("Unknown option", res["out"]) + self.assertEqual(res["tf_calls"], []) diff --git a/community/front-end/ofe/website/ghpcfe/tests/test_tf_state_backend.py b/community/front-end/ofe/website/ghpcfe/tests/test_tf_state_backend.py new file mode 100644 index 0000000000..2630af5f17 --- /dev/null +++ b/community/front-end/ofe/website/ghpcfe/tests/test_tf_state_backend.py @@ -0,0 +1,735 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for storing Terraform state in a GCS bucket. + +Every resource the front end provisions must write its Terraform state to the +shared state bucket rather than to a terraform.tfstate on the server's local +disk. State on local disk is lost when the server is rebuilt, and a resource +whose state is lost cannot be destroyed by Terraform. + +These tests are deliberately importable WITHOUT Django. The repository's +pre-commit configuration runs a bare `pytest` from the repository root on any +changed Python file, with no rootdir configuration and no DJANGO_SETTINGS_MODULE, +so a module that imported Django (or any ghpcfe module that does) at import time +would error during collection in CI. `utils` itself imports only the standard +library and PyYAML, so it is loaded here directly by path. filesystem.py is +loaded the same way, with its Django-dependent imports (models, site settings) +stubbed. +""" + +import importlib.util +import json +import re +import subprocess +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest import mock + +import yaml + +OFE_DIR = Path(__file__).resolve().parents[3] +CLUSTER_MANAGER = OFE_DIR / "website" / "ghpcfe" / "cluster_manager" + + +def _load_utils(): + """Import cluster_manager/utils.py without importing the ghpcfe package.""" + spec = importlib.util.spec_from_file_location( + "ofe_cluster_manager_utils", CLUSTER_MANAGER / "utils.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +utils = _load_utils() + + +class TerraformStatePrefixTests(unittest.TestCase): + """The prefix scheme partitions state per database and per resource. + + Every prefix is scoped under the database's state namespace, so a fresh + database (whose primary keys restart at 1) cannot reuse an earlier + database's prefixes. Below the namespace, prefixes mirror the + per-resource working-directory names the application creates on disk. + """ + + NS = "20261015-0a1b2c3d" + + def test_prefix_per_resource_kind(self): + expected = { + "cluster": "clusters/cluster_7", + "filesystem": "fs/fs_7", + "vpc": "vpcs/vpc_7", + "workbench": "workbenches/workbench_7", + "image": "images/image_7", + } + for kind, prefix in expected.items(): + with self.subTest(kind=kind): + self.assertEqual( + utils.terraform_state_prefix(kind, 7, namespace=self.NS), + f"{self.NS}/{prefix}", + ) + + def test_default_namespace_comes_from_the_database(self): + with mock.patch.object( + utils, "get_state_namespace", return_value=self.NS + ) as get_ns: + self.assertEqual( + utils.terraform_state_prefix("cluster", 1), + f"{self.NS}/clusters/cluster_1", + ) + get_ns.assert_called_once_with() + + def test_same_id_in_different_databases_does_not_collide(self): + self.assertNotEqual( + utils.terraform_state_prefix("cluster", 1, namespace=self.NS), + utils.terraform_state_prefix("cluster", 1, namespace="20261016-ffffffff"), + ) + + def test_new_namespace_format_and_uniqueness(self): + first, second = utils.new_state_namespace(), utils.new_state_namespace() + for ns in (first, second): + self.assertRegex(ns, r"^\d{8}-[0-9a-f]{8}$") + self.assertNotEqual(first, second) + + def test_every_provisioned_kind_has_a_prefix(self): + # All five resource types the front end provisions with Terraform. + self.assertEqual( + set(utils.TERRAFORM_STATE_PREFIXES), + {"cluster", "filesystem", "vpc", "workbench", "image"}, + ) + + def test_prefixes_are_distinct(self): + prefixes = [ + utils.terraform_state_prefix(kind, 1, namespace=self.NS) + for kind in utils.TERRAFORM_STATE_PREFIXES + ] + self.assertEqual(len(prefixes), len(set(prefixes))) + + def test_unknown_kind_is_rejected_before_any_database_lookup(self): + with mock.patch.object(utils, "get_state_namespace") as get_ns: + with self.assertRaises(ValueError): + utils.terraform_state_prefix("not-a-resource", 1) + get_ns.assert_not_called() + + +class StateBucketConfigTests(unittest.TestCase): + """The bucket name reaches the application through configuration.yaml.""" + + def test_returns_configured_bucket(self): + config = {"server": {"terraform_state_bucket": "my-deploy-tfstate-ab12"}} + with mock.patch.object(utils, "load_config", return_value=config): + self.assertEqual( + utils.get_terraform_state_bucket(), "my-deploy-tfstate-ab12" + ) + + def test_missing_bucket_is_fatal(self): + # Falling back to local state would reinstate exactly the risk the + # bucket exists to remove, so this must raise rather than default. + for server in ({}, {"terraform_state_bucket": ""}): + with self.subTest(server=server): + with mock.patch.object( + utils, "load_config", return_value={"server": server} + ): + with self.assertRaises(RuntimeError): + utils.get_terraform_state_bucket() + + +class WriteBackendTfTests(unittest.TestCase): + """Raw-template resources get a backend.tf at template-copy time.""" + + BUCKET = "my-deploy-tfstate-ab12" + + def _write(self, target_dir, prefix): + with mock.patch.object( + utils, "get_terraform_state_bucket", return_value=self.BUCKET + ): + return utils.write_backend_tf(target_dir, prefix) + + def test_writes_gcs_backend_with_bucket_and_prefix(self): + with tempfile.TemporaryDirectory() as tmp: + path = self._write(tmp, "vpcs/vpc_3") + self.assertEqual(path, Path(tmp) / "backend.tf") + content = path.read_text() + self.assertIn('backend "gcs"', content) + self.assertIn(f'bucket = "{self.BUCKET}"', content) + self.assertIn('prefix = "vpcs/vpc_3"', content) + + def test_is_idempotent(self): + # create_subnet re-enters a VPC directory created earlier, so writing + # the backend again must leave it unchanged rather than duplicating it. + with tempfile.TemporaryDirectory() as tmp: + first = self._write(tmp, "vpcs/vpc_3").read_text() + second = self._write(tmp, "vpcs/vpc_3").read_text() + self.assertEqual(first, second) + self.assertEqual(first.count("backend"), 1) + + +class GetTerraformStateTests(unittest.TestCase): + """State is read through `terraform state pull`, not from a local file.""" + + STATE = {"version": 4, "outputs": {"vpc_id": {"value": "my-net"}}} + + def test_parses_pulled_state(self): + completed = subprocess.CompletedProcess( + args=[], returncode=0, stdout=json.dumps(self.STATE).encode() + ) + with mock.patch.object(subprocess, "run", return_value=completed) as run: + state = utils.get_terraform_state("/some/dir", {"A": "b"}) + self.assertEqual(state["outputs"]["vpc_id"]["value"], "my-net") + cmdline = run.call_args.args[0] + self.assertEqual(cmdline, ["terraform", "state", "pull"]) + + def test_credentials_are_passed_through(self): + completed = subprocess.CompletedProcess( + args=[], returncode=0, stdout=json.dumps(self.STATE).encode() + ) + creds = {"GOOGLE_APPLICATION_CREDENTIALS": "/tmp/creds.json"} + with mock.patch.object(subprocess, "run", return_value=completed) as run: + utils.get_terraform_state("/some/dir", creds) + env = run.call_args.kwargs["env"] + self.assertEqual( + env["GOOGLE_APPLICATION_CREDENTIALS"], "/tmp/creds.json" + ) + + def test_unparseable_state_raises(self): + completed = subprocess.CompletedProcess( + args=[], returncode=0, stdout=b"not json" + ) + with mock.patch.object(subprocess, "run", return_value=completed): + with self.assertRaises(RuntimeError): + utils.get_terraform_state("/some/dir") + + +class RunTerraformInputTests(unittest.TestCase): + """Unattended terraform runs must never wait for input. + + A backend change (e.g. the first init of a resource created before state + moved to GCS) makes `terraform init` ask whether to migrate state. Without + -input=false that prompt blocks the provisioning task forever; with it, + Terraform fails and says why. validate and output reject the flag. + """ + + def _cmdline(self, command): + with tempfile.TemporaryDirectory() as tmp: + with mock.patch.object(subprocess, "run") as run: + utils.run_terraform(tmp, command) + return run.call_args.args[0] + + def test_prompting_commands_get_input_false(self): + for command in ("init", "plan", "apply", "destroy"): + with self.subTest(command=command): + self.assertIn("-input=false", self._cmdline(command)) + + def test_commands_without_the_flag_do_not_get_it(self): + for command in ("validate", "output"): + with self.subTest(command=command): + self.assertNotIn("-input=false", self._cmdline(command)) + + +class BlueprintBackendTests(unittest.TestCase): + """Cluster Toolkit blueprints carry terraform_backend_defaults. + + The Toolkit writes the backend into every Terraform deployment group it + generates (pkg/config/expand.go, pkg/modulewriter/tfwriter.go), so setting + it at the top of the blueprint is all the generated clusters, filesystems + and image-builder environments need. + """ + + TEMPLATE = ( + OFE_DIR + / "website" + / "ghpcfe" + / "templates" + / "blueprint" + / "cluster_config.yaml.j2" + ) + + def test_cluster_template_declares_backend(self): + text = self.TEMPLATE.read_text() + self.assertIn("terraform_backend_defaults:", text) + self.assertIn("type: gcs", text) + self.assertIn("bucket: {{ state_bucket }}", text) + self.assertIn("prefix: {{ state_prefix }}", text) + + def test_cluster_backend_precedes_deployment_groups(self): + text = self.TEMPLATE.read_text() + self.assertLess( + text.index("terraform_backend_defaults:"), + text.index("deployment_groups:"), + ) + + def test_generators_declare_backend(self): + # filesystem.py and image.py build their blueprints as inline f-strings + # and cannot be imported without Django, so assert on their source. + for name in ("filesystem.py", "image.py"): + with self.subTest(module=name): + text = (CLUSTER_MANAGER / name).read_text() + self.assertIn("terraform_backend_defaults:", text) + self.assertIn("get_terraform_state_bucket()", text) + self.assertIn("terraform_state_prefix(", text) + + def test_single_terraform_group_per_blueprint(self): + # An explicit prefix is applied by the Toolkit to EVERY deployment + # group, so two Terraform groups in one blueprint would collide on a + # single state object. Each blueprint must keep exactly one. + self.assertEqual(self.TEMPLATE.read_text().count("- group:"), 1) + fs_text = (CLUSTER_MANAGER / "filesystem.py").read_text() + self.assertEqual(fs_text.count("- group:"), 1) + # The image blueprint has two groups, but the second is a Packer group + # and the Toolkit writes a backend only for Terraform groups. Modules + # with no explicit kind default to terraform (pkg/config/config.go + # addKindToModules), so the Terraform group is the one WITHOUT + # "kind: packer". + image_text = (CLUSTER_MANAGER / "image.py").read_text() + self.assertEqual(image_text.count("- group:"), 2) + self.assertEqual(image_text.count("kind: packer"), 1) + + +class RawTemplateBackendTests(unittest.TestCase): + """Networks and workbenches get their backend written at copy time.""" + + def test_backend_written_after_template_copy(self): + for name in ("vpc.py", "workbenchinfo.py"): + with self.subTest(module=name): + text = (CLUSTER_MANAGER / name).read_text() + self.assertIn("write_backend_tf(", text) + self.assertLess( + text.index("shutil.copytree"), + text.index("write_backend_tf("), + "backend must be written before any terraform init", + ) + + def test_no_module_reads_a_local_state_file(self): + for path in CLUSTER_MANAGER.glob("*.py"): + with self.subTest(module=path.name): + for line in path.read_text().splitlines(): + if line.lstrip().startswith("#") or '"""' in line: + continue + # .terraform/terraform.tfstate is the backend record + # `terraform init` writes, not the resource's state. + if '".terraform"' in line: + continue + self.assertNotIn( + 'terraform.tfstate"', line, + f"{path.name} still references a local state file", + ) + + +class TerraformInitCredentialsTests(unittest.TestCase): + """`terraform init` must authenticate as the provisioning credential. + + A GCS backend is contacted during init, so without + GOOGLE_APPLICATION_CREDENTIALS Terraform falls back to application default + credentials -- on the server, the front end's own service account, a + different identity from the one that provisions the resource and possibly + in a different project. + """ + + PROVISIONING_MODULES = ( + "clusterinfo.py", + "image.py", + "vpc.py", + "filesystem.py", + "workbenchinfo.py", + ) + + def test_every_init_passes_credentials(self): + # Matches a run_terraform(..., "init" ...) call and the rest of its + # argument list, across line breaks. + call = re.compile( + r"run_terraform\((?P[^()]*?\"init\"[^()]*?)\)", re.S + ) + for name in self.PROVISIONING_MODULES: + text = (CLUSTER_MANAGER / name).read_text() + calls = list(call.finditer(text)) + self.assertTrue(calls, f"no terraform init call found in {name}") + for match in calls: + with self.subTest(module=name): + self.assertIn( + "extra_env", + match.group("args"), + f"{name}: terraform init must pass extra_env, or it " + "will contact the state bucket as the wrong identity", + ) + + +def _load_filesystem(): + """Import cluster_manager/filesystem.py without Django. + + The module's only Django-dependent imports are the ghpcfe models and the + site settings, neither of which write_filestore_yaml touches. They are + stubbed in sys.modules for the duration of the import, and the module is + placed in a throwaway package so its relative imports resolve to the + stubs and to the already-loaded `utils`. + """ + pkg = types.ModuleType("ofe_stub") + pkg.__path__ = [] + manager = types.ModuleType("ofe_stub.cluster_manager") + manager.__path__ = [] + manager.utils = utils + models = types.ModuleType("ofe_stub.models") + for name in ("GCPFilestoreFilesystem", "Filesystem", "FilesystemImpl"): + setattr(models, name, mock.MagicMock()) + website = types.ModuleType("website") + website.__path__ = [] + settings = types.ModuleType("website.settings") + settings.SITE_NAME = "test-site" + stubs = { + "ofe_stub": pkg, + "ofe_stub.cluster_manager": manager, + "ofe_stub.cluster_manager.utils": utils, + "ofe_stub.models": models, + "website": website, + "website.settings": settings, + } + with mock.patch.dict(sys.modules, stubs): + spec = importlib.util.spec_from_file_location( + "ofe_stub.cluster_manager.filesystem", + CLUSTER_MANAGER / "filesystem.py", + ) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +filesystem = _load_filesystem() + + +class FilesystemBlueprintYamlTests(unittest.TestCase): + """The real Filestore blueprint writer emits the GCS state backend. + + Runs the real write_filestore_yaml against a stand-in filesystem object, + so no database or Django settings are needed. + """ + + NS = "20261015-0a1b2c3d" + + def test_generated_filesystem_yaml_has_backend(self): + fs = mock.MagicMock() + fs.id = 12 + fs.name = "testfs" + fs.cloud_credential.detail = json.dumps({"project_id": "proj"}) + fs.cloud_region = "europe-west4" + fs.cloud_zone = "europe-west4-a" + fs.vpc.cloud_id = "testnet" + fs.capacity = 1024 + fs.exports.first.return_value.export_name = "/share" + fs.get_performance_tier_display.return_value = "BASIC_HDD" + + with tempfile.TemporaryDirectory() as tmp: + with mock.patch.object( + utils, "get_terraform_state_bucket", return_value="bucket-x" + ), mock.patch.object( + utils, "get_state_namespace", return_value=self.NS + ): + filesystem.write_filestore_yaml(fs, Path(tmp)) + doc = yaml.safe_load((Path(tmp) / "filesystem.yaml").read_text()) + + backend = doc["terraform_backend_defaults"] + self.assertEqual(backend["type"], "gcs") + self.assertEqual(backend["configuration"]["bucket"], "bucket-x") + self.assertEqual( + backend["configuration"]["prefix"], f"{self.NS}/fs/fs_12" + ) + terraform_groups = [ + g for g in doc["deployment_groups"] + if any(m.get("kind") == "terraform" for m in g["modules"]) + ] + self.assertEqual(len(terraform_groups), 1) + self.assertEqual(len(doc["deployment_groups"]), 1) + + +class RunTerraformLockTimeoutTests(unittest.TestCase): + """A second operation on one resource waits for the state lock. + + Terraform's default -lock-timeout is 0, so without the flag a second + operation fails at once with "Error acquiring the state lock". Only the + commands that take the lock get it. + """ + + def _cmdline(self, command): + with tempfile.TemporaryDirectory() as tmp: + with mock.patch.object(subprocess, "run") as run: + utils.run_terraform(tmp, command) + return run.call_args.args[0] + + def test_locking_commands_get_lock_timeout(self): + for command in ("plan", "apply", "destroy", "import", "refresh"): + with self.subTest(command=command): + self.assertIn("-lock-timeout=45m", self._cmdline(command)) + + def test_other_commands_do_not_get_it(self): + for command in ("init", "validate", "output"): + with self.subTest(command=command): + self.assertFalse( + any(a.startswith("-lock-timeout") for a in + self._cmdline(command)) + ) + + +class StaleStateLockTests(unittest.TestCase): + """A provably stale lock fails fast with the recovery command. + + A lock is stale only if Terraform was killed while holding it. It is + reported only when it was taken on this host and no terraform process + here runs in the resource's directory; otherwise Terraform runs and + waits. The lock is never removed automatically. + """ + + HOST = "livetest1-server" + GENERATION = 1760566563123456 + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dir = Path(tmp.name) + self.creds = str(self.dir / "cloud_credentials") + self._write_backend("gcs") + self.env = {"GOOGLE_APPLICATION_CREDENTIALS": self.creds} + for target, value in ( + ("socket.gethostname", f"{self.HOST}.c.example.internal"), + ("_terraform_running_in", False), + ): + patcher = mock.patch.object( + utils.socket if target.startswith("socket") else utils, + target.split(".")[-1], return_value=value, + ) + setattr(self, target.split(".")[-1].strip("_"), patcher.start()) + self.addCleanup(patcher.stop) + + def _write_backend(self, backend_type): + marker = self.dir / ".terraform" / "terraform.tfstate" + marker.parent.mkdir(exist_ok=True) + marker.write_text(json.dumps({ + "version": 3, + "backend": { + "type": backend_type, + "config": {"bucket": "state-bkt", "prefix": "ns/vpcs/vpc_1"}, + }, + })) + + def _client(self, who=None): + """A storage client whose lock object is held by `who` (None: free).""" + client = mock.MagicMock() + if who is None: + client.bucket.return_value.get_blob.return_value = None + return client + blob = mock.MagicMock() + blob.generation = self.GENERATION + blob.download_as_bytes.return_value = json.dumps({ + "ID": "8c0ab9f4-random-id", + "Operation": "OperationTypeApply", + "Who": who, + "Version": "1.16.1", + "Created": "2026-10-15T22:16:03.123456789Z", + "Path": "gs://state-bkt/ns/vpcs/vpc_1/default.tflock", + }).encode() + client.bucket.return_value.get_blob.return_value = blob + return client + + def _run(self, client, command="plan"): + with mock.patch.object(utils, "_gcs_client", return_value=client): + with mock.patch.object(subprocess, "run") as run: + utils.run_terraform(self.dir, command, extra_env=self.env) + return run + + def test_no_lock_object_runs_terraform(self): + client = self._client(None) + run = self._run(client) + run.assert_called_once() + client.bucket.assert_called_with("state-bkt") + client.bucket.return_value.get_blob.assert_called_with( + "ns/vpcs/vpc_1/default.tflock" + ) + + def test_lock_held_by_another_host_runs_and_waits(self): + run = self._run(self._client("gcluster@other-server")) + run.assert_called_once() + self.assertIn("-lock-timeout=45m", run.call_args.args[0]) + + def test_lock_held_by_live_process_here_runs(self): + self.terraform_running_in.return_value = True + run = self._run(self._client(f"gcluster@{self.HOST}")) + run.assert_called_once() + + def test_stale_lock_raises_before_terraform_runs(self): + client = self._client(f"gcluster@{self.HOST}") + with self.assertLogs(utils.logger, "ERROR"): + with self.assertRaises(utils.TerraformStateLockError) as ctx: + self._run(client) + message = str(ctx.exception) + for expected in ( + str(self.dir), + f"Lock ID: {self.GENERATION}", + "OperationTypeApply", + f"gcluster@{self.HOST}", + "Age:", + "2026-10-15T22:16:03", + "no Terraform process on this server holds it", + f"cd {self.dir} && GOOGLE_APPLICATION_CREDENTIALS={self.creds}" + f" terraform force-unlock -force {self.GENERATION}", + ): + with self.subTest(expected=expected): + self.assertIn(expected, message) + self.assertNotIn("Age: unknown", message) + self.assertEqual(ctx.exception.stderr, message.encode()) + client.bucket.return_value.get_blob.return_value.delete.assert_not_called() + + def test_stale_lock_not_reported_if_released_meanwhile(self): + client = self._client(f"gcluster@{self.HOST}") + blob = client.bucket.return_value.get_blob.return_value + client.bucket.return_value.get_blob.side_effect = [blob, None] + run = self._run(client) + run.assert_called_once() + + def test_check_error_logs_warning_and_runs(self): + with mock.patch.object( + utils, "_gcs_client", side_effect=RuntimeError("storage down") + ): + with mock.patch.object(subprocess, "run") as run: + with self.assertLogs(utils.logger, "WARNING") as logs: + utils.run_terraform(self.dir, "plan", extra_env=self.env) + run.assert_called_once() + self.assertIn("storage down", "\n".join(logs.output)) + + def test_non_gcs_backend_skips_check(self): + self._write_backend("local") + with mock.patch.object(utils, "_gcs_client") as gcs_client: + with mock.patch.object(subprocess, "run") as run: + utils.run_terraform(self.dir, "plan", extra_env=self.env) + gcs_client.assert_not_called() + run.assert_called_once() + + def test_missing_backend_record_skips_check(self): + (self.dir / ".terraform" / "terraform.tfstate").unlink() + with mock.patch.object(utils, "_gcs_client") as gcs_client: + with mock.patch.object(subprocess, "run") as run: + utils.run_terraform(self.dir, "plan", extra_env=self.env) + gcs_client.assert_not_called() + run.assert_called_once() + + def test_non_locking_command_skips_check(self): + with mock.patch.object(utils, "_gcs_client") as gcs_client: + self._run(gcs_client, "init") + gcs_client.assert_not_called() + + def test_lock_timeout_is_reported_from_stderr(self): + stderr = ( + "\nError: Error acquiring the state lock\n\n" + "Error message: writing \"gs://state-bkt/ns/vpcs/vpc_1/" + "default.tflock\" failed: googleapi: Error 412\n" + "Lock Info:\n" + " ID: 1760566999000001\n" + " Path: gs://state-bkt/ns/vpcs/vpc_1/default.tflock\n" + " Operation: OperationTypeApply\n" + " Who: gcluster@other-server\n" + " Version: 1.16.1\n" + " Created: 2026-10-15 22:16:03.123456789 +0000 UTC\n" + " Info:\n" + ) + + def fail(cmdline, **kwargs): + kwargs["stderr"].write(stderr.encode()) + raise subprocess.CalledProcessError(1, cmdline) + + with mock.patch.object( + utils, "_gcs_client", return_value=self._client(None) + ): + with mock.patch.object(subprocess, "run", side_effect=fail): + with self.assertLogs(utils.logger, "ERROR"): + with self.assertRaises( + utils.TerraformStateLockError + ) as ctx: + utils.run_terraform( + self.dir, "plan", extra_env=self.env + ) + message = str(ctx.exception) + self.assertIn("waited 45m", message) + self.assertIn("gcluster@other-server", message) + self.assertIn("terraform force-unlock -force 1760566999000001", message) + self.assertIsInstance( + ctx.exception.__cause__, subprocess.CalledProcessError + ) + + def test_other_failures_are_reraised_unchanged(self): + original = subprocess.CalledProcessError(1, ["terraform", "plan"]) + + def fail(cmdline, **kwargs): + kwargs["stderr"].write(b"Error: Invalid reference\n") + raise original + + with mock.patch.object( + utils, "_gcs_client", return_value=self._client(None) + ): + with mock.patch.object(subprocess, "run", side_effect=fail): + with self.assertRaises(subprocess.CalledProcessError) as ctx: + utils.run_terraform(self.dir, "plan", extra_env=self.env) + self.assertIs(ctx.exception, original) + + def test_error_is_a_called_process_error(self): + err = utils.TerraformStateLockError(["terraform", "plan"], "locked") + self.assertIsInstance(err, subprocess.CalledProcessError) + self.assertEqual(err.returncode, 1) + self.assertEqual(str(err), "locked") + + +@unittest.skipUnless(Path("/proc/self/cwd").exists(), "needs Linux /proc") +class TerraformProcessScanTests(unittest.TestCase): + """The process scan finds a process named terraform by working directory.""" + + def test_finds_terraform_only_in_its_directory(self): + with tempfile.TemporaryDirectory() as tmp: + work, other = Path(tmp) / "work", Path(tmp) / "other" + work.mkdir() + other.mkdir() + fake = Path(tmp) / "terraform" + fake.symlink_to(sys.executable) + proc = subprocess.Popen( + [str(fake), "-c", "import time; time.sleep(30)"], cwd=work + ) + try: + self.assertTrue(utils._terraform_running_in(work)) + self.assertFalse(utils._terraform_running_in(other)) + finally: + proc.kill() + proc.wait() + + def test_unreadable_process_directory_counts_as_running(self): + # Another user's terraform: its cwd can't be read, so it may hold the + # lock and must not let the lock be called stale. + with tempfile.TemporaryDirectory() as tmp: + work, proc_root = Path(tmp) / "work", Path(tmp) / "proc" + (proc_root / "4242").mkdir(parents=True) + (proc_root / "4242" / "comm").write_text("terraform\n") + (proc_root / "4242" / "cwd").symlink_to(work) + work.mkdir() + real_resolve = Path.resolve + + def resolve(path, strict=False): + if path.name == "cwd": + raise PermissionError(13, "Permission denied", str(path)) + return real_resolve(path, strict=strict) + + with mock.patch.object(Path, "resolve", resolve): + self.assertTrue(utils._terraform_running_in(work, proc_root=proc_root)) + + +if __name__ == "__main__": + unittest.main() diff --git a/community/front-end/ofe/website/ghpcfe/tests/test_workbench_changes.py b/community/front-end/ofe/website/ghpcfe/tests/test_workbench_changes.py new file mode 100644 index 0000000000..ac1e398e89 --- /dev/null +++ b/community/front-end/ofe/website/ghpcfe/tests/test_workbench_changes.py @@ -0,0 +1,129 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Source-level checks on the Workbench form, templates and Terraform. + +Workbench Instances always use the default Workbench image, so an image-family +choice in the portal would be silently ignored: it must stay out of the form, +the templates and the generated tfvars, and Terraform must keep rejecting it. +The data disk size must come from a variable, not a hardcoded number. + +These read files as text, or run the real startup-script writer with +stand-in objects, so they need neither Django nor Terraform. +""" + +import importlib.util +import pathlib +import re +import sys +import types +from unittest import mock + +OFE = pathlib.Path(__file__).resolve().parents[3] +GHPCFE = OFE / "website" / "ghpcfe" +TF_DIR = OFE / "infrastructure_files" / "workbench_tf" / "google" + + +def _workbench_form_blocks(): + src = (GHPCFE / "forms.py").read_text() + # Slice the Workbench form class(es) so other forms' fields are ignored. + blocks = re.findall( + r"^class \w*Workbench\w*\(.*?(?=^class |\Z)", src, re.M | re.S + ) + assert blocks, "no Workbench form classes found" + return blocks + + +def test_workbench_forms_do_not_expose_image_family(): + for block in _workbench_form_blocks(): + assert "image_family" not in block + + +def test_workbench_templates_do_not_show_image_family(): + templates = list((GHPCFE / "templates" / "workbench").glob("*.html")) + assert templates + for path in templates: + assert "image_family" not in path.read_text(), path.name + + +def test_workbench_tfvars_do_not_write_image_family(): + src = (GHPCFE / "cluster_manager" / "workbenchinfo.py").read_text() + assert "image_family" not in src + + +def test_terraform_still_rejects_image_family(): + src = (TF_DIR / "variables.tf").read_text() + assert re.search(r"condition\s*=\s*var\.image_family == null", src) + + +def test_data_disk_size_is_a_variable(): + main = (TF_DIR / "main.tf").read_text() + block = re.search(r"data_disks\s*\{(.*?)\n\s*\}", main, re.S).group(1) + assert re.search(r"disk_size_gb\s*=\s*var\.data_disk_size_gb\b", block) + assert not re.search(r"disk_size_gb\s*=\s*\d+", block) + variables = (TF_DIR / "variables.tf").read_text() + assert 'variable "data_disk_size_gb"' in variables + + +def _load_workbenchinfo(): + """Import cluster_manager/workbenchinfo.py without Django or its packages. + + copy_startup_script uses neither `utils` nor `retry`, so both are stubbed, + and the module goes in a throwaway package so `from . import utils` + resolves to the stub. + """ + pkg = types.ModuleType("wb_stub") + pkg.__path__ = [] + pkg.utils = types.ModuleType("wb_stub.utils") + retry_mod = types.ModuleType("retry") + retry_mod.retry = lambda *args, **kwargs: (lambda func: func) + stubs = {"wb_stub": pkg, "wb_stub.utils": pkg.utils, "retry": retry_mod} + with mock.patch.dict(sys.modules, stubs): + spec = importlib.util.spec_from_file_location( + "wb_stub.workbenchinfo", GHPCFE / "cluster_manager" / "workbenchinfo.py" + ) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _render_startup_script(tmp_path, social_account): + wbinfo = _load_workbenchinfo() + template_dir = tmp_path / "infrastructure_files" / "gcs_bucket" / "workbench" + template_dir.mkdir(parents=True) + (template_dir / "startup_script_template.sh").write_text("# template\n") + user = mock.Mock() + user.socialaccount_set.first.return_value = social_account + workbench = mock.Mock(id=1, trusted_user=user, attached_cluster=None) + workbench.mount_points.order_by.return_value = [] + info = wbinfo.WorkbenchInfo.__new__(wbinfo.WorkbenchInfo) + info.config = {"baseDir": tmp_path} + info.workbench = workbench + info.workbench_dir = tmp_path + info.copy_startup_script() + return (tmp_path / "startup_script.sh").read_text() + + +def test_startup_script_without_a_linked_google_account(tmp_path): + # The OS Login name is only logged, so a trusted user with no social + # account must still get a startup script. + script = _render_startup_script(tmp_path, None) + assert 'select ( .name == "")' in script + assert "oslogin user resolved to:" in script + + +def test_startup_script_looks_up_the_linked_google_account(tmp_path): + script = _render_startup_script(tmp_path, mock.Mock(uid="1234567890")) + assert 'select ( .name == "1234567890")' in script diff --git a/community/front-end/ofe/website/ghpcfe/views/clusters.py b/community/front-end/ofe/website/ghpcfe/views/clusters.py index bbcf3d0a5c..a230f8ee06 100644 --- a/community/front-end/ofe/website/ghpcfe/views/clusters.py +++ b/community/front-end/ofe/website/ghpcfe/views/clusters.py @@ -1120,7 +1120,6 @@ def get_orm(self, cluster_id): def cmd(self, unused_task_id, unused_token, cluster): ci = ClusterInfo(cluster) - ci.update() ci.reconfigure_cluster() async def get(self, request, pk): diff --git a/community/front-end/ofe/website/ghpcfe/views/credentials.py b/community/front-end/ofe/website/ghpcfe/views/credentials.py index a06c79e4ac..6052505b72 100644 --- a/community/front-end/ofe/website/ghpcfe/views/credentials.py +++ b/community/front-end/ofe/website/ghpcfe/views/credentials.py @@ -13,6 +13,8 @@ # limitations under the License. """ credentials.py """ +import logging + from django.http import HttpResponseRedirect, JsonResponse from django.urls import reverse, reverse_lazy from django.views import generic @@ -32,6 +34,22 @@ from .. import grafana from grafana_api.grafana_api import GrafanaClientError +logger = logging.getLogger(__name__) + + +def _warn_on_state_bucket_access(request, detail): + """Warn, without blocking, if a credential cannot use the state bucket.""" + try: + warning = validate_credential.check_state_bucket_access(detail) + except Exception: # pylint: disable=broad-except + logger.exception("Could not check the credential's state bucket access") + warning = ( + "Could not check whether this credential can use the Terraform " + "state bucket; see the server log." + ) + if warning: + messages.warning(request, warning) + class CredentialListView(SuperUserRequiredMixin, generic.ListView): """Custom ListView for Credential model""" @@ -87,6 +105,7 @@ def form_valid(self, form): self.object.save() messages.success(self.request, f"Credential {self.object.name} validated and saved.") + _warn_on_state_bucket_access(self.request, self.object.detail) return HttpResponseRedirect(self.get_success_url()) class CredentialUpdateView(SuperUserRequiredMixin, UpdateView): @@ -108,6 +127,11 @@ def get_initial(self): initial[ "detail" ] = "" return initial + def form_valid(self, form): + response = super().form_valid(form) + _warn_on_state_bucket_access(self.request, self.object.detail) + return response + class CredentialDeleteView(SuperUserRequiredMixin, DeleteView): """Custom DeleteView for Credential model"""