packer clean ups and better logging

Dockerfile-multigres

@@ -202,7 +202,7 @@ COPY docker/pgctld/postgresql.conf.tmpl /etc/pgctld-custom/postgresql.conf.tmpl
 # Wrapper: injects --postgres-config-template on every pgctld call AND bridges
 # postgres's JSON log file to container stdout via a /proc/1/fd/1 symlink so
 # kubelet + Vector can ship it without a sidecar. See docker/pgctld/pgctld-wrapper.
-COPY --chmod=755 docker/pgctld/pgctld-wrapper /usr/local/bin/pgctld
+COPY --chmod=755 docker/pgctld/pgctld-wrapper.sh /usr/local/bin/pgctld
 ENV POSTGRES_CONFIG_TEMPLATE_PATH=/etc/pgctld-custom/postgresql.conf.tmpl
 
 # Strip extensions absent from pg17 vanilla build

README.md

@@ -74,7 +74,6 @@ Here's a comprehensive overview of the project's directory structure:
 | **tests/** | Integration and system tests |
 | testinfra/ | Infrastructure tests using pytest framework |
 | tests/ | General integration test suites |
-| **scripts/** | Utility scripts for development and deployment |
 | **docs/** | Additional documentation, images, and resources |
 | **ebssurrogate/** | AWS EBS surrogate building for AMI creation |
 | **http/** | HTTP-related configurations and files |

amazon-amd64-nix.pkr.hcl

@@ -194,57 +194,47 @@ build {
 
   provisioner "file" {
     source = "ebssurrogate/files/ebsnvme-id"
-    destination = "/tmp/ebsnvme-id"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "ebssurrogate/files/70-ec2-nvme-devices.rules"
-    destination = "/tmp/70-ec2-nvme-devices.rules"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "ebssurrogate/scripts/chroot-bootstrap-nix.sh"
-    destination = "/tmp/chroot-bootstrap-nix.sh"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "ebssurrogate/files/cloud.cfg"
-    destination = "/tmp/cloud.cfg"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "ebssurrogate/files/vector.timer"
-    destination = "/tmp/vector.timer"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "ebssurrogate/files/apparmor_profiles"
-    destination = "/tmp"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "migrations"
-    destination = "/tmp"
-  }
-
-  # Copy ansible playbook
-  provisioner "shell" {
-    inline = ["mkdir /tmp/ansible-playbook"]
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "ansible"
-    destination = "/tmp/ansible-playbook"
-  }
-
-  provisioner "file" {
-    source = "scripts"
-    destination = "/tmp/ansible-playbook"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
-    source = "ansible/vars.yml"
-    destination = "/tmp/ansible-playbook/vars.yml"
+    source = "ebssurrogate/scripts/90-cleanup.sh"
+    destination = "/tmp/"
   }
 
   provisioner "shell" {
@@ -258,7 +248,7 @@ build {
     ]
     use_env_var_file = true
     script = "ebssurrogate/scripts/surrogate-bootstrap-nix.sh"
-    execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && cd /tmp/ansible-playbook && {{.Path}}'"
+    execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && {{.Path}}'"
     start_retry_timeout = "5m"
     skip_clean = true
   }

amazon-arm64-nix.pkr.hcl

@@ -194,57 +194,47 @@ build {
 
   provisioner "file" {
     source = "ebssurrogate/files/ebsnvme-id"
-    destination = "/tmp/ebsnvme-id"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "ebssurrogate/files/70-ec2-nvme-devices.rules"
-    destination = "/tmp/70-ec2-nvme-devices.rules"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "ebssurrogate/scripts/chroot-bootstrap-nix.sh"
-    destination = "/tmp/chroot-bootstrap-nix.sh"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "ebssurrogate/files/cloud.cfg"
-    destination = "/tmp/cloud.cfg"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "ebssurrogate/files/vector.timer"
-    destination = "/tmp/vector.timer"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "ebssurrogate/files/apparmor_profiles"
-    destination = "/tmp"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "migrations"
-    destination = "/tmp"
-  }
-
-  # Copy ansible playbook
-  provisioner "shell" {
-    inline = ["mkdir /tmp/ansible-playbook"]
+    destination = "/tmp/"
   }
 
   provisioner "file" {
     source = "ansible"
-    destination = "/tmp/ansible-playbook"
-  }
-
-  provisioner "file" {
-    source = "scripts"
-    destination = "/tmp/ansible-playbook"
+    destination = "/tmp/"
   }
 
   provisioner "file" {
-    source = "ansible/vars.yml"
-    destination = "/tmp/ansible-playbook/vars.yml"
+    source = "ebssurrogate/scripts/90-cleanup.sh"
+    destination = "/tmp/"
   }
 
   provisioner "shell" {
@@ -258,7 +248,7 @@ build {
     ]
     use_env_var_file = true
     script = "ebssurrogate/scripts/surrogate-bootstrap-nix.sh"
-    execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && cd /tmp/ansible-playbook && {{.Path}}'"
+    execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && {{.Path}}'"
     start_retry_timeout = "5m"
     skip_clean = true
   }

ansible/files/admin_api_scripts/grow_fs.sh

@@ -8,14 +8,14 @@ VOLUME_TYPE=${1:-data}
 UBUNTU_VERSION=$(lsb_release -rs)
 
 if pgrep resizefs; then
-    echo "resize2fs is already running"
-    exit 1
+	echo "resize2fs is already running"
+	exit 1
 fi
 
 # install amazon disk utilities if not present on 24.04
 if [ "${UBUNTU_VERSION}" = "24.04" ] && ! /usr/bin/dpkg-query -W amazon-ec2-utils >/dev/null 2>&1; then
-    apt-get update
-    apt-get install -y amazon-ec2-utils || true
+	apt-get update
+	apt-get install -y amazon-ec2-utils || true
 fi
 
 # We currently mount 3 possible disks
@@ -27,15 +27,15 @@ XVDA_DEVICE="/dev/nvme0n1"
 XVDH_DEVICE="/dev/nvme1n1"
 # Map AWS devices to NVMe for ubuntu 24.04 and later
 if [ "${UBUNTU_VERSION}" = "24.04" ] && /usr/bin/dpkg-query -W amazon-ec2-utils >/dev/null 2>&1; then
-    for nvme_dev in $(lsblk -dprno name,type | grep disk | awk '{print $1}'); do
-        if [ -b "$nvme_dev" ]; then
-            mapping=$(ebsnvme-id -b "$nvme_dev" 2>/dev/null)
-            case "$mapping" in
-                "xvda"|"/dev/xvda") XVDA_DEVICE="$nvme_dev" ;;
-                "xvdh"|"/dev/xvdh") XVDH_DEVICE="$nvme_dev" ;;
-            esac
-        fi
-    done
+	for nvme_dev in $(lsblk -dprno name,type | grep disk | awk '{print $1}'); do
+		if [ -b "$nvme_dev" ]; then
+			mapping=$(ebsnvme-id -b "$nvme_dev" 2>/dev/null)
+			case "$mapping" in
+			"xvda" | "/dev/xvda") XVDA_DEVICE="$nvme_dev" ;;
+			"xvdh" | "/dev/xvdh") XVDH_DEVICE="$nvme_dev" ;;
+			esac
+		fi
+	done
 fi
 
 echo "Using devices - Root: $XVDA_DEVICE, Data: $XVDH_DEVICE"
@@ -45,35 +45,35 @@ ROOT_DEVICE_FULL=$(findmnt -no SOURCE /)
 ROOT_DEVICE=$(lsblk -no PKNAME "$ROOT_DEVICE_FULL")
 ROOT_PARTITION_NUMBER=$(echo "$ROOT_DEVICE_FULL" | sed "s|.*${ROOT_DEVICE}p||")
 
-if ! [[ "$ROOT_PARTITION_NUMBER" =~ ^[0-9]+$ ]]; then
-  echo "Error: ROOT_PARTITION_NUMBER is not a valid number: $ROOT_PARTITION_NUMBER"
-  exit 1
+if ! [[ $ROOT_PARTITION_NUMBER =~ ^[0-9]+$ ]]; then
+	echo "Error: ROOT_PARTITION_NUMBER is not a valid number: $ROOT_PARTITION_NUMBER"
+	exit 1
 fi
 
-if [ -b "${XVDH_DEVICE}" ] ; then
-    if [[ "${VOLUME_TYPE}" == "data" ]]; then
-        resize2fs "${XVDH_DEVICE}"
+if [ -b "${XVDH_DEVICE}" ]; then
+	if [[ ${VOLUME_TYPE} == "data" ]]; then
+		resize2fs "${XVDH_DEVICE}"
 
-        # Explicitly reserving 100MiB worth of blocks for the data volume
-        #
-        # This is owned in $GIT_DIR/ebssurrogate/scripts/surrogate-bootstrap-nix.sh
-        RESERVED_DATA_VOLUME_BLOCK_COUNT=$((100 * 1024 * 1024 / 4096))
-        tune2fs -r $RESERVED_DATA_VOLUME_BLOCK_COUNT "${XVDH_DEVICE}"
+		# Explicitly reserving 100MiB worth of blocks for the data volume
+		#
+		# This is owned in $GIT_DIR/ebssurrogate/scripts/surrogate-bootstrap-nix.sh
+		RESERVED_DATA_VOLUME_BLOCK_COUNT=$((100 * 1024 * 1024 / 4096))
+		tune2fs -r $RESERVED_DATA_VOLUME_BLOCK_COUNT "${XVDH_DEVICE}"
 
-    elif [[ "${VOLUME_TYPE}" == "root" ]] ; then
-        PLACEHOLDER_FL=/home/ubuntu/50M_PLACEHOLDER
-        rm -f "${PLACEHOLDER_FL}" || true
-        growpart "${XVDA_DEVICE}" "${ROOT_PARTITION_NUMBER}"
-        resize2fs "${XVDA_DEVICE}p${ROOT_PARTITION_NUMBER}"
-        if [[ ! -f "${PLACEHOLDER_FL}" ]] ; then
-            fallocate -l50M "${PLACEHOLDER_FL}"
-        fi
-    else
-        echo "Invalid disk specified: ${VOLUME_TYPE}"
-        exit 1
-    fi
+	elif [[ ${VOLUME_TYPE} == "root" ]]; then
+		PLACEHOLDER_FL=/home/ubuntu/50M_PLACEHOLDER
+		rm -f "${PLACEHOLDER_FL}" || true
+		growpart "${XVDA_DEVICE}" "${ROOT_PARTITION_NUMBER}"
+		resize2fs "${XVDA_DEVICE}p${ROOT_PARTITION_NUMBER}"
+		if [[ ! -f ${PLACEHOLDER_FL} ]]; then
+			fallocate -l50M "${PLACEHOLDER_FL}"
+		fi
+	else
+		echo "Invalid disk specified: ${VOLUME_TYPE}"
+		exit 1
+	fi
 else
-    growpart "${XVDA_DEVICE}" "${ROOT_PARTITION_NUMBER}"
-    resize2fs "${XVDA_DEVICE}p${ROOT_PARTITION_NUMBER}"
+	growpart "${XVDA_DEVICE}" "${ROOT_PARTITION_NUMBER}"
+	resize2fs "${XVDA_DEVICE}p${ROOT_PARTITION_NUMBER}"
 fi
 echo "Done resizing disk"

ansible/files/admin_api_scripts/manage_readonly_mode.sh

@@ -5,13 +5,14 @@ set -euo pipefail
 SUBCOMMAND=$1
 
 function set_mode {
-    MODE=$1
-    psql -h localhost -U supabase_admin -d postgres -c "ALTER SYSTEM SET default_transaction_read_only to ${MODE};"
-    psql -h localhost -U supabase_admin -d postgres -c "SELECT pg_reload_conf();"
+	MODE=$1
+	psql -h localhost -U supabase_admin -d postgres -c "ALTER SYSTEM SET default_transaction_read_only to ${MODE};"
+	psql -h localhost -U supabase_admin -d postgres -c "SELECT pg_reload_conf();"
 }
 
 function check_override {
-    COMMAND=$(cat <<EOF
+	COMMAND=$(
+		cat <<EOF
 WITH role_comment as (
     SELECT pg_catalog.shobj_description(r.oid, 'pg_authid') AS content
     FROM pg_catalog.pg_roles r
@@ -25,21 +26,21 @@ SELECT
            END as override_active
 FROM role_comment;
 EOF
-)
-    RESULT=$(psql -h localhost -U supabase_admin -d postgres -At -c "$COMMAND")
-    echo -n "$RESULT"
+	)
+	RESULT=$(psql -h localhost -U supabase_admin -d postgres -At -c "$COMMAND")
+	echo -n "$RESULT"
 }
 
 case $SUBCOMMAND in
-    "check_override")
-        check_override
-        ;;
-    "set")
-       shift
-        set_mode "$@"
-        ;;
-    *)
-        echo "Error: '$SUBCOMMAND' is not a known subcommand."
-        exit 1
-        ;;
-esac
+"check_override")
+	check_override
+	;;
+"set")
+	shift
+	set_mode "$@"
+	;;
+*)
+	echo "Error: '$SUBCOMMAND' is not a known subcommand."
+	exit 1
+	;;
+esac