fix(ci): predict the promoted app digest instead of trusting a read right after the retag (#9416)

* fix(ci): predict the promoted app digest instead of trusting a read right after the retag

* fix(ci): bound the promoted-tag wait by elapsed time and each ECR read by a timeout

Author: waleedlatif1

.github/scripts/get-ecr-image-digest.sh

@@ -9,7 +9,7 @@ if [ -n "$ALLOW_MISSING" ] && [ "$ALLOW_MISSING" != '--allow-missing' ]; then
   exit 1
 fi
 export AWS_PAGER=''
-aws ecr batch-get-image --repository-name "$REPOSITORY" --image-ids imageTag="$TAG" --output json |
+aws --cli-connect-timeout 10 --cli-read-timeout 20 ecr batch-get-image --repository-name "$REPOSITORY" --image-ids imageTag="$TAG" --output json |
   ALLOW_MISSING="$ALLOW_MISSING" python3 -c '
 import json, os, re, sys
 response = json.load(sys.stdin)

.github/scripts/promote-app-image.sh

@@ -9,9 +9,22 @@ RELEASE_TAG="${5:-}"
 : "${GITHUB_OUTPUT:?GitHub output file required}"
 SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 PREVIOUS=$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG" --allow-missing)
+# A tag read straight after the move can still return the previous image, which
+# reports a moved tag as unchanged. The dry run prints the exact manifest the
+# move pushes (plus a trailing newline), so its hash is the digest the tag will
+# point at, known before the move.
+MANIFEST=$(docker buildx imagetools create --dry-run -t "$REGISTRY/$REPOSITORY:$DEPLOY_TAG" "$REGISTRY/$REPOSITORY:$SOURCE_TAG")
+DIGEST="sha256:$(printf '%s' "$MANIFEST" | sha256sum | cut -d' ' -f1)"
 EPOCH=$(date +%s)
 bash "$SCRIPT_DIR/promote-image.sh" "$REGISTRY" "$REPOSITORY" "$SOURCE_TAG" "$DEPLOY_TAG" "$RELEASE_TAG"
-DIGEST=$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG")
+deadline=$(( $(date +%s) + 60 ))
+until [ "$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG" --allow-missing)" = "$DIGEST" ]; do
+  if [ "$(date +%s)" -ge "$deadline" ]; then
+    echo "ERROR: $DEPLOY_TAG did not resolve to the pushed $DIGEST within 60s; cutover cannot be correlated" >&2
+    exit 1
+  fi
+  sleep 3
+done
 CHANGED=true
 if [ "$DIGEST" = "$PREVIOUS" ]; then CHANGED=false; fi
 {