From d038d9aabcbe5e91d9903b1e6ce68115ffc396f2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 28 Oct 2026 11:54:12 -0700 Subject: [PATCH 1/2] fix(ci): predict the promoted app digest instead of trusting a read right after the retag --- .github/scripts/promote-app-image.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/scripts/promote-app-image.sh b/.github/scripts/promote-app-image.sh index 86cb8a6d77d..78fc9275ba4 100644 --- a/.github/scripts/promote-app-image.sh +++ b/.github/scripts/promote-app-image.sh @@ -9,9 +9,22 @@ RELEASE_TAG="${5:-}" : "${GITHUB_OUTPUT:?GitHub output file required}" SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) PREVIOUS=$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG" --allow-missing) +# A tag read straight after the move can still return the previous image, which +# reports a moved tag as unchanged. The dry run prints the exact manifest the +# move pushes (plus a trailing newline), so its hash is the digest the tag will +# point at, known before the move. +MANIFEST=$(docker buildx imagetools create --dry-run -t "$REGISTRY/$REPOSITORY:$DEPLOY_TAG" "$REGISTRY/$REPOSITORY:$SOURCE_TAG") +DIGEST="sha256:$(printf '%s' "$MANIFEST" | sha256sum | cut -d' ' -f1)" EPOCH=$(date +%s) bash "$SCRIPT_DIR/promote-image.sh" "$REGISTRY" "$REPOSITORY" "$SOURCE_TAG" "$DEPLOY_TAG" "$RELEASE_TAG" -DIGEST=$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG") +for attempt in $(seq 1 20); do + [ "$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG" --allow-missing)" = "$DIGEST" ] && break + if [ "$attempt" -eq 20 ]; then + echo "ERROR: $DEPLOY_TAG never resolved to the pushed $DIGEST; cutover cannot be correlated" >&2 + exit 1 + fi + sleep 3 +done CHANGED=true if [ "$DIGEST" = "$PREVIOUS" ]; then CHANGED=false; fi { From 58357e2364a97cc28c20e5928a8b09533c8ccab1 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 28 Oct 2026 12:03:00 -0700 Subject: [PATCH 2/2] fix(ci): bound the promoted-tag wait by elapsed time and each ECR read by a timeout --- .github/scripts/get-ecr-image-digest.sh | 2 +- .github/scripts/promote-app-image.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/scripts/get-ecr-image-digest.sh b/.github/scripts/get-ecr-image-digest.sh index f8de53f925e..ad0f6c3a363 100644 --- a/.github/scripts/get-ecr-image-digest.sh +++ b/.github/scripts/get-ecr-image-digest.sh @@ -9,7 +9,7 @@ if [ -n "$ALLOW_MISSING" ] && [ "$ALLOW_MISSING" != '--allow-missing' ]; then exit 1 fi export AWS_PAGER='' -aws ecr batch-get-image --repository-name "$REPOSITORY" --image-ids imageTag="$TAG" --output json | +aws --cli-connect-timeout 10 --cli-read-timeout 20 ecr batch-get-image --repository-name "$REPOSITORY" --image-ids imageTag="$TAG" --output json | ALLOW_MISSING="$ALLOW_MISSING" python3 -c ' import json, os, re, sys response = json.load(sys.stdin) diff --git a/.github/scripts/promote-app-image.sh b/.github/scripts/promote-app-image.sh index 78fc9275ba4..bb8f174e60f 100644 --- a/.github/scripts/promote-app-image.sh +++ b/.github/scripts/promote-app-image.sh @@ -17,10 +17,10 @@ MANIFEST=$(docker buildx imagetools create --dry-run -t "$REGISTRY/$REPOSITORY:$ DIGEST="sha256:$(printf '%s' "$MANIFEST" | sha256sum | cut -d' ' -f1)" EPOCH=$(date +%s) bash "$SCRIPT_DIR/promote-image.sh" "$REGISTRY" "$REPOSITORY" "$SOURCE_TAG" "$DEPLOY_TAG" "$RELEASE_TAG" -for attempt in $(seq 1 20); do - [ "$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG" --allow-missing)" = "$DIGEST" ] && break - if [ "$attempt" -eq 20 ]; then - echo "ERROR: $DEPLOY_TAG never resolved to the pushed $DIGEST; cutover cannot be correlated" >&2 +deadline=$(( $(date +%s) + 60 )) +until [ "$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG" --allow-missing)" = "$DIGEST" ]; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "ERROR: $DEPLOY_TAG did not resolve to the pushed $DIGEST within 60s; cutover cannot be correlated" >&2 exit 1 fi sleep 3