From cec3ac6c51d4cd730668b0cf6de46e00475d7c66 Mon Sep 17 00:00:00 2001 From: "Juan Carlos Moreno [ackstorm]" Date: Thu, 29 Oct 2026 09:08:41 +0100 Subject: [PATCH 1/2] fix(e2e): compare config digests in the kind image check Since 2026-10-26 the GitHub runners build with Docker's containerd image store, where `docker image inspect .Id` is the manifest digest. crictl reports the config digest, so verify_image_in_node failed every run even though the load worked: on the failing run the node held exactly the config digest buildx exported. Read the config digest from docker save's manifest.json instead, which names it under either store. Also pass -o before the image to crictl inspecti: the crictl in kind v0.31 nodes (1.33) treats a trailing -o as an image name, so the node id read as absent. Co-Authored-By: Claude Opus 5.5 --- hack/cluster.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/hack/cluster.sh b/hack/cluster.sh index 77253a4..ed8c2db 100755 --- a/hack/cluster.sh +++ b/hack/cluster.sh @@ -101,15 +101,22 @@ build_stamp() { # on paths that do not end with the node holding the new bytes, and every one # of the four times this trap fired, the load — not the build — was what # silently did nothing. +# +# Both sides compare the image CONFIG digest, which is what crictl reports as +# the id. The host's `docker image inspect .Id` is not usable for that: with +# Docker's containerd image store (the GitHub runner default since 2026-10-26) +# it is the manifest digest, so it never matches even when the load worked. +# docker save's manifest.json names the config blob under either store. verify_image_in_node() { local img="$1" host_id node_id - host_id="$(docker image inspect "${img}" --format '{{.Id}}' 2>/dev/null || true)" + host_id="$(docker save "${img}" 2>/dev/null | tar -xO manifest.json 2>/dev/null \ + | sed -n 's/.*"Config":"[^"]*\([0-9a-f]\{64\}\)[^"]*".*/sha256:\1/p' | head -1 || true)" if [ -z "${host_id}" ]; then echo "hack/cluster.sh: ${img} was never built on the host" >&2 return 1 fi node_id="$(docker exec "${CLUSTER_NAME}-control-plane" \ - crictl inspecti "${img}" -o json 2>/dev/null \ + crictl inspecti -o json "${img}" 2>/dev/null \ | sed -n 's/.*"id": "\(sha256:[0-9a-f]*\)".*/\1/p' | head -1 || true)" if [ "${host_id}" != "${node_id}" ]; then echo "hack/cluster.sh: ${img} in the node is '${node_id:-}'," >&2 From 75c89376300af29a0c29fc13a2773934ac4fd06d Mon Sep 17 00:00:00 2001 From: "Juan Carlos Moreno [ackstorm]" Date: Thu, 29 Oct 2026 09:19:46 +0100 Subject: [PATCH 2/2] fix(e2e): pull dstack and postgres inside the kind node Same runner change as the previous commit: these images are multi-arch, the containerd image store keeps only the host platform after docker pull, and kind load's ctr import --all-platforms then fails with "content digest ... not found". The node pulls them with crictl, once: an image already in the node is not pulled again. Co-Authored-By: Claude Opus 5.5 --- hack/cluster.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/hack/cluster.sh b/hack/cluster.sh index ed8c2db..11739bd 100755 --- a/hack/cluster.sh +++ b/hack/cluster.sh @@ -195,13 +195,16 @@ build_and_load_images() { done [ "${failed}" -eq 0 ] || exit 1 - # dstack is pulled once and loaded, so the cluster never reaches the - # network during a test run. - docker image inspect "${DSTACK_IMAGE}" >/dev/null 2>&1 || docker pull "${DSTACK_IMAGE}" - kind load docker-image "${DSTACK_IMAGE}" --name "${CLUSTER_NAME}" - - docker image inspect "${POSTGRES_IMAGE}" >/dev/null 2>&1 || docker pull "${POSTGRES_IMAGE}" - kind load docker-image "${POSTGRES_IMAGE}" --name "${CLUSTER_NAME}" + # dstack and postgres are pulled once, so the cluster never reaches the + # network during a test run. The node pulls them itself: they are + # multi-arch, and with Docker's containerd image store the host keeps only + # its own platform while `docker save` exports the whole index, so + # `kind load` fails with "content digest ... not found". + local node="${CLUSTER_NAME}-control-plane" img + for img in "${DSTACK_IMAGE}" "${POSTGRES_IMAGE}"; do + docker exec "${node}" crictl inspecti -o json "${img}" >/dev/null 2>&1 \ + || docker exec "${node}" crictl pull "${img}" + done } # wait_for_operator_leadership blocks until squall-operator actually HOLDS its