ci: continuous drift verification of released artifacts

.github/workflows/registry-drift.yml

@@ -0,0 +1,102 @@
+# Copyright 2026 ExtendDB contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Continuous drift verification for released container artifacts.
+#
+# Daily, and on demand: every released version tag, signature, and `latest`
+# on every registry must still be exactly what was released, per
+# docker/released-artifacts.json. All checks are ANONYMOUS reads — this
+# verifies what users see, holds no credentials, and needs no environment.
+# Logic lives in devtools/verify-released-artifacts so it runs identically
+# on a workstation.
+#
+# Alerting: scheduled-run failure emails go only to the workflow author, so
+# drift instead creates (or comments on) a 'registry-drift' issue with the
+# failing report, and a clean run closes any open ones. DRIFT (reachable
+# and wrong) and UNREACHABLE (registry errors after retries) are labeled
+# distinctly — an outage is not a security event.
+
+name: registry-drift
+
+on:
+  schedule:
+    - cron: '17 6 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write        # drift alerting only; no registry credentials exist here
+
+concurrency:
+  group: registry-drift
+  cancel-in-progress: false
+
+env:
+  COSIGN_VERSION: v3.1.3
+  COSIGN_SHA256: 4629c757b7618056f8ddd7e2625ae9fdd94c0372a65049520bc7d9df9efc7f71
+  CRANE_VERSION: v0.21.9
+  CRANE_SHA256: 5c16d8ddb971cb1d5e6ed8b1e743da8224414eeba2c2762d8f1a61b2f095699e
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - name: Install pinned crane and cosign (checksum-verified)
+        run: |
+          set -euo pipefail
+          curl -sSfL -o /tmp/crane.tgz \
+            "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz"
+          echo "${CRANE_SHA256}  /tmp/crane.tgz" | sha256sum --check --strict
+          tar -xzf /tmp/crane.tgz -C /tmp crane
+          sudo install -m 0755 /tmp/crane /usr/local/bin/crane
+          curl -sSfL -o /tmp/cosign \
+            "https://github.com/sigstore/cosign/releases/download/${COSIGN_VERSION}/cosign-linux-amd64"
+          echo "${COSIGN_SHA256}  /tmp/cosign" | sha256sum --check --strict
+          sudo install -m 0755 /tmp/cosign /usr/local/bin/cosign
+
+      - name: Verify all released artifacts (anonymous)
+        id: verify
+        run: |
+          set +e
+          ./devtools/verify-released-artifacts | tee /tmp/report.txt
+          echo "rc=$?" >> "$GITHUB_OUTPUT"
+
+      - name: Alert on drift or persistent unreachability
+        if: steps.verify.outputs.rc != '0'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RC: ${{ steps.verify.outputs.rc }}
+        run: |
+          set -euo pipefail
+          KIND=$([[ "$RC" == "1" ]] && echo "DRIFT DETECTED" || echo "registries unreachable")
+          TITLE="registry-drift: $KIND"
+          BODY_FILE=/tmp/issue.md
+          {
+            echo "Automated report from the registry-drift verification run: $KIND"
+            echo
+            echo '```'
+            cat /tmp/report.txt
+            echo '```'
+            echo
+            echo "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          } > "$BODY_FILE"
+          EXISTING=$(gh issue list --label registry-drift --state open --json number --jq '.[0].number // empty')
+          if [[ -n "$EXISTING" ]]; then
+            gh issue comment "$EXISTING" --body-file "$BODY_FILE"
+          else
+            gh issue create --title "$TITLE" --label registry-drift --body-file "$BODY_FILE"
+          fi
+          exit 1
+
+      - name: Close drift issues on a clean run
+        if: steps.verify.outputs.rc == '0'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          for n in $(gh issue list --label registry-drift --state open --json number --jq '.[].number'); do
+            gh issue close "$n" --comment "Clean verification run: all released artifacts intact again. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          done

devtools/verify-released-artifacts

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# Copyright 2026 ExtendDB contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Continuous drift verification: every released artifact, on every registry
+# it shipped to, must still be exactly what was released.
+#
+# Reads docker/released-artifacts.json and asserts, entirely through
+# ANONYMOUS reads (what users see, with nothing to steal):
+#
+#   1. every released version tag resolves to its recorded digest
+#   2. the Cosign signature tag exists and cryptographically VERIFIES
+#      against the committed release public key (with the transparency log
+#      for releases that have entries; --insecure-ignore-tlog only for the
+#      two pre-Rekor releases)
+#   3. `latest` on each registry points at the highest release shipped to
+#      that registry (the forward-only invariant, audited)
+#
+# Failure semantics: DRIFT (reachable and wrong) and UNREACHABLE (still
+# failing after retries) both exit nonzero, but are labeled separately —
+# a registry outage is not a security event. Transient errors are absorbed
+# by bounded retries so a blip cannot cry wolf.
+#
+# Gates nothing, fixes nothing, writes nothing. Output is the signal.
+
+set -uo pipefail
+exec < /dev/null
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+cd "$(cd "$SCRIPT_DIR/.." && pwd)"
+
+MANIFEST=docker/released-artifacts.json
+for tool in crane cosign jq; do
+  command -v "$tool" >/dev/null || { echo "error: $tool is required" >&2; exit 2; }
+done
+PUBKEY=$(jq -r .public_key "$MANIFEST")
+[[ -s "$PUBKEY" ]] || { echo "error: public key $PUBKEY missing" >&2; exit 2; }
+
+DRIFT=0
+UNREACHABLE=0
+
+# Bounded-retry digest read. Prints digest; rc 0 = read, rc 9 = definitely
+# absent, rc 1 = unreachable after retries.
+read_digest() {
+  local ref="$1" out
+  for _ in 1 2 3; do
+    if out=$(crane digest "$ref" 2>&1); then
+      if [[ "$out" =~ ^sha256:[0-9a-f]{64}$ ]]; then printf '%s' "$out"; return 0; fi
+    elif grep -qiE 'manifest unknown|name unknown|not found' <<< "$out"; then
+      return 9
+    fi
+    sleep 10
+  done
+  return 1
+}
+
+check() { # check <label> <ref> <expected-digest>
+  local label="$1" ref="$2" want="$3" got rc
+  got=$(read_digest "$ref"); rc=$?
+  case $rc in
+    0) if [[ "$got" == "$want" ]]; then echo "  OK          $label"
+       else echo "  DRIFT       $label -> $got (expected $want)"; DRIFT=1; fi ;;
+    9) echo "  DRIFT       $label -> MISSING"; DRIFT=1 ;;
+    *) echo "  UNREACHABLE $label"; UNREACHABLE=1 ;;
+  esac
+}
+
+echo "drift verification $(date -u +%FT%TZ) — manifest $(jq -r '.images | to_entries | map("\(.key):\(.value.releases | length)") | join(", ")' "$MANIFEST")"
+
+for image in $(jq -r '.images | keys[]' "$MANIFEST"); do
+  default_regs=$(jq -r --arg i "$image" '.images[$i].registries[]' "$MANIFEST")
+  n=$(jq -r --arg i "$image" '.images[$i].releases | length' "$MANIFEST")
+  for idx in $(seq 0 $((n-1))); do
+    version=$(jq -r --arg i "$image" --argjson x "$idx" '.images[$i].releases[$x].version' "$MANIFEST")
+    digest=$(jq -r --arg i "$image" --argjson x "$idx" '.images[$i].releases[$x].digest' "$MANIFEST")
+    rekor=$(jq -r --arg i "$image" --argjson x "$idx" '.images[$i].releases[$x].rekor' "$MANIFEST")
+    regs=$(jq -r --arg i "$image" --argjson x "$idx" '.images[$i].releases[$x].registries // empty | .[]' "$MANIFEST")
+    [[ -z "$regs" ]] && regs="$default_regs"
+    sig_tag="sha256-${digest#sha256:}.sig"
+    for reg in $regs; do
+      check "$reg:$version" "$reg:$version" "$digest"
+      # Signature: existence via check is insufficient — it must verify.
+      got=$(read_digest "$reg:$sig_tag"); rc=$?
+      if [[ $rc == 9 ]]; then echo "  DRIFT       $reg:$sig_tag -> MISSING"; DRIFT=1; continue
+      elif [[ $rc != 0 ]]; then echo "  UNREACHABLE $reg:$sig_tag"; UNREACHABLE=1; continue; fi
+      # Plain string, not an array: empty-array expansion under set -u is
+      # an error on bash < 5.1, and the flag contains no whitespace.
+      TLOG_FLAG=""
+      [[ "$rekor" == "false" ]] && TLOG_FLAG="--insecure-ignore-tlog=true"
+      ok=false
+      for _ in 1 2 3; do
+        # shellcheck disable=SC2086
+        if cosign verify --key "$PUBKEY" $TLOG_FLAG "$reg@$digest" >/dev/null 2>&1; then ok=true; break; fi
+        sleep 10
+      done
+      if $ok; then echo "  OK          $reg:$sig_tag (verifies$([[ "$rekor" == "true" ]] && echo ' + rekor'))"
+      else echo "  DRIFT       $reg:$sig_tag present but DOES NOT VERIFY"; DRIFT=1; fi
+    done
+  done
+  # latest: highest release shipped to each registry.
+  for reg in $default_regs; do
+    want=$(jq -r --arg i "$image" --arg r "$reg" '
+      .images[$i] as $img
+      | [$img.releases[] | select((.registries // $img.registries) | index($r))]
+      | sort_by(.version | split(".") | map(tonumber)) | last | .digest // empty' "$MANIFEST")
+    [[ -z "$want" ]] && continue
+    check "$reg:latest" "$reg:latest" "$want"
+  done
+done
+
+echo
+if (( DRIFT )); then echo "RESULT: DRIFT DETECTED"; exit 1
+elif (( UNREACHABLE )); then echo "RESULT: REGISTRIES UNREACHABLE (no confirmed drift)"; exit 3
+else echo "RESULT: all released artifacts intact"; exit 0; fi

docker/released-artifacts.json

@@ -0,0 +1,30 @@
+{
+  "_comment": "Source of truth for what every registry must serve. Append one release entry per shipped container version; the drift-verification cron (verify-released-artifacts) fails if any registry disagrees. 'rekor: false' marks pre-transparency-log releases (<= 0.1.6). A release-level 'registries' list narrows scope (dev 0.1.7 shipped to Docker Hub only).",
+  "public_key": "extenddb-signing.pub.pem",
+  "images": {
+    "extenddb-postgres": {
+      "registries": [
+        "docker.io/extenddb/extenddb-postgres",
+        "ghcr.io/extenddb/extenddb-postgres",
+        "public.ecr.aws/extenddb/extenddb-postgres"
+      ],
+      "releases": [
+        { "version": "0.1.5", "digest": "sha256:4890886b1a1fb8fc7e01f7288846ace06abb3b1fbadcf7f9be48ce9b08109721", "rekor": false },
+        { "version": "0.1.6", "digest": "sha256:6c7c12f3fd09ed099b90b18a77171cc19509ba7eabebc5d908521d8988cb56cf", "rekor": false },
+        { "version": "0.1.8", "digest": "sha256:ccca5497715656e9f020b15a9598896e280d3641b1720a00aa82fb2ae1ad435d", "rekor": true }
+      ]
+    },
+    "extenddb-dev": {
+      "registries": [
+        "docker.io/extenddb/extenddb-dev",
+        "ghcr.io/extenddb/extenddb-dev",
+        "public.ecr.aws/extenddb/extenddb-dev"
+      ],
+      "releases": [
+        { "version": "0.1.7", "digest": "sha256:91a38aae5bf25b14c09a77301c228d3e83c48d951067580d537b221e7d041ee3", "rekor": true,
+          "registries": [ "docker.io/extenddb/extenddb-dev" ] },
+        { "version": "0.1.8", "digest": "sha256:01a9eb24e96e14585ca9b48166a938aa90145e85f50fbf1df26014631ad6235a", "rekor": true }
+      ]
+    }
+  }
+}