From a4696685454d46ddcd23d940ca3fc06372265eed Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 10:05:55 +0545 Subject: [PATCH 01/22] worker: redirect-only HTTPS front for the GHCR mirror --- worker/src/index.js | 165 +++++++++++++++++++++++++++++++++++++++++++ worker/wrangler.toml | 20 ++++++ 2 files changed, 185 insertions(+) create mode 100644 worker/src/index.js create mode 100644 worker/wrangler.toml diff --git a/worker/src/index.js b/worker/src/index.js new file mode 100644 index 0000000..4cf96e5 --- /dev/null +++ b/worker/src/index.js @@ -0,0 +1,165 @@ +// ghcr-mirror-archlinux-pkgforge: an HTTPS front for the GHCR mirror. +// +// Server = https://ghcr.mirror.archlinux.pkgforge.dev/$arch/$repo +// +// pacman uses its own downloader with this Server: ParallelDownloads, resume +// and the server error limit apply, and no XferCommand is necessary. +// +// The Worker sends no package bytes. It finds the blob digest for the file, +// asks GHCR for that blob, and returns GHCR's redirect to the signed blob URL. +// pacman then reads the bytes from GitHub's blob host. pacman checks each +// file's sha256 against the database and each signature against the keyring, +// as for every mirror. +// +// Paths: +// /ARCH/REPO/FILE the database from tag latest +// /ARCH/REPO:TAG/FILE the database from a tag, e.g. :2026.10.14 +// /ARCH/REPO@sha256:HEX/FILE the database from one manifest +// A package file and its .sig always come from the package's own tag. +// +// Cost control: the Cache API holds each manifest (immutable references for a +// day or a year, moving ones for 60 s) and each signed redirect until shortly +// before it expires. A cache hit costs no subrequest. + +const REGISTRY = 'https://ghcr.io/v2/pkgforge-dev/mirror-archlinux'; +const AUTH = 'Bearer QQ=='; +const MANIFEST_TYPE = 'application/vnd.oci.image.manifest.v1+json'; +const MANIFEST_MAX = 65536; +const CACHE_ORIGIN = 'https://cache.invalid'; + +const PATH_RE = + /^\/([a-z0-9_]{1,32})\/([a-z0-9]+(?:-[a-z0-9]+)*)(?::([A-Za-z0-9_][A-Za-z0-9._-]{0,127})|@(sha256:[0-9a-f]{64}))?\/([!-.0-~]{1,255})$/; + +const enc = new TextEncoder(); + +function text(status, body, extra = {}) { + return new Response(body + '\n', { + status, + headers: { 'content-type': 'text/plain; charset=utf-8', 'cache-control': 'no-store', ...extra }, + }); +} + +async function sha256hex(s) { + const d = new Uint8Array(await crypto.subtle.digest('SHA-256', enc.encode(s))); + let h = ''; + for (const b of d) h += b.toString(16).padStart(2, '0'); + return h; +} + +// The tag a package file is stored under. The same mapping as +// `client/pacman-oci --tag`; tests/unit/50-worker.sh compares the two. +export async function tagOf(name) { + const t = name.replace(/_([pcau])/g, '_u$1').replace(/\+/g, '_p').replace(/:/g, '_c').replace(/@/g, '_a'); + if (/^[a-z0-9@._+:-]+$/.test(name) && /^[a-z0-9_][a-z0-9._-]*$/.test(t) && t.length <= 128) return t; + return 'fn-' + (await sha256hex(name)); +} + +// A reference whose manifest never changes, or changes only to gain a +// repaired signature layer. +function ttlOf(ref, kind) { + if (ref.startsWith('sha256:')) return 31536000; + if (/^\d{4}\.\d{2}\.\d{2}$/.test(ref) || /^(db|files)-[0-9a-f]{64}$/.test(ref)) return 31536000; + if (kind === 'pkg' || kind === 'pkgsig') return 86400; + return 60; +} + +function cacheKey(...parts) { + return new Request(CACHE_ORIGIN + '/' + parts.map(encodeURIComponent).join('/')); +} + +// The layer digest for FILE in REPO:REF, or a Response that ends the request. +async function layerDigest(ctx, repoPath, ref, kind, file) { + const cache = caches.default; + const key = cacheKey('m', repoPath, ref); + let res = await cache.match(key); + if (!res) { + const up = await fetch(`${REGISTRY}/${repoPath}/manifests/${ref}`, { + headers: { authorization: AUTH, accept: MANIFEST_TYPE }, + redirect: 'manual', + }); + const ttl = ttlOf(ref, kind); + if (up.status === 404) { + res = new Response('', { status: 404, headers: { 'cache-control': 'max-age=60' } }); + ctx.waitUntil(cache.put(key, res.clone())); + } else if (up.status !== 200) { + up.body?.cancel(); + return text(502, `registry answered ${up.status} for ${repoPath}:${ref}`, { 'retry-after': '30' }); + } else { + const body = await up.text(); + if (body.length > MANIFEST_MAX) return text(502, `manifest for ${repoPath}:${ref} is over 64 KiB`); + res = new Response(body, { status: 200, headers: { 'cache-control': `max-age=${ttl}` } }); + ctx.waitUntil(cache.put(key, res.clone())); + } + } + if (res.status === 404) return text(404, `no ${repoPath}:${ref}`); + let m; + try { + m = JSON.parse(await res.text()); + } catch { + return text(502, `manifest for ${repoPath}:${ref} is not JSON`); + } + const hits = (m.layers || []).filter((l) => l?.annotations?.['org.opencontainers.image.title'] === file); + if (hits.length !== 1) return text(404, `${repoPath}:${ref} has ${hits.length} layer(s) named ${file}`); + const digest = hits[0].digest; + if (typeof digest !== 'string' || !/^sha256:[0-9a-f]{64}$/.test(digest)) { + return text(502, `layer ${file} has no sha256 digest`); + } + return digest; +} + +// GHCR answers a blob request with 307 to a signed URL that expires at `se`. +async function blobRedirect(ctx, repoPath, digest) { + const cache = caches.default; + const key = cacheKey('b', repoPath, digest); + const hit = await cache.match(key); + if (hit) return new Response(null, { status: 302, headers: { location: await hit.text(), 'cache-control': 'no-store' } }); + const up = await fetch(`${REGISTRY}/${repoPath}/blobs/${digest}`, { + headers: { authorization: AUTH }, + redirect: 'manual', + }); + if (up.status === 404) return text(404, `no blob ${digest}`); + if (up.status === 200) return up; // a registry that serves the blob itself + up.body?.cancel(); + const loc = up.headers.get('location'); + if (up.status < 300 || up.status > 399 || !loc || !loc.startsWith('https://')) { + return text(502, `registry answered ${up.status} for blob ${digest}`, { 'retry-after': '30' }); + } + const se = Date.parse(new URL(loc).searchParams.get('se') || ''); + const ttl = Math.floor((se - Date.now()) / 1000) - 120; + if (ttl >= 30) { + ctx.waitUntil(cache.put(key, new Response(loc, { headers: { 'cache-control': `max-age=${Math.min(ttl, 600)}` } }))); + } + return new Response(null, { status: 302, headers: { location: loc, 'cache-control': 'no-store' } }); +} + +export default { + async fetch(request, env, ctx) { + if (request.method !== 'GET' && request.method !== 'HEAD') return text(405, 'GET or HEAD only', { allow: 'GET, HEAD' }); + const url = new URL(request.url); + if (url.pathname === '/') { + return text(200, 'Server = https://' + url.host + '/$arch/$repo\nhttps://github.com/pkgforge-dev/mirror-archlinux', { + 'cache-control': 'max-age=3600', + }); + } + const p = PATH_RE.exec(url.pathname); + if (!p) return text(404, 'not /ARCH/REPO/FILE'); + const [, arch, repo, tag, pin, file] = p; + if (file === '.' || file === '..') return text(404, 'not a file name'); + const repoPath = `${arch}/${repo}`; + let kind, ref, layer; + if (/\.pkg\.tar(\.[^/]+)?\.sig$/.test(file)) { + kind = 'pkgsig'; + ref = await tagOf(file.slice(0, -4)); + } else if (/\.pkg\.tar(\.[^/]+)?$/.test(file)) { + kind = 'pkg'; + ref = await tagOf(file); + } else { + kind = 'db'; + ref = pin || tag || 'latest'; + } + layer = file; + const digest = await layerDigest(ctx, repoPath, ref, kind, layer); + if (digest instanceof Response) return digest; + return blobRedirect(ctx, repoPath, digest); + }, +}; diff --git a/worker/wrangler.toml b/worker/wrangler.toml new file mode 100644 index 0000000..75f2eb6 --- /dev/null +++ b/worker/wrangler.toml @@ -0,0 +1,20 @@ +# ghcr-mirror-archlinux-pkgforge: the HTTPS front for the GHCR mirror. +# Deploy: npx wrangler@4 deploy --config worker/wrangler.toml +name = "ghcr-mirror-archlinux-pkgforge" +main = "src/index.js" +account_id = "2a97c8f21aff7a773bf976c61ace1381" +compatibility_date = "2026-10-01" +workers_dev = false +preview_urls = false + +routes = [ + { pattern = "ghcr.mirror.archlinux.pkgforge.dev", custom_domain = true }, +] + +# Each request is a cache lookup and at most two small subrequests. +[limits] +cpu_ms = 50 + +[observability] +enabled = true +head_sampling_rate = 0.01 From c1b5e25ad6032744a6bd4ea0bec128d8c9d1bfdf Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 10:51:03 +0545 Subject: [PATCH 02/22] worker: v2 redirect front with checked pins, versioned cache, tests and deploy workflow --- .github/workflows/worker.yml | 95 +++++++ tests/worker/10-node.sh | 65 +++++ tests/worker/live.sh | 192 ++++++++++++++ worker/.gitignore | 2 + worker/package.json | 6 + worker/src/index.js | 483 +++++++++++++++++++++++++++++++++++ worker/test/cache.test.js | 302 ++++++++++++++++++++++ worker/test/config.test.js | 64 +++++ worker/test/ghcr.test.js | 310 ++++++++++++++++++++++ worker/test/paths.test.js | 209 +++++++++++++++ worker/test/tags.test.js | 84 ++++++ worker/test/world.js | 293 +++++++++++++++++++++ worker/wrangler.toml | 34 +++ 13 files changed, 2139 insertions(+) create mode 100644 .github/workflows/worker.yml create mode 100644 tests/worker/10-node.sh create mode 100644 tests/worker/live.sh create mode 100644 worker/.gitignore create mode 100644 worker/package.json create mode 100644 worker/src/index.js create mode 100644 worker/test/cache.test.js create mode 100644 worker/test/config.test.js create mode 100644 worker/test/ghcr.test.js create mode 100644 worker/test/paths.test.js create mode 100644 worker/test/tags.test.js create mode 100644 worker/test/world.js create mode 100644 worker/wrangler.toml diff --git a/.github/workflows/worker.yml b/.github/workflows/worker.yml new file mode 100644 index 0000000..43c82e2 --- /dev/null +++ b/.github/workflows/worker.yml @@ -0,0 +1,95 @@ +# The Cloudflare Worker in worker/ (ghcr.mirror.archlinux.pkgforge.dev). +# +# test node --test 'worker/test/*.test.js' in the pinned node image +# deploy wrangler deploy from main, then tests/worker/live.sh against the +# deployed Worker. With no CLOUDFLARE_API_TOKEN secret, the job +# shows a notice and ends with success, with no deploy. +# +# The Worker holds no secret: the API token is used only by wrangler-action, +# and the account id is in worker/wrangler.toml. +name: Worker + +on: + push: + branches: [main] + paths: + - 'worker/**' + - '.github/workflows/worker.yml' + workflow_dispatch: + +permissions: {} + +concurrency: + group: worker-${{ github.ref }} + cancel-in-progress: false + +defaults: + run: + shell: bash + +jobs: + test: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Worker tests in the pinned node image + run: | + # tag: 24-alpine + image=docker.io/library/node:24-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1 + docker run --rm -v "$PWD:/work:ro" -w /work "$image" node --test 'worker/test/*.test.js' + + deploy: + needs: test + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Check for the Cloudflare API token + id: token + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + run: | + if [ -z "$CLOUDFLARE_API_TOKEN" ]; then + echo "::notice::The CLOUDFLARE_API_TOKEN secret is not set, so the Worker is not deployed. Make it from the Edit Cloudflare Workers token template." + echo "deploy=false" >> "$GITHUB_OUTPUT" + else + echo "deploy=true" >> "$GITHUB_OUTPUT" + fi + - name: Deploy the Worker + if: steps.token.outputs.deploy == 'true' + uses: cloudflare/wrangler-action@953926a2e2182532811c01a25e53647d93bf07c0 # v4.1.3 + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + wranglerVersion: "4.150.0" + workingDirectory: worker + command: deploy + - name: Check the deployed Worker + if: steps.token.outputs.deploy == 'true' + env: + WORKER_LIVE: '1' + run: | + # A new version can take some seconds to reach every data centre, so + # the check runs up to 3 times. Each failed attempt stays in the log. + rc=0 + for attempt in 1 2 3; do + rc=0 + REPO_ROOT="$PWD" bash tests/worker/live.sh || rc=$? + if [ "$rc" -eq 0 ]; then + exit 0 + fi + echo "::warning::tests/worker/live.sh attempt $attempt ended $rc" + if [ "$attempt" -lt 3 ]; then + sleep 20 + fi + done + exit "$rc" diff --git a/tests/worker/10-node.sh b/tests/worker/10-node.sh new file mode 100644 index 0000000..1970249 --- /dev/null +++ b/tests/worker/10-node.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# +# The Worker's own tests: node --test over worker/test/*.test.js. +# +# They run worker/src/index.js against a mock fetch (a GHCR stand-in) and a +# Cache API stub with a clock. No network. One test runs client/pacman-oci +# --tags under sh and compares every tag with the Worker's tagOf. +# +# REPO_ROOT=$PWD bash tests/worker/10-node.sh +# +# Needs node 22 or newer and sh on PATH. Each node test is one TAP line here. +set -euo pipefail +# shellcheck source=../lib/harness.sh +. "$REPO_ROOT/tests/lib/harness.sh" + +IMAGE_HINT="the pinned image in .github/workflows/worker.yml" +REPRODUCE="node --test 'worker/test/*.test.js'" + +if ! command -v node > /dev/null 2>&1; then + fail "node is on PATH" "the worker suite needs node 22 or newer" "reproduce: run $REPRODUCE in $IMAGE_HINT" + summary + exit 1 +fi +major="$(node -p 'process.versions.node.split(".")[0]')" +if [ "$major" -lt 22 ]; then + fail "node is 22 or newer" "node on PATH is $(node --version)" "reproduce: run $REPRODUCE in $IMAGE_HINT" + summary + exit 1 +fi +ok "node $(node --version) is 22 or newer" + +work="$(work_dir)" +trap 'rm -rf "$work"' EXIT + +cd "$REPO_ROOT" +rc=0 +node --test --test-reporter=tap 'worker/test/*.test.js' > "$work/node.tap" 2>&1 || rc=$? + +# Each top-level test point becomes one line of this plan. Subtests and the +# YAML diagnostics are indented and stay in the log below on a failure. +points=0 +while IFS= read -r line; do + case "$line" in + 'ok '*) + points=$((points + 1)) + ok "node: ${line#ok * - }" + ;; + 'not ok '*) + points=$((points + 1)) + fail "node: ${line#not ok * - }" "reproduce: $REPRODUCE" + ;; + esac +done < "$work/node.tap" + +if [ "$points" -lt 40 ]; then + fail "node ran the worker tests" "only $points top-level tests reported" "$(sed -n 1,40p "$work/node.tap")" "reproduce: $REPRODUCE" +fi +if [ "$rc" -eq 0 ]; then + ok "node --test exits 0" +else + fail "node --test exits 0" "exit $rc; the TAP log follows" "reproduce: $REPRODUCE" + sed -e 's/^/# /' "$work/node.tap" +fi + +summary diff --git a/tests/worker/live.sh b/tests/worker/live.sh new file mode 100644 index 0000000..4c4763d --- /dev/null +++ b/tests/worker/live.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# +# A smoke test of the deployed Worker. It runs only when WORKER_LIVE=1, so the +# worker suite stays free of network otherwise. +# +# WORKER_LIVE=1 REPO_ROOT=$PWD bash tests/worker/live.sh +# +# WORKER_URL names the Worker (default https://ghcr.mirror.archlinux.pkgforge.dev). +# WORKER_ARCH and WORKER_REPO name the repository (default x86_64 core). +# It only reads: from the Worker, from GHCR with the anonymous token, and from +# the blob host. It needs curl, tar, gzip, awk and sha256sum. +set -euo pipefail +# shellcheck source=../lib/harness.sh +. "$REPO_ROOT/tests/lib/harness.sh" + +if [ "${WORKER_LIVE:-0}" != 1 ]; then + ok "skipped: set WORKER_LIVE=1 to check the deployed Worker" + summary + exit 0 +fi + +BASE="${WORKER_URL:-https://ghcr.mirror.archlinux.pkgforge.dev}" +BASE="${BASE%/}" +ARCH="${WORKER_ARCH:-x86_64}" +REPO="${WORKER_REPO:-core}" +GHCR="https://ghcr.io/v2/pkgforge-dev/mirror-archlinux/$ARCH/$REPO" +MANIFEST_TYPE='application/vnd.oci.image.manifest.v1+json' + +work="$(work_dir)" +trap 'rm -rf "$work"' EXIT + +# probe NAME METHOD PATH: one request to the Worker, no redirect followed. +# Sets CODE, LOCATION and SECONDS_TAKEN; the body is in $work/NAME.body. +probe() { + local name="$1" method="$2" path="$3" out rc=0 + local -a args=(-sS --connect-timeout 10 --max-time 60 -o "$work/$name.body" -D "$work/$name.headers" + -w '%{http_code} %{time_total}') + if [ "$method" = HEAD ]; then + args+=(-I) + fi + out="$(curl "${args[@]}" "$BASE$path")" || rc=$? + if [ "$rc" -ne 0 ]; then + CODE="curl-$rc" LOCATION="" SECONDS_TAKEN="" + return 0 + fi + CODE="${out%% *}" + SECONDS_TAKEN="${out#* }" + LOCATION="$(awk 'tolower($1) == "location:" { sub(/\r$/, "", $2); print $2 }' "$work/$name.headers")" +} + +# fetch URL FILE: follow a signed URL; prints the HTTP code. +fetch() { + local out rc=0 + out="$(curl -sS --connect-timeout 10 --max-time 300 -o "$2" -w '%{http_code}' "$1")" || rc=$? + if [ "$rc" -ne 0 ]; then + printf 'curl-%s\n' "$rc" + else + printf '%s\n' "$out" + fi +} + +# location_ok: the Location is https on *.githubusercontent.com. +location_ok() { + case "$LOCATION" in + https://*.githubusercontent.com/*) return 0 ;; + *) return 1 ;; + esac +} + +# The blob digest a signed URL names, from its path. +digest_of() { + printf '%s\n' "$1" | sed -n 's|^https://[^/]*/.*/blobs/sha256:\([0-9a-f]\{64\}\).*$|\1|p' +} + +#---------------------------------------------------------------------------# +# The usage text +#---------------------------------------------------------------------------# +probe root GET / +check "GET / answers 200" 200 "$CODE" "curl -sS $BASE/" +if grep -qF "Server = https://" "$work/root.body"; then + ok "GET / shows the Server line" +else + fail "GET / shows the Server line" "$(sed -n 1,5p "$work/root.body")" "reproduce: curl -sS $BASE/" +fi + +#---------------------------------------------------------------------------# +# The database, by latest, by HEAD, by pin, and the refusals +#---------------------------------------------------------------------------# +probe db GET "/$ARCH/$REPO/$REPO.db" +check "GET $REPO.db answers 302" 302 "$CODE" "curl -sS -o /dev/null -w '%{http_code}' $BASE/$ARCH/$REPO/$REPO.db" +diag "GET /$ARCH/$REPO/$REPO.db took $SECONDS_TAKEN s" +if location_ok; then + ok "the $REPO.db Location is https on *.githubusercontent.com" +else + fail "the $REPO.db Location is https on *.githubusercontent.com" "Location: $LOCATION" "reproduce: curl -sSI $BASE/$ARCH/$REPO/$REPO.db" +fi +db_url="$LOCATION" +db_digest="$(digest_of "$db_url")" + +probe dbhead HEAD "/$ARCH/$REPO/$REPO.db" +check "HEAD $REPO.db answers 302" 302 "$CODE" "curl -sSI $BASE/$ARCH/$REPO/$REPO.db" +if location_ok; then + ok "HEAD $REPO.db has a Location on *.githubusercontent.com" +else + fail "HEAD $REPO.db has a Location on *.githubusercontent.com" "Location: $LOCATION" "reproduce: curl -sSI $BASE/$ARCH/$REPO/$REPO.db" +fi + +got="$(fetch "$db_url" "$work/$REPO.db")" +check "the signed URL of $REPO.db answers 200" 200 "$got" "curl -sS -o /dev/null -w '%{http_code}' ''" +check "$REPO.db hashes to the digest its Location names" "$db_digest" "$(sha256sum "$work/$REPO.db" | cut -d ' ' -f 1)" \ + "curl -sSL $BASE/$ARCH/$REPO/$REPO.db | sha256sum" + +# What GHCR says latest is, read directly with the anonymous token. +rc=0 +curl -sS --connect-timeout 10 --max-time 60 -H 'Authorization: Bearer QQ==' -H "Accept: $MANIFEST_TYPE" \ + -D "$work/latest.headers" -o "$work/latest.json" "$GHCR/manifests/latest" || rc=$? +check "GHCR answers for $ARCH/$REPO:latest" 0 "$rc" "curl -sS -H 'Authorization: Bearer QQ==' $GHCR/manifests/latest" +pin="$(awk 'tolower($1) == "docker-content-digest:" { sub(/\r$/, "", $2); print $2 }' "$work/latest.headers")" +ghcr_db="$(sed -n 's/.*"dev\.pkgforge\.mirror\.db-sha256":"\([0-9a-f]\{64\}\)".*/\1/p' "$work/latest.json")" +# A promote between the two reads is the one harmless cause of a difference. +check "the Worker's $REPO.db is the one GHCR latest names" "$ghcr_db" "$db_digest" \ + "compare $BASE/$ARCH/$REPO/$REPO.db with the db-sha256 annotation of $GHCR/manifests/latest" + +probe pin GET "/$ARCH/$REPO@$pin/$REPO.db" +check "GET @$pin/$REPO.db answers 302" 302 "$CODE" "curl -sSI $BASE/$ARCH/$REPO@$pin/$REPO.db" +check "the pin names the same $REPO.db" "$db_digest" "$(digest_of "$LOCATION")" "curl -sSI $BASE/$ARCH/$REPO@$pin/$REPO.db" + +zero="sha256:$(printf '%064d' 0)" +for path in "/$ARCH/$REPO@$zero/$REPO.db" "/$ARCH/$REPO:staging/$REPO.db" "/$ARCH/$REPO/$REPO.db.sig" "/$ARCH/$REPO/README" "/$ARCH/$REPO/extra-$REPO.db"; do + probe refused GET "$path" + check "GET $path answers 404" 404 "$CODE" "curl -sS $BASE$path" +done + +probe files GET "/$ARCH/$REPO/$REPO.files" +check "GET $REPO.files answers 302" 302 "$CODE" "curl -sSI $BASE/$ARCH/$REPO/$REPO.files" + +#---------------------------------------------------------------------------# +# A package and its .sig, named by the database. One with a + in its name is +# also asked for with %2B. +#---------------------------------------------------------------------------# +mkdir "$work/db" +rc=0 +tar -xf "$work/$REPO.db" -C "$work/db" || rc=$? +check "$REPO.db unpacks with tar" 0 "$rc" "curl -sSL $BASE/$ARCH/$REPO/$REPO.db | tar -t" +awk ' + FNR == 1 { if (f != "") print f "\t" s; f = ""; s = "" } + prev == "%FILENAME%" { f = $0 } + prev == "%SHA256SUM%" { s = $0 } + { prev = $0 } + END { if (f != "") print f "\t" s } +' "$work"/db/*/desc | sort > "$work/files.tsv" +pick="$(awk -F '\t' '$1 ~ /[+]/ { print; n++ } END { if (n == 0) print "" }' "$work/files.tsv" | sed -n 1p)" +if [ -z "$pick" ]; then + pick="$(sed -n 1p "$work/files.tsv")" +fi +file="${pick%%$'\t'*}" +sum="${pick#*$'\t'}" +if [ -z "$file" ] || [ -z "$sum" ]; then + fail "$REPO.db names a package with a sha256" "files.tsv: $(sed -n 1,3p "$work/files.tsv")" "reproduce: tar -xOf $REPO.db --wildcards '*/desc'" +else + ok "$REPO.db names $file" + probe pkg GET "/$ARCH/$REPO/$file" + check "GET $file answers 302" 302 "$CODE" "curl -sSI '$BASE/$ARCH/$REPO/$file'" + diag "GET $file took $SECONDS_TAKEN s (cold or warm)" + check "the Location of $file names its %SHA256SUM%" "$sum" "$(digest_of "$LOCATION")" "curl -sSI '$BASE/$ARCH/$REPO/$file'" + probe pkg2 GET "/$ARCH/$REPO/$file" + check "GET $file again answers 302" 302 "$CODE" "curl -sSI '$BASE/$ARCH/$REPO/$file'" + diag "GET $file again took $SECONDS_TAKEN s" + + probe sig GET "/$ARCH/$REPO/$file.sig" + check "GET $file.sig answers 302" 302 "$CODE" "curl -sSI '$BASE/$ARCH/$REPO/$file.sig'" + diag "GET $file.sig took $SECONDS_TAKEN s" + got="$(fetch "$LOCATION" "$work/pkg.sig")" + check "the signed URL of $file.sig answers 200" 200 "$got" "curl -sSL '$BASE/$ARCH/$REPO/$file.sig' | wc -c" + if [ -s "$work/pkg.sig" ]; then + ok "$file.sig is not empty" + else + fail "$file.sig is not empty" "0 bytes" "reproduce: curl -sSL '$BASE/$ARCH/$REPO/$file.sig' | wc -c" + fi + + case "$file" in + *+*) + encoded="${file//+/%2B}" + probe enc GET "/$ARCH/$REPO/$encoded" + check "GET $encoded answers 302" 302 "$CODE" "curl -sSI '$BASE/$ARCH/$REPO/$encoded'" + check "the %2B name gets the same blob" "$sum" "$(digest_of "$LOCATION")" "curl -sSI '$BASE/$ARCH/$REPO/$encoded'" + ;; + *) diag "no package name in $REPO.db has a +; the %2B check did not run" ;; + esac +fi + +summary diff --git a/worker/.gitignore b/worker/.gitignore new file mode 100644 index 0000000..41a2578 --- /dev/null +++ b/worker/.gitignore @@ -0,0 +1,2 @@ +.wrangler/ +node_modules/ diff --git a/worker/package.json b/worker/package.json new file mode 100644 index 0000000..1c4a850 --- /dev/null +++ b/worker/package.json @@ -0,0 +1,6 @@ +{ + "name": "ghcr-mirror-archlinux-pkgforge", + "private": true, + "description": "The HTTPS front for the GHCR mirror. No dependencies: tests/worker/10-node.sh runs the tests.", + "type": "module" +} diff --git a/worker/src/index.js b/worker/src/index.js new file mode 100644 index 0000000..87b9fd1 --- /dev/null +++ b/worker/src/index.js @@ -0,0 +1,483 @@ +// ghcr-mirror-archlinux-pkgforge, v2: an HTTPS front for the GHCR mirror. +// +// Server = https://ghcr.mirror.archlinux.pkgforge.dev/$arch/$repo +// +// pacman uses its own downloader with this Server: ParallelDownloads, resume +// and the server error limit apply, and no XferCommand is necessary. +// +// The Worker sends no package bytes. It finds the blob digest of the file in +// the manifest, asks GHCR for that blob, and returns GHCR's redirect to the +// signed blob URL. A GHCR answer that holds the bytes gets 502. pacman checks +// the sha256 of each file against the database and each signature against the +// keyring, as for every mirror. +// +// Paths: +// / this usage text +// /ARCH/REPO/FILE FILE from tag latest +// /ARCH/REPO:latest/FILE the same +// /ARCH/REPO:YYYY.MM.DD/FILE the first snapshot of that UTC day +// /ARCH/REPO:db-HEX/FILE the snapshot whose REPO.db has sha256 HEX +// /ARCH/REPO:files-HEX/FILE the snapshot whose REPO.files has sha256 HEX +// /ARCH/REPO@sha256:HEX/FILE one manifest; its body must hash to HEX +// FILE is REPO.db, REPO.files, a package file, or one of these plus .sig. +// FILE can be percent-encoded. A package file and its .sig always come from +// the package's own tag. Any other tag (staging is one) and any other path +// gets 404 with no subrequest. +// +// Answers: 302 to the signed URL; 404 when the mirror does not hold the file; +// 502 with retry-after: 30 when GHCR fails or answers something unexpected. +// HEAD gets the same answer as GET, with no body. +// +// Cache: the Cache API, one cache per data centre. Every key is on the zone +// host under /__cache/v1, so a change of entry format only needs a new version. +// loc one key per (repository, reference, file): the final answer. A hit +// is the whole request. TTL: until the signed URL's se minus 120 s, at +// most 600 s, and never past the manifest entry it came from. A signed +// URL with less than 30 s left before se minus 120 s is not stored. +// man one key per (repository, reference): the layers of the manifest. +// TTL: pins, dated, db- and files- tags 1 year; a package manifest +// with FILE and FILE.sig 6 h, without the .sig 60 s; latest 30 s; +// a 404 60 s. +// blob one key per (repository, digest): the signed URL, TTL as for loc. +// A package request also fills the loc entry of its .sig (ctx.waitUntil). +// +// Logs: console.error on every 5xx path and on every cache failure. Nothing +// else is logged. +// +// Tests: tests/worker/10-node.sh runs worker/test/*.test.js. tests/worker/live.sh +// checks the deployed Worker when WORKER_LIVE=1. + +const REGISTRY = 'https://ghcr.io/v2/pkgforge-dev/mirror-archlinux'; +const AUTH = 'Bearer QQ=='; +const MANIFEST_TYPE = 'application/vnd.oci.image.manifest.v1+json'; +const TITLE = 'org.opencontainers.image.title'; +const BLOB_HOST_SUFFIX = '.githubusercontent.com'; +const CACHE_PATH = '/__cache/v1'; + +export const LIMITS = Object.freeze({ + manifestBytes: 65536, + subrequestMs: 10000, + retryAfter: '30', +}); + +// Seconds. +export const TTL = Object.freeze({ + immutable: 31536000, + packageWithSig: 21600, + packageNoSig: 60, + latest: 30, + missing: 60, + urlMax: 600, + urlMin: 30, + urlMargin: 120, +}); + +// The URL before the file name is decoded. ARCH and REPO are the GHCR path +// names; a reference is a tag or a pin. FILE is printable ASCII with no /, +// which is all the URL parser leaves. +const PATH_RE = + /^\/([a-z0-9][a-z0-9_]{0,31})\/([a-z0-9]+(?:-[a-z0-9]+)*)(?::([^/@:]{1,128})|@sha256:([0-9a-f]{64}))?\/([!-.0-~]{1,1024})$/; +const TAG_RE = /^(?:latest|[0-9]{4}\.[0-9]{2}\.[0-9]{2}|(?:db|files)-[0-9a-f]{64})$/; +const IMMUTABLE_TAG_RE = /^(?:[0-9]{4}\.[0-9]{2}\.[0-9]{2}|(?:db|files)-[0-9a-f]{64})$/; +// The file name after decoding: the same character set, at most 255 bytes. +const FILE_RE = /^[!-.0-~]{1,255}$/; +// The same globs as client/pacman-oci: *.pkg.tar.sig | *.pkg.tar.*.sig, and +// *.pkg.tar | *.pkg.tar.*. +const PKGSIG_RE = /\.pkg\.tar(?:\..*)?\.sig$/; +const PKG_RE = /\.pkg\.tar(?:\..*)?$/; +const DIGEST_RE = /^sha256:[0-9a-f]{64}$/; + +const enc = new TextEncoder(); + +async function sha256hex(data) { + const bytes = typeof data === 'string' ? enc.encode(data) : data; + const d = new Uint8Array(await crypto.subtle.digest('SHA-256', bytes)); + let h = ''; + for (const b of d) h += b.toString(16).padStart(2, '0'); + return h; +} + +// The tag a package file is stored under. The same mapping as +// `client/pacman-oci --tag`; worker/test/tags.test.js compares the two for +// every name in tests/fixtures/filenames.txt. +export async function tagOf(name) { + const t = name.replace(/_([pcau])/g, '_u$1').replace(/\+/g, '_p').replace(/:/g, '_c').replace(/@/g, '_a'); + if (/^[a-z0-9@._+:-]+$/.test(name) && /^[a-z0-9_][a-z0-9._-]*$/.test(t) && t.length <= 128) return t; + return 'fn-' + (await sha256hex(name)); +} + +function text(status, body, extra = {}) { + return new Response(body + '\n', { + status, + headers: { + 'content-type': 'text/plain; charset=utf-8', + 'cache-control': 'no-store', + 'x-content-type-options': 'nosniff', + ...extra, + }, + }); +} + +function logError(job, status, error) { + console.error({ status, path: job.path, error }); +} + +// Every failure of the Worker or of GHCR is 502 with retry-after, so pacman +// tries the next Server now and this one later. +function fail(job, error) { + logError(job, 502, error); + return text(502, error, { 'retry-after': LIMITS.retryAfter }); +} + +// A body this Worker does not read is cancelled, so that it frees memory. +async function discard(job, res) { + if (!res || !res.body) return; + try { + await res.body.cancel(); + } catch (e) { + logError(job, 500, `could not cancel an unused body: ${e}`); + } +} + +// At most MAX bytes of the body, or null when it is larger. The +// content-length header is read first, then the bytes themselves. +async function readCapped(job, res, max) { + const len = res.headers.get('content-length'); + if (len !== null && Number(len) > max) { + await discard(job, res); + return null; + } + if (!res.body) return new Uint8Array(0); + const reader = res.body.getReader(); + const chunks = []; + let n = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + n += value.byteLength; + if (n > max) { + await reader.cancel(); + return null; + } + chunks.push(value); + } + const out = new Uint8Array(n); + let o = 0; + for (const c of chunks) { + out.set(c, o); + o += c.byteLength; + } + return out; +} + +// One GHCR subrequest: anonymous token, no redirect followed, 10 s deadline. +function ghcr(url, headers = {}) { + return fetch(url, { + headers: { authorization: AUTH, ...headers }, + redirect: 'manual', + signal: AbortSignal.timeout(LIMITS.subrequestMs), + }); +} + +//---------------------------------------------------------------------------// +// Cache entries are JSON with an `expires` time in ms. The Cache API also +// drops them at max-age; `expires` lets a derived entry end no later than +// the entry it came from. +//---------------------------------------------------------------------------// +function cacheKey(host, kind, parts) { + return new Request(`https://${host}${CACHE_PATH}/${kind}/${parts.map(encodeURIComponent).join('/')}`); +} + +async function cacheGet(job, key) { + let res; + try { + res = await caches.default.match(key); + } catch (e) { + logError(job, 500, `cache match ${key.url} failed: ${e}`); + return null; + } + if (!res) return null; + let v; + try { + v = await res.json(); + } catch (e) { + logError(job, 500, `cache entry ${key.url} is not JSON: ${e}`); + return null; + } + if (!v || typeof v.expires !== 'number' || v.expires <= Date.now()) return null; + return v; +} + +async function cachePut(job, key, entry, ttl) { + try { + await caches.default.put( + key, + new Response(JSON.stringify(entry), { + headers: { 'content-type': 'application/json', 'cache-control': `max-age=${ttl}` }, + }), + ); + } catch (e) { + logError(job, 500, `cache put ${key.url} failed: ${e}`); + } +} + +function store(job, key, entry, ttl) { + job.defer(cachePut(job, key, entry, ttl)); +} + +function secondsLeft(expires) { + return Math.floor((expires - Date.now()) / 1000); +} + +//---------------------------------------------------------------------------// +// Requests +//---------------------------------------------------------------------------// + +// The request the path names, or { refuse } for a 404 with no subrequest. +export async function parsePath(pathname) { + const p = PATH_RE.exec(pathname); + if (!p) return { refuse: 'not /ARCH/REPO/FILE, /ARCH/REPO:TAG/FILE or /ARCH/REPO@sha256:HEX/FILE' }; + const [, arch, repo, tag, pin, raw] = p; + if (tag !== undefined && !TAG_RE.test(tag)) { + return { refuse: `tag ${tag} is not served: only latest, YYYY.MM.DD, db-HEX, files-HEX and @sha256:HEX` }; + } + let file; + try { + file = decodeURIComponent(raw); + } catch { + return { refuse: 'the file name is not valid percent-encoding' }; + } + if (!FILE_RE.test(file) || file === '.' || file === '..') return { refuse: 'not a file name pacman asks for' }; + const r = { arch, repo, file, repoPath: `${arch}/${repo}`, pin: null, pkgFile: null }; + if (PKGSIG_RE.test(file)) { + r.kind = 'pkgsig'; + r.pkgFile = file.slice(0, -4); + } else if (PKG_RE.test(file)) { + r.kind = 'pkg'; + r.pkgFile = file; + } else if ([`${repo}.db`, `${repo}.files`, `${repo}.db.sig`, `${repo}.files.sig`].includes(file)) { + r.kind = 'db'; + } else { + return { refuse: `${file} is not ${repo}.db, ${repo}.files, a package file or the .sig of one` }; + } + if (r.pkgFile !== null) { + r.ref = await tagOf(r.pkgFile); + } else if (pin !== undefined) { + r.ref = `sha256:${pin}`; + r.pin = pin; + } else { + r.ref = tag || 'latest'; + } + r.repoRef = r.ref.startsWith('sha256:') ? `${r.repoPath}@${r.ref}` : `${r.repoPath}:${r.ref}`; + return r; +} + +function manifestTtl(r, layers) { + if (r.ref.startsWith('sha256:') || IMMUTABLE_TAG_RE.test(r.ref)) return TTL.immutable; + if (r.pkgFile !== null) { + const n = (title) => layers.filter(([t]) => t === title).length; + return n(r.pkgFile) === 1 && n(`${r.pkgFile}.sig`) === 1 ? TTL.packageWithSig : TTL.packageNoSig; + } + return TTL.latest; +} + +// The manifest entry { found, layers, expires }, or a 502 Response. +async function manifestOf(job, r) { + const key = cacheKey(job.host, 'man', [r.arch, r.repo, r.ref]); + const hit = await cacheGet(job, key); + if (hit) return hit; + let up; + try { + up = await ghcr(`${REGISTRY}/${r.repoPath}/manifests/${r.ref}`, { accept: MANIFEST_TYPE }); + } catch (e) { + return fail(job, `the GHCR request for the manifest ${r.repoRef} failed: ${e}`); + } + if (up.status === 404) { + await discard(job, up); + const entry = { found: false, expires: Date.now() + TTL.missing * 1000 }; + store(job, key, entry, TTL.missing); + return entry; + } + if (up.status !== 200) { + await discard(job, up); + return fail(job, `GHCR answered ${up.status} for the manifest ${r.repoRef}`); + } + let bytes; + try { + bytes = await readCapped(job, up, LIMITS.manifestBytes); + } catch (e) { + return fail(job, `reading the manifest ${r.repoRef} failed: ${e}`); + } + if (bytes === null) return fail(job, `the manifest ${r.repoRef} is over ${LIMITS.manifestBytes} bytes`); + if (r.pin !== null) { + const got = await sha256hex(bytes); + if (got !== r.pin) return fail(job, `the manifest ${r.repoRef} hashes to sha256:${got}`); + } + let m; + try { + m = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); + } catch (e) { + return fail(job, `the manifest ${r.repoRef} is not JSON: ${e}`); + } + if (!m || typeof m !== 'object' || !Array.isArray(m.layers)) { + return fail(job, `the manifest ${r.repoRef} has no layers array`); + } + const layers = []; + for (const l of m.layers) { + const title = l?.annotations?.[TITLE]; + if (typeof title === 'string') layers.push([title, typeof l.digest === 'string' ? l.digest : null]); + } + const ttl = manifestTtl(r, layers); + const entry = { found: true, layers, expires: Date.now() + ttl * 1000 }; + store(job, key, entry, ttl); + return entry; +} + +// The blob entry { location, expires }, or a 404 or 502 Response. GHCR +// answers a blob request with 307 to a signed URL that ends at `se`. +async function blobOf(job, r, digest) { + const key = cacheKey(job.host, 'blob', [r.arch, r.repo, digest]); + const hit = await cacheGet(job, key); + if (hit) return hit; + let up; + try { + up = await ghcr(`${REGISTRY}/${r.repoPath}/blobs/${digest}`); + } catch (e) { + return fail(job, `the GHCR request for the blob ${digest} of ${r.repoPath} failed: ${e}`); + } + // The Worker never reads a blob body: it sends redirects only. + await discard(job, up); + if (up.status === 404) return text(404, `GHCR has no blob ${digest} in ${r.repoPath}`); + if (up.status === 200) return fail(job, `GHCR answered 200 with the bytes of ${digest}; this Worker sends redirects only`); + if (up.status < 300 || up.status > 399) return fail(job, `GHCR answered ${up.status} for the blob ${digest}`); + const location = up.headers.get('location'); + let u; + try { + u = new URL(location); + } catch { + return fail(job, `GHCR answered ${up.status} for the blob ${digest} with no valid Location`); + } + if (u.protocol !== 'https:' || !u.hostname.endsWith(BLOB_HOST_SUFFIX)) { + return fail(job, `GHCR redirected the blob ${digest} to ${u.protocol}//${u.hostname}, not https on *${BLOB_HOST_SUFFIX}`); + } + const now = Date.now(); + const se = Date.parse(u.searchParams.get('se') || ''); + const expires = Number.isFinite(se) ? Math.min(se - TTL.urlMargin * 1000, now + TTL.urlMax * 1000) : now; + const entry = { location, expires }; + const ttl = secondsLeft(expires); + if (ttl >= TTL.urlMin) store(job, key, entry, ttl); + return entry; +} + +function respond(answer) { + if (answer.status === 302) { + return new Response(null, { status: 302, headers: { location: answer.location, 'cache-control': 'no-store' } }); + } + return text(404, answer.message); +} + +// Stores the final answer under the loc key until answer.expires. A redirect +// is not stored when its signed URL has less than 30 s left before se minus +// 120 s (urlExpires). +function storeAnswer(job, key, answer, urlExpires) { + const ttl = secondsLeft(answer.expires); + if (ttl < 1) return; + if (answer.status === 302 && secondsLeft(urlExpires) < TTL.urlMin) return; + store(job, key, answer, ttl); +} + +// The answer for FILE from the manifest entry MAN. +async function answerFrom(job, r, man, file) { + const key = cacheKey(job.host, 'loc', [r.arch, r.repo, r.ref, file]); + const missing = (message) => { + const answer = { status: 404, message, expires: Math.min(man.expires, Date.now() + TTL.missing * 1000) }; + storeAnswer(job, key, answer); + return respond(answer); + }; + if (!man.found) return missing(`GHCR has no ${r.repoRef}`); + const hits = man.layers.filter(([t]) => t === file); + if (hits.length !== 1) return missing(`${r.repoRef} has ${hits.length} layers named ${file}`); + const digest = hits[0][1]; + if (digest === null || !DIGEST_RE.test(digest)) { + return fail(job, `the layer ${file} of ${r.repoRef} has no sha256 digest`); + } + const blob = await blobOf(job, r, digest); + if (blob instanceof Response) return blob; + const answer = { status: 302, location: blob.location, expires: Math.min(blob.expires, man.expires) }; + storeAnswer(job, key, answer, blob.expires); + return respond(answer); +} + +// Fills the loc entry of a package's .sig while the package answer is made +// and sent. pacman asks for the .sig next, from the same Server. +async function prewarmSig(parent, r, man) { + const pending = []; + const job = { host: parent.host, path: `${parent.path}.sig`, defer: (p) => pending.push(p) }; + try { + const file = `${r.file}.sig`; + const hit = await cacheGet(job, cacheKey(job.host, 'loc', [r.arch, r.repo, r.ref, file])); + if (!hit) { + const res = await answerFrom(job, r, man, file); + await discard(job, res); + } + } catch (e) { + logError(job, 502, `filling the .sig entry failed: ${e && e.stack ? e.stack : e}`); + } + await Promise.all(pending); +} + +async function route(job, request, ctx) { + const url = new URL(request.url); + if (url.pathname === '/') { + return text(200, usage(url.host), { 'cache-control': 'max-age=3600' }); + } + const r = await parsePath(url.pathname); + if (r.refuse) return text(404, r.refuse); + const hot = await cacheGet(job, cacheKey(job.host, 'loc', [r.arch, r.repo, r.ref, r.file])); + if (hot) return respond(hot); + const man = await manifestOf(job, r); + if (man instanceof Response) return man; + // The .sig blob is asked for at the same time as the package blob. + if (r.kind === 'pkg' && man.found) ctx.waitUntil(prewarmSig(job, r, man)); + return answerFrom(job, r, man, r.file); +} + +function usage(host) { + return [ + `Server = https://${host}/$arch/$repo`, + '', + 'Paths:', + ' /ARCH/REPO/FILE FILE from tag latest', + ' /ARCH/REPO:YYYY.MM.DD/FILE the first snapshot of that UTC day', + ' /ARCH/REPO:db-HEX/FILE the snapshot whose REPO.db has sha256 HEX', + ' /ARCH/REPO:files-HEX/FILE the snapshot whose REPO.files has sha256 HEX', + ' /ARCH/REPO@sha256:HEX/FILE one manifest, checked against HEX', + 'A package file and its .sig always come from the package\'s own tag.', + 'The answer is a redirect to GitHub\'s blob host. pacman checks every file.', + '', + 'https://github.com/pkgforge-dev/mirror-archlinux', + ].join('\n'); +} + +export default { + async fetch(request, env, ctx) { + const url = new URL(request.url); + const job = { host: url.host, path: url.pathname, defer: (p) => ctx.waitUntil(p) }; + let res; + if (request.method !== 'GET' && request.method !== 'HEAD') { + await discard(job, request); + res = text(405, 'GET or HEAD only', { allow: 'GET, HEAD' }); + } else { + try { + res = await route(job, request, ctx); + } catch (e) { + res = fail(job, `internal error: ${e && e.stack ? e.stack : e}`); + } + } + if (request.method === 'HEAD') { + await discard(job, res); + return new Response(null, { status: res.status, headers: res.headers }); + } + return res; + }, +}; diff --git a/worker/test/cache.test.js b/worker/test/cache.test.js new file mode 100644 index 0000000..9f64205 --- /dev/null +++ b/worker/test/cache.test.js @@ -0,0 +1,302 @@ +// The Cache API: keys, the one-lookup hot path, every TTL rule, the .sig +// pre-warm, negative entries and cache failures. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { inWorld, HOST, PKG, PKG_TAG, NOSIG, UPPER, D, digest, hex, manifestJson, sha256hex } from './world.js'; +import { TTL } from '../src/index.js'; + +const ttlOf = (w, kind, match) => { + const hits = w.cache.ofKind(kind).filter((p) => decodeURIComponent(p.url).includes(match)); + assert.ok(hits.length > 0, `a ${kind} entry for ${match} is stored; stored: ${w.cache.puts.map((p) => p.url)}`); + return hits[hits.length - 1].ttl; +}; + +test('every cache key is on the request host, under /__cache/v1', () => + inWorld(async (w) => { + for (const p of ['/x86_64/core/core.db', `/x86_64/core/${PKG}`, '/x86_64/core/core.db.sig', '/x86_64/extra/extra.db']) { + await (await w.get(p)).text(); + } + assert.ok(w.cache.puts.length >= 6, `entries stored: ${w.cache.puts.length}`); + for (const { url } of w.cache.puts) { + assert.ok(url.startsWith(`https://${HOST}/__cache/v1/`), url); + assert.match(new URL(url).pathname, /^\/__cache\/v1\/(loc|man|blob)\//); + } + })); + +test('a repeat request is one cache lookup and no subrequest', () => + inWorld(async (w) => { + const paths = [ + '/x86_64/core:2026.10.14/core.db', + `/x86_64/core/${PKG}`, + `/x86_64/core/${UPPER}`, + '/x86_64/core/core.db.sig', + '/x86_64/extra/extra.db', + ]; + w.manifest('x86_64/core:2026.10.14', [['core.db', D.db]]); + const first = []; + for (const p of paths) { + const res = await w.get(p); + first.push([res.status, res.headers.get('location')]); + } + for (const [i, p] of paths.entries()) { + const matches = w.cache.matches; + const requests = w.requests.length; + const res = await w.get(p); + assert.deepEqual([res.status, res.headers.get('location')], first[i], p); + assert.equal(w.cache.matches - matches, 1, `${p}: one cache lookup`); + assert.equal(w.requests.length - requests, 0, `${p}: no subrequest`); + } + assert.deepEqual(w.errors, []); + })); + +test('a redirect is stored until se minus 120 s, at most 600 s, and not with less than 30 s left', () => + inWorld(async (w) => { + const cases = [ + [400, 280], + [1000, 600], + [720, 600], + [150, 30], + [149, null], + [60, null], + ]; + for (const [i, [seAfter, want]] of cases.entries()) { + w.seAfter = seAfter; + const name = `p${i}-1-1-any.pkg.tar.zst`; + const d = digest('567804'[i]); + w.manifest(`x86_64/core:${name}`, [ + [name, d], + [`${name}.sig`, digest('9')], + ]); + w.blobs.set(`x86_64/core@${d}`, true); + w.blobs.set(`x86_64/core@${digest('9')}`, true); + const res = await w.get(`/x86_64/core/${name}`); + assert.equal(res.status, 302); + const loc = w.cache.ofKind('loc').filter((p) => p.url.endsWith(`/${encodeURIComponent(name)}`)); + const blob = w.cache.ofKind('blob').filter((p) => p.url.endsWith(encodeURIComponent(d))); + if (want === null) { + assert.equal(loc.length, 0, `se after ${seAfter} s: no loc entry`); + assert.equal(blob.length, 0, `se after ${seAfter} s: no blob entry`); + } else { + assert.equal(loc[0].ttl, want, `se after ${seAfter} s: loc TTL`); + assert.equal(blob[0].ttl, want, `se after ${seAfter} s: blob TTL`); + } + } + })); + +test('a signed URL with no se is not stored', () => + inWorld(async (w) => { + w.blobs.set(`x86_64/core@${D.pkg}`, 'https://pkg-containers.githubusercontent.com/ghcrblobs10/blobs/x?sig=1'); + const res = await w.get(`/x86_64/core/${PKG}`); + assert.equal(res.status, 302); + assert.equal(w.cache.ofKind('loc').filter((p) => p.url.endsWith(encodeURIComponent(PKG))).length, 0); + assert.equal(w.cache.ofKind('blob').filter((p) => p.url.endsWith(encodeURIComponent(D.pkg))).length, 0); + })); + +test('manifest TTLs: pins and dated, db- and files- tags 1 year; packages 6 h or 60 s; latest 30 s; 404 60 s', () => + inWorld(async (w) => { + const layers = [ + ['core.db', D.db], + ['core.files', D.files], + ]; + const body = manifestJson(layers); + const pin = await sha256hex(body); + w.manifests.set(`x86_64/core:sha256:${pin}`, body); + for (const ref of ['2026.10.14', `db-${hex('a')}`, `files-${hex('b')}`]) w.manifest(`x86_64/core:${ref}`, layers); + const cases = [ + ['/x86_64/core/core.db', 'man/x86_64/core/latest', TTL.latest, 30], + [`/x86_64/core@sha256:${pin}/core.db`, `man/x86_64/core/sha256:${pin}`, TTL.immutable, 31536000], + ['/x86_64/core:2026.10.14/core.db', 'man/x86_64/core/2026.10.14', TTL.immutable, 31536000], + [`/x86_64/core:db-${hex('a')}/core.db`, `man/x86_64/core/db-${hex('a')}`, TTL.immutable, 31536000], + [`/x86_64/core:files-${hex('b')}/core.files`, `man/x86_64/core/files-${hex('b')}`, TTL.immutable, 31536000], + [`/x86_64/core/${PKG}.sig`, `man/x86_64/core/${PKG_TAG}`, TTL.packageWithSig, 21600], + [`/x86_64/core/${NOSIG}`, `man/x86_64/core/${NOSIG}`, TTL.packageNoSig, 60], + ['/x86_64/extra/extra.db', 'man/x86_64/extra/latest', TTL.missing, 60], + ]; + for (const [p, key, constant, seconds] of cases) { + await (await w.get(p)).text(); + assert.equal(constant, seconds, `the constant for ${key}`); + assert.equal(ttlOf(w, 'man', key.slice(4)), seconds, `${p}: manifest TTL`); + } + })); + +test('a package manifest with a second layer of the file name is not kept 6 h', () => + inWorld(async (w) => { + const name = 'dup-1-1-any.pkg.tar.zst'; + w.manifest(`x86_64/core:${name}`, [ + [name, D.pkg], + [`${name}.sig`, D.pkgSig], + [`${name}.sig`, D.plusSig], + ]); + const res = await w.get(`/x86_64/core/${name}`); + assert.equal(res.status, 302); + assert.equal(ttlOf(w, 'man', name), 60); + })); + +test('a latest redirect never outlives the latest manifest entry, so a new latest shows within 30 s', () => + inWorld(async (w) => { + const first = await w.get('/x86_64/core/core.db'); + assert.equal(new URL(first.headers.get('location')).pathname, `/ghcrblobs10/blobs/${D.db}`); + assert.ok(ttlOf(w, 'loc', 'latest/core.db') <= 30, 'the loc entry of latest is at most 30 s'); + const next = digest('8'); + w.manifest('x86_64/core:latest', [ + ['core.db', next], + ['core.files', D.files], + ]); + w.blobs.set(`x86_64/core@${next}`, true); + w.advance(20); + const cached = await w.get('/x86_64/core/core.db'); + assert.equal(new URL(cached.headers.get('location')).pathname, `/ghcrblobs10/blobs/${D.db}`); + w.advance(11); + const moved = await w.get('/x86_64/core/core.db'); + assert.equal(new URL(moved.headers.get('location')).pathname, `/ghcrblobs10/blobs/${next}`); + })); + +test('a latest redirect is a one-lookup hit while its manifest entry lasts, also when GHCR is slow', () => + inWorld(async (w) => { + // Time moves on during each subrequest, as it does in the runtime. + w.intercept = () => { + w.advance(1); + }; + const first = await w.get('/x86_64/core/core.db'); + assert.equal(first.status, 302); + // The manifest arrives at T0+1 s and is kept 30 s; the blob answer comes at T0+2 s. + assert.equal(ttlOf(w, 'loc', 'latest/core.db'), 29, 'the loc entry ends with the latest manifest entry'); + const matches = w.cache.matches; + const requests = w.requests.length; + w.advance(27); + const again = await w.get('/x86_64/core/core.db'); + assert.equal(again.headers.get('location'), first.headers.get('location')); + assert.equal(w.cache.matches - matches, 1); + assert.equal(w.requests.length, requests); + })); + +test('a blob entry serves every reference that names the same digest', () => + inWorld(async (w) => { + w.manifest('x86_64/core:2026.10.14', [['core.db', D.db]]); + await w.get('/x86_64/core/core.db'); + const before = w.requests.length; + const res = await w.get('/x86_64/core:2026.10.14/core.db'); + assert.equal(res.status, 302); + assert.deepEqual( + w.since(before).map((r) => r.url.replace(/^.*\/v2\/pkgforge-dev\/mirror-archlinux\//, '')), + ['x86_64/core/manifests/2026.10.14'], + ); + })); + +test('a package answer fills the .sig entry: the .sig is then one lookup and no subrequest', () => + inWorld(async (w) => { + const res = await w.get(`/x86_64/core/${PKG}`); + assert.equal(res.status, 302); + assert.equal(w.requests.length, 3, 'the manifest, the package blob and the .sig blob'); + const matches = w.cache.matches; + const sig = await w.get(`/x86_64/core/${PKG}.sig`); + assert.equal(sig.status, 302); + assert.equal(new URL(sig.headers.get('location')).pathname, `/ghcrblobs10/blobs/${D.pkgSig}`); + assert.equal(w.cache.matches - matches, 1); + assert.equal(w.requests.length, 3); + assert.deepEqual(w.errors, []); + })); + +test('a package request asks for the .sig blob at the same time as the package blob', () => + inWorld(async (w) => { + let sigAsked; + const sigSeen = new Promise((resolve) => { + sigAsked = resolve; + }); + let parallel = null; + w.intercept = async (url) => { + if (url.endsWith(`/blobs/${D.pkgSig}`)) sigAsked(); + if (url.endsWith(`/blobs/${D.pkg}`)) { + // The package blob answers only after the .sig blob is asked for, or + // after 500 ms. + parallel = await Promise.race([ + sigSeen.then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 500)), + ]); + } + return undefined; + }; + const res = await w.get(`/x86_64/core/${PKG}`); + assert.equal(res.status, 302); + assert.equal(parallel, true, 'the .sig blob request starts before the package blob answers'); + assert.deepEqual(w.errors, []); + })); + +test('a .sig request does not fill anything more, and a database request fills no .sig', () => + inWorld(async (w) => { + const locFiles = () => w.cache.ofKind('loc').map((p) => decodeURIComponent(p.url.slice(p.url.lastIndexOf('/') + 1))); + await w.get(`/x86_64/core/${PKG}.sig`); + assert.equal(w.requests.length, 2, 'the manifest and the .sig blob'); + assert.deepEqual(locFiles(), [`${PKG}.sig`]); + await w.get('/x86_64/core/core.db'); + assert.equal(w.requests.length, 4, 'the latest manifest and the core.db blob'); + assert.deepEqual(locFiles(), [`${PKG}.sig`, 'core.db']); + })); + +test('a package with no .sig layer: its .sig gets 404 from the cache, kept at most 60 s', () => + inWorld(async (w) => { + const res = await w.get(`/x86_64/core/${NOSIG}`); + assert.equal(res.status, 302); + const requests = w.requests.length; + const matches = w.cache.matches; + const sig = await w.get(`/x86_64/core/${NOSIG}.sig`); + assert.equal(sig.status, 404); + assert.match(await sig.text(), /has 0 layers named/); + assert.equal(w.requests.length, requests); + assert.equal(w.cache.matches - matches, 1); + assert.ok(ttlOf(w, 'loc', `${NOSIG}.sig`) <= 60); + assert.deepEqual(w.errors, []); + })); + +test('a manifest 404 is kept 60 s, then asked for again', () => + inWorld(async (w) => { + const a = await w.get('/x86_64/extra/extra.db'); + assert.equal(a.status, 404); + assert.match(await a.text(), /GHCR has no x86_64\/extra:latest/); + assert.equal(w.requests.length, 1); + w.advance(59); + await (await w.get('/x86_64/extra/extra.files')).text(); + assert.equal(w.requests.length, 1, 'the 404 is still kept after 59 s'); + w.advance(2); + await (await w.get('/x86_64/extra/extra.db')).text(); + assert.equal(w.requests.length, 2, 'the 404 is asked for again after 61 s'); + })); + +test('a package manifest entry expires: 6 h with its .sig, 60 s without', () => + inWorld(async (w) => { + await w.get(`/x86_64/core/${PKG}`); + await w.get(`/x86_64/core/${NOSIG}`); + w.advance(601); + const before = w.requests.length; + await w.get(`/x86_64/core/${PKG}`); + await w.get(`/x86_64/core/${NOSIG}`); + const urls = w.since(before).map((r) => r.url); + assert.ok(!urls.some((u) => u.includes(`/manifests/${PKG_TAG}`)), 'the 6 h manifest is still kept'); + assert.ok(urls.some((u) => u.includes(`/manifests/${NOSIG}`)), 'the 60 s manifest is asked for again'); + })); + +test('a cache that fails is logged, and the request still gets its answer from GHCR', () => + inWorld(async (w) => { + w.cache.failMatch = true; + const a = await w.get('/x86_64/core/core.db'); + assert.equal(a.status, 302); + assert.ok(w.errors.some(([e]) => /cache match/.test(e.error)), 'the failed match is logged'); + w.cache.failMatch = false; + w.cache.failPut = true; + w.errors.length = 0; + const b = await w.get(`/x86_64/core/${PKG}`); + assert.equal(b.status, 302); + assert.ok(w.errors.some(([e]) => /cache put/.test(e.error)), 'the failed put is logged'); + })); + +test('an entry past its expires time is not used, whatever the cache returns', () => + inWorld(async (w) => { + await w.get('/x86_64/core/core.db'); + // A cache that keeps entries longer than max-age, as eviction allows. + for (const e of w.cache.entries.values()) e.until = Infinity; + w.advance(3600); + const before = w.requests.length; + const res = await w.get('/x86_64/core/core.db'); + assert.equal(res.status, 302); + assert.equal(w.since(before).length, 2, 'the manifest and the blob are asked for again'); + })); diff --git a/worker/test/config.test.js b/worker/test/config.test.js new file mode 100644 index 0000000..a645099 --- /dev/null +++ b/worker/test/config.test.js @@ -0,0 +1,64 @@ +// worker/wrangler.toml and .github/workflows/worker.yml keep the settings +// the Worker depends on. A deploy reads them with no review of its own. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +const read = (rel) => readFileSync(fileURLToPath(new URL(`../../${rel}`, import.meta.url)), 'utf8'); + +// The flat keys of a TOML file by section ('' is the top level). Enough for +// this file: one key per line, no multi-line values except the routes array. +function toml(text) { + const out = { '': {} }; + let section = ''; + for (const raw of text.split('\n')) { + const line = raw.replace(/\s+#.*$/, '').trim(); + if (line === '' || line.startsWith('#')) continue; + const s = /^\[([a-z_.]+)\]$/.exec(line); + if (s) { + section = s[1]; + out[section] = out[section] || {}; + continue; + } + const kv = /^([a-z_]+)\s*=\s*(.+)$/.exec(line); + if (kv) out[section][kv[1]] = kv[2]; + } + return out; +} + +test('wrangler.toml names the Worker, its account, its domain and no workers.dev route', () => { + const text = read('worker/wrangler.toml'); + const t = toml(text); + assert.equal(t[''].name, '"ghcr-mirror-archlinux-pkgforge"'); + assert.equal(t[''].main, '"src/index.js"'); + assert.equal(t[''].account_id, '"2a97c8f21aff7a773bf976c61ace1381"'); + assert.equal(t[''].workers_dev, 'false'); + assert.equal(t[''].preview_urls, 'false'); + assert.match(text, /\{\s*pattern\s*=\s*"ghcr\.mirror\.archlinux\.pkgforge\.dev",\s*custom_domain\s*=\s*true\s*\}/); +}); + +test('wrangler.toml sets the compatibility date, the flags, the CPU limit and the logs', () => { + const t = toml(read('worker/wrangler.toml')); + assert.equal(t[''].compatibility_date, '"2026-10-13"'); + assert.equal(t[''].compatibility_flags, '["no_nodejs_compat", "no_nodejs_compat_v2"]'); + assert.equal(t.limits.cpu_ms, '50'); + assert.equal(t.observability.enabled, 'true'); + assert.equal(t.observability.head_sampling_rate, '1'); + assert.equal(t['observability.logs'].invocation_logs, 'false'); +}); + +test('worker.yml tests in a pinned node image and deploys with a pinned wrangler-action', () => { + const wf = read('.github/workflows/worker.yml'); + const image = /# tag: (\d+)-alpine\n\s*image=docker\.io\/library\/node:(\d+)-alpine@sha256:[0-9a-f]{64}\n/.exec(wf); + assert.ok(image, 'the node image is pinned by digest under a # tag: marker'); + assert.equal(image[1], image[2], 'the marker names the tag of the image line'); + assert.ok(Number(image[2]) >= 22, `node ${image[2]} is 22 or newer`); + assert.match(wf, /node --test 'worker\/test\/\*\.test\.js'/); + assert.match(wf, /uses: cloudflare\/wrangler-action@953926a2e2182532811c01a25e53647d93bf07c0 # v4\.1\.3\n/); + assert.match(wf, /wranglerVersion: "4\.150\.0"/); + assert.match(wf, /workingDirectory: worker\n/); + assert.match(wf, /apiToken: \$\{\{ secrets\.CLOUDFLARE_API_TOKEN \}\}/); + assert.match(wf, /if: github\.ref == 'refs\/heads\/main'/); + assert.doesNotMatch(wf, /if:.*secrets\./, 'the secret is checked through env, not in if:'); +}); diff --git a/worker/test/ghcr.test.js b/worker/test/ghcr.test.js new file mode 100644 index 0000000..1222cc9 --- /dev/null +++ b/worker/test/ghcr.test.js @@ -0,0 +1,310 @@ +// GHCR subrequests: deadlines, failures, the never-stream rule, the Location +// check, pins, the manifest size cap, and bodies that are not read. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { inWorld, REGISTRY, MANIFEST_TYPE, PKG, D, digest, hex, manifestJson, sha256hex } from './world.js'; + +function assert502(res, w) { + assert.equal(res.status, 502); + assert.equal(res.headers.get('retry-after'), '30'); + assert.ok(w.errors.length > 0, 'a 502 is logged with console.error'); + for (const [e] of w.errors) assert.equal(typeof e, 'object'); +} + +const isManifest = (url) => url.includes('/manifests/'); +const isBlob = (url) => url.includes('/blobs/'); + +test('every subrequest has a 10 s deadline, the anonymous token and no redirect followed', () => + inWorld(async (w) => { + const res = await w.get('/x86_64/core/core.db'); + assert.equal(res.status, 302); + assert.equal(w.requests.length, 2); + for (const { url, init } of w.requests) { + assert.ok(url.startsWith(`${REGISTRY}/x86_64/core/`), url); + assert.ok(init.signal instanceof AbortSignal, `${url} has a signal`); + assert.equal(init.redirect, 'manual'); + assert.equal(init.headers.authorization, 'Bearer QQ=='); + } + assert.equal(w.requests[0].init.headers.accept, MANIFEST_TYPE); + assert.deepEqual(w.timeouts, [10000, 10000]); + assert.deepEqual(w.errors, []); + })); + +test('a network error on the manifest request gets 502 with retry-after 30', () => + inWorld(async (w) => { + w.intercept = (url) => { + if (isManifest(url)) throw new TypeError('fetch failed'); + }; + const res = await w.get('/x86_64/core/core.db'); + assert502(res, w); + assert.match(await res.text(), /fetch failed/); + assert.equal(w.cache.ofKind('man').length, 0); + })); + +test('a network error on the blob request gets 502 with retry-after 30', () => + inWorld(async (w) => { + w.intercept = (url) => { + if (isBlob(url)) throw new TypeError('fetch failed'); + }; + const res = await w.get(`/x86_64/core/${PKG}`); + assert502(res, w); + assert.equal(w.cache.ofKind('loc').length, 0); + })); + +test('a GHCR request that runs past its deadline gets 502', () => + inWorld(async (w) => { + // The runtime's own timeout timer does not keep node's event loop alive, + // so a referenced timer stands in for it, 5 ms instead of 10 s. + AbortSignal.timeout = (ms) => { + w.timeouts.push(ms); + const c = new AbortController(); + setTimeout(() => c.abort(new DOMException('The operation timed out.', 'TimeoutError')), 5); + return c.signal; + }; + w.intercept = (url, init) => + new Promise((resolve, reject) => { + if (init.signal.aborted) reject(init.signal.reason); + init.signal.addEventListener('abort', () => reject(init.signal.reason)); + }); + const t = performance.now(); + const res = await w.get('/x86_64/core/core.db'); + assert502(res, w); + assert.ok(performance.now() - t < 5000, 'the stalled request ends at its deadline'); + assert.deepEqual(w.timeouts, [10000]); + })); + +test('a GHCR answer other than 200 or 404 for a manifest gets 502, and its body is cancelled', () => + inWorld(async (w) => { + for (const status of [500, 502, 503, 429, 401, 403, 303, 307, 204]) { + w.errors.length = 0; + w.intercept = (url) => { + if (isManifest(url)) { + const body = status === 204 ? '' : `{"errors":[{"code":"E${status}"}]}`; + return w.response(status, body, status === 303 || status === 307 ? { location: 'https://github.com/' } : {}); + } + }; + const res = await w.get('/x86_64/core/core.db'); + assert502(res, w); + assert.match(await res.text(), new RegExp(`GHCR answered ${status}`)); + } + assert.equal(w.cache.ofKind('man').length, 0); + assert.deepEqual(w.unusedBodies(), []); + })); + +test('a blob that GHCR answers with 200 gets 502: the Worker never sends the bytes', () => + inWorld(async (w) => { + const bytes = new Uint8Array(1 << 20); + w.intercept = (url) => { + if (isBlob(url)) return w.response(200, bytes, { 'content-length': String(bytes.byteLength) }, { url }); + }; + const res = await w.get(`/x86_64/core/${PKG}`); + assert502(res, w); + const blobBody = w.bodies.find((b) => isBlob(b.url)); + assert.equal(blobBody.read, false, 'the blob body is not read'); + assert.equal(blobBody.cancelled, true, 'the blob body is cancelled'); + assert.equal(w.cache.ofKind('loc').length, 0); + })); + +test('the Location must be https and its host must end in .githubusercontent.com', () => + inWorld(async (w) => { + const se = encodeURIComponent(new Date(w.clock.now + 400000).toISOString()); + const bad = [ + `http://pkg-containers.githubusercontent.com/b?se=${se}`, + `https://evil.example/b?se=${se}`, + `https://pkg-containers.githubusercontent.com.evil.example/b?se=${se}`, + `https://githubusercontent.com/b?se=${se}`, + `https://xgithubusercontent.com/b?se=${se}`, + `ftp://pkg-containers.githubusercontent.com/b?se=${se}`, + '/relative/b', + 'not a url', + null, + ]; + for (const location of bad) { + w.errors.length = 0; + w.intercept = (url) => { + if (isBlob(url)) return w.response(307, '', location === null ? {} : { location }, { url }); + }; + const res = await w.get(`/x86_64/core/${PKG}`); + assert502(res, w); + } + assert.equal(w.cache.ofKind('blob').length, 0); + assert.equal(w.cache.ofKind('loc').length, 0); + w.intercept = (url) => { + if (isBlob(url)) return w.response(302, '', { location: `https://objects.githubusercontent.com/b?se=${se}` }, { url }); + }; + const ok = await w.get(`/x86_64/core/${PKG}`); + assert.equal(ok.status, 302); + assert.equal(ok.headers.get('location'), `https://objects.githubusercontent.com/b?se=${se}`); + })); + +test('a blob answer that is not a redirect gets 502, even with a Location', () => + inWorld(async (w) => { + for (const status of [201, 204, 401, 403, 429, 500, 503]) { + w.errors.length = 0; + w.intercept = (url) => { + if (isBlob(url)) return w.response(status, status === 204 ? '' : 'x', { location: w.signedUrl(D.pkg) }, { url }); + }; + const res = await w.get(`/x86_64/core/${PKG}`); + assert502(res, w); + assert.match(await res.text(), new RegExp(`GHCR answered ${status} for the blob`)); + } + })); + +test('a blob GHCR does not hold gets 404, with no log', () => + inWorld(async (w) => { + w.blobs.delete(`x86_64/core@${D.pkg}`); + const res = await w.get(`/x86_64/core/${PKG}`); + assert.equal(res.status, 404); + assert.match(await res.text(), /GHCR has no blob/); + assert.deepEqual(w.errors, []); + })); + +test('an @sha256 pin is served when the manifest hashes to it', () => + inWorld(async (w) => { + const body = manifestJson([ + ['core.db', D.db], + ['core.files', D.files], + ]); + const pin = await sha256hex(body); + w.manifests.set(`x86_64/core:sha256:${pin}`, body); + const res = await w.get(`/x86_64/core@sha256:${pin}/core.db`); + assert.equal(res.status, 302); + assert.equal(new URL(res.headers.get('location')).pathname, `/ghcrblobs10/blobs/${D.db}`); + assert.deepEqual(w.errors, []); + })); + +test('an @sha256 pin whose manifest hashes to another value gets 502 and is not stored', () => + inWorld(async (w) => { + const body = manifestJson([['core.db', D.db]]); + const pin = hex('7'); + w.manifests.set(`x86_64/core:sha256:${pin}`, body); + const res = await w.get(`/x86_64/core@sha256:${pin}/core.db`); + assert502(res, w); + assert.match(await res.text(), new RegExp(`hashes to sha256:${await sha256hex(body)}`)); + assert.equal(w.cache.ofKind('man').length, 0); + assert.equal(w.cache.ofKind('loc').length, 0); + const again = await w.get(`/x86_64/core@sha256:${pin}/core.db`); + assert.equal(again.status, 502); + assert.equal(w.requests.filter((r) => isManifest(r.url)).length, 2, 'a refused pin is asked for again'); + })); + +test('a manifest over 64 KiB by content-length gets 502, and its body is not read', () => + inWorld(async (w) => { + w.intercept = (url) => { + if (isManifest(url)) return w.response(200, manifestJson([['core.db', D.db]]), { 'content-length': '65537' }, { url }); + }; + const res = await w.get('/x86_64/core/core.db'); + assert502(res, w); + assert.match(await res.text(), /over 65536 bytes/); + const b = w.bodies.find((x) => isManifest(x.url)); + assert.equal(b.read, false); + assert.equal(b.cancelled, true); + })); + +test('a manifest over 64 KiB with no or a false content-length is cut at 64 KiB', () => + inWorld(async (w) => { + for (const headers of [{}, { 'content-length': '100' }]) { + w.errors.length = 0; + const big = new Uint8Array(200000).fill(0x20); + w.intercept = (url) => { + if (isManifest(url)) return w.response(200, big, headers, { url, chunk: 16384 }); + }; + const res = await w.get('/x86_64/core/core.db'); + assert502(res, w); + assert.match(await res.text(), /over 65536 bytes/); + } + for (const b of w.bodies) assert.equal(b.cancelled, true, 'the reader is cancelled at the cap'); + })); + +test('a manifest of exactly 64 KiB is accepted', () => + inWorld(async (w) => { + const json = manifestJson([['core.db', D.db]]); + const padded = json + ' '.repeat(65536 - json.length); + assert.equal(new TextEncoder().encode(padded).byteLength, 65536); + w.intercept = (url) => { + if (isManifest(url)) return w.response(200, padded, {}, { url, chunk: 4096 }); + }; + const res = await w.get('/x86_64/core/core.db'); + assert.equal(res.status, 302); + assert.deepEqual(w.errors, []); + })); + +test('a manifest that is not JSON, or has no layers array, gets 502 and is not stored', () => + inWorld(async (w) => { + for (const body of ['', 'not json', '{"layers": 3}', '[]', 'null', '\xff\xfe']) { + w.errors.length = 0; + w.intercept = (url) => { + if (isManifest(url)) { + const bytes = body === '\xff\xfe' ? new Uint8Array([0xff, 0xfe, 0x7b, 0x7d]) : body; + return w.response(200, bytes, { 'content-type': MANIFEST_TYPE }, { url }); + } + }; + const res = await w.get('/x86_64/core/core.db'); + assert502(res, w); + } + assert.equal(w.cache.ofKind('man').length, 0); + })); + +test('a file with two layers of its name gets 404; a layer with no sha256 digest gets 502', () => + inWorld(async (w) => { + w.manifest('x86_64/core:latest', [ + ['core.db', D.db], + ['core.db', D.files], + ['core.files', 'md5:abc'], + ]); + const two = await w.get('/x86_64/core/core.db'); + assert.equal(two.status, 404); + assert.match(await two.text(), /has 2 layers named core\.db/); + assert.deepEqual(w.errors, []); + const bad = await w.get('/x86_64/core/core.files'); + assert502(bad, w); + assert.match(await bad.text(), /no sha256 digest/); + w.manifest('x86_64/core:2026.10.14', [['core.db', digest('a').toUpperCase()]]); + const upper = await w.get('/x86_64/core:2026.10.14/core.db'); + assert.equal(upper.status, 502); + })); + +test('console.error is called on 5xx answers only', () => + inWorld(async (w) => { + const ok = [ + '/', + '/x86_64/core/core.db', + '/x86_64/core/core.files', + '/x86_64/core/core.db.sig', + `/x86_64/core/${PKG}`, + `/x86_64/core/${PKG}.sig`, + '/x86_64/core/nosig-1-1-any.pkg.tar.zst.sig', + '/x86_64/extra/extra.db', + '/x86_64/core:2026.10.13/core.db', + '/x86_64/core:staging/core.db', + '/foo', + ]; + for (const p of ok) { + const res = await w.get(p); + assert.ok(res.status < 500, `${p}: ${res.status}`); + await res.text(); + } + assert.deepEqual(w.errors, []); + w.intercept = () => { + throw new TypeError('fetch failed'); + }; + const res = await w.get('/x86_64/core/other-1-1-any.pkg.tar.zst'); + assert.equal(res.status, 502); + assert.equal(w.errors.length, 1); + assert.equal(w.errors[0][0].status, 502); + assert.equal(w.errors[0][0].path, '/x86_64/core/other-1-1-any.pkg.tar.zst'); + })); + +test('an unexpected error inside the Worker gets 502 with retry-after, not an error page', () => + inWorld(async (w) => { + w.intercept = () => ({ + get status() { + throw new Error('broken status'); + }, + headers: new Headers(), + body: null, + }); + const res = await w.get('/x86_64/core/core.db'); + assert502(res, w); + assert.match(await res.text(), /^internal error: Error: broken status/); + })); diff --git a/worker/test/paths.test.js b/worker/test/paths.test.js new file mode 100644 index 0000000..acc1d45 --- /dev/null +++ b/worker/test/paths.test.js @@ -0,0 +1,209 @@ +// Paths, references, file names, methods and HEAD. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { inWorld, HOST, REGISTRY, PKG, PKG_TAG, PLUS, PLUS_TAG, D, hex, sha256hex } from './world.js'; + +const manifestUrls = (w) => w.requests.map((r) => r.url).filter((u) => u.includes('/manifests/')); + +test('GET / shows the usage text with the Server line and makes no subrequest', () => + inWorld(async (w) => { + const res = await w.get('/'); + assert.equal(res.status, 200); + const body = await res.text(); + assert.match(body, new RegExp(`^Server = https://${HOST.replace(/\./g, '\\.')}/\\$arch/\\$repo$`, 'm')); + assert.match(body, /\/ARCH\/REPO@sha256:HEX\/FILE/); + assert.equal(res.headers.get('cache-control'), 'max-age=3600'); + assert.equal(w.requests.length, 0); + assert.equal(w.cache.matches, 0); + })); + +test('an unknown path gets 404 with no subrequest and no cache lookup', () => + inWorld(async (w) => { + const paths = [ + '/foo', + '/favicon.ico', + '/x86_64', + '/x86_64/core', + '/x86_64/core/', + '/x86_64/core/README', + '/x86_64/core/extra.db', + '/x86_64/core/core.db.tar.gz', + '/x86_64/core/core.dbx', + '/x86_64/core/sub/core.db', + '//x86_64/core/core.db', + '/X86_64/core/core.db', + '/x86_64/Core/core.db', + '/_x86_64/core/core.db', + '/__cache/v1/loc/x86_64/core/latest/core.db', + `/x86_64/core@sha256:${'A'.repeat(64)}/core.db`, + '/x86_64/core@sha256:abc/core.db', + `/x86_64/core@sha512:${hex('a')}/core.db`, + `/x86_64/core/${'a'.repeat(250)}.pkg.tar.zst`, + ]; + for (const p of paths) { + const res = await w.get(p); + assert.equal(res.status, 404, p); + await res.text(); + } + assert.deepEqual(w.requests, []); + assert.equal(w.cache.matches, 0); + assert.deepEqual(w.errors, []); + })); + +test('latest, YYYY.MM.DD, db-HEX, files-HEX and @sha256 pins are served', () => + inWorld(async (w) => { + const layers = [ + ['core.db', D.db], + ['core.files', D.files], + ]; + const pinBody = w.manifest('x86_64/core:pin-placeholder', layers, { annotations: { a: 'pin' } }); + const pin = await sha256hex(pinBody); + w.manifests.set(`x86_64/core:sha256:${pin}`, pinBody); + for (const ref of ['2026.10.14', `db-${hex('a')}`, `files-${hex('b')}`]) w.manifest(`x86_64/core:${ref}`, layers); + const cases = [ + ['/x86_64/core/core.db', 'latest'], + ['/x86_64/core:latest/core.files', 'latest'], + ['/x86_64/core:2026.10.14/core.db', '2026.10.14'], + [`/x86_64/core:db-${hex('a')}/core.db`, `db-${hex('a')}`], + [`/x86_64/core:files-${hex('b')}/core.files`, `files-${hex('b')}`], + [`/x86_64/core@sha256:${pin}/core.db`, `sha256:${pin}`], + ]; + for (const [p, ref] of cases) { + const before = w.requests.length; + const res = await w.get(p); + assert.equal(res.status, 302, p); + const urls = w.since(before).map((r) => r.url); + if (urls.some((u) => u.includes('/manifests/'))) { + assert.ok(urls.includes(`${REGISTRY}/x86_64/core/manifests/${ref}`), `${p} asks GHCR for ${ref}: ${urls}`); + } + } + assert.deepEqual(w.errors, []); + })); + +test('any other tag gets 404 with no subrequest, staging included', () => + inWorld(async (w) => { + const tags = [ + 'staging', + 'staging-wip', + 'LATEST', + 'Latest', + '2026.10', + '2026-10-14', + '20261014', + 'db-abc', + `db-${'A'.repeat(64)}`, + `files-${hex('b')}0`, + `sha256-${hex('a')}`, + PKG_TAG, + 'v1', + ]; + for (const t of tags) { + for (const f of ['core.db', PKG]) { + const res = await w.get(`/x86_64/core:${t}/${f}`); + assert.equal(res.status, 404, `${t} ${f}`); + assert.match(await res.text(), /is not served/); + } + } + assert.deepEqual(w.requests, []); + assert.equal(w.cache.matches, 0); + })); + +test('a package and its .sig come from the package tag, whatever reference the URL names', () => + inWorld(async (w) => { + for (const p of [ + `/x86_64/core/${PKG}`, + `/x86_64/core:2026.10.14/${PKG}`, + `/x86_64/core@sha256:${hex('9')}/${PKG}.sig`, + ]) { + const res = await w.get(p); + assert.equal(res.status, 302, p); + } + // One manifest request for all three, by the package's own tag. The pin + // in the third URL is not checked: it names a database, not a package. + assert.deepEqual(manifestUrls(w), [`${REGISTRY}/x86_64/core/manifests/${PKG_TAG}`]); + assert.deepEqual(w.errors, []); + })); + +test('a percent-encoded file name is decoded before its tag is made', () => + inWorld(async (w) => { + const plain = await w.get(`/x86_64/core/${PLUS}`); + assert.equal(plain.status, 302); + const encoded = await w.get(`/x86_64/core/${PLUS.replace(/\+/g, '%2B')}`); + assert.equal(encoded.status, 302); + assert.equal(encoded.headers.get('location'), plain.headers.get('location')); + const lower = await w.get(`/x86_64/core/${PLUS.replace(/\+/g, '%2b')}.sig`); + assert.equal(lower.status, 302); + const colon = await w.get(`/x86_64/core/${PKG.replace(':', '%3A')}`); + assert.equal(colon.status, 302); + const dot = await w.get('/x86_64/core/core%2Edb'); + assert.equal(dot.status, 302); + assert.deepEqual(manifestUrls(w), [ + `${REGISTRY}/x86_64/core/manifests/${PLUS_TAG}`, + `${REGISTRY}/x86_64/core/manifests/${PKG_TAG}`, + `${REGISTRY}/x86_64/core/manifests/latest`, + ]); + assert.deepEqual(w.errors, []); + })); + +test('a decoded file name outside the character set gets 404 with no subrequest', () => + inWorld(async (w) => { + const names = [ + 'a%2Fb-1-1-any.pkg.tar.zst', + 'a%20b-1-1-any.pkg.tar.zst', + 'a%0Ab-1-1-any.pkg.tar.zst', + 'a%00b-1-1-any.pkg.tar.zst', + 'caf%C3%A9-1-1-any.pkg.tar.zst', + 'a%7Fb-1-1-any.pkg.tar.zst', + 'bad%E0%A4%A-1-1-any.pkg.tar.zst', + 'bad%ZZ-1-1-any.pkg.tar.zst', + '%2E', + `${'a'.repeat(240)}%2B%2B%2B%2B%2B%2B.pkg.tar.zst`, + ]; + for (const n of names) { + const res = await w.get(`/x86_64/core/${n}`); + assert.equal(res.status, 404, n); + await res.text(); + } + assert.deepEqual(w.requests, []); + assert.deepEqual(w.errors, []); + })); + +test('a method other than GET and HEAD gets 405', () => + inWorld(async (w) => { + for (const m of ['POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS']) { + const res = await w.request('/x86_64/core/core.db', m); + assert.equal(res.status, 405, m); + assert.equal(res.headers.get('allow'), 'GET, HEAD'); + await res.text(); + } + assert.deepEqual(w.requests, []); + })); + +test('HEAD gets the same answer as GET, with no body', () => + inWorld(async (w) => { + const head = await w.head('/x86_64/core/core.db'); + assert.equal(head.status, 302); + assert.equal(head.body, null); + const get = await w.get('/x86_64/core/core.db'); + assert.equal(get.status, 302); + assert.equal(head.headers.get('location'), get.headers.get('location')); + assert.match(head.headers.get('location'), /^https:\/\/pkg-containers\.githubusercontent\.com\//); + const root = await w.head('/'); + assert.equal(root.status, 200); + assert.equal(root.body, null); + const missing = await w.head('/x86_64/core/core.db.sig'); + assert.equal(missing.status, 404); + assert.equal(missing.body, null); + const unknown = await w.head('/foo'); + assert.equal(unknown.status, 404); + assert.equal(unknown.body, null); + })); + +test('a 302 has no body and cache-control no-store', () => + inWorld(async (w) => { + const res = await w.get(`/x86_64/core/${PKG}`); + assert.equal(res.status, 302); + assert.equal(res.headers.get('cache-control'), 'no-store'); + assert.equal(res.body, null); + assert.equal(new URL(res.headers.get('location')).pathname, `/ghcrblobs10/blobs/${D.pkg}`); + })); diff --git a/worker/test/tags.test.js b/worker/test/tags.test.js new file mode 100644 index 0000000..a8fd504 --- /dev/null +++ b/worker/test/tags.test.js @@ -0,0 +1,84 @@ +// tagOf and `client/pacman-oci --tag` give the same tag for every name. +// +// A difference sends the Worker to another manifest than the wrapper, so one +// of the two serves a 404 for a file the mirror holds. The client runs under +// sh (busybox in the node alpine image, bash or dash elsewhere). +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { createHash } from 'node:crypto'; +import { tagOf } from '../src/index.js'; + +const ROOT = fileURLToPath(new URL('../../', import.meta.url)); +const CLIENT = fileURLToPath(new URL('../../client/pacman-oci', import.meta.url)); +const FIXTURE = fileURLToPath(new URL('../../tests/fixtures/filenames.txt', import.meta.url)); + +function client(args, input) { + const r = spawnSync('sh', [CLIENT, ...args], { cwd: ROOT, input, encoding: 'utf8', timeout: 120000 }); + if (r.error) throw new Error(`sh ${CLIENT} ${args.join(' ')} did not start: ${r.error.message}`); + assert.equal(r.status, 0, `sh client/pacman-oci ${args.join(' ')} exited ${r.status}: ${r.stderr}`); + return r.stdout; +} + +const hashed = (name) => `fn-${createHash('sha256').update(name, 'utf8').digest('hex')}`; + +const at128 = `${'a'.repeat(108)}-1-1-any.pkg.tar.zst`; +const at129 = `${'a'.repeat(109)}-1-1-any.pkg.tar.zst`; + +// The cases of tests/unit/10-tag-encoding.sh, plus edges of the mapping. +const HAND = [ + ['zlib-1:1.3.2-3-x86_64.pkg.tar.zst', 'zlib-1_c1.3.2-3-x86_64.pkg.tar.zst'], + ['gcc-libs-15.2.1+r22+gc4e96a094636-1-x86_64.pkg.tar.zst', 'gcc-libs-15.2.1_pr22_pgc4e96a094636-1-x86_64.pkg.tar.zst'], + ['python-typing_extensions-4.15.0-1-any.pkg.tar.zst', 'python-typing_extensions-4.15.0-1-any.pkg.tar.zst'], + ['a_pb-1-1-any.pkg.tar.zst', 'a_upb-1-1-any.pkg.tar.zst'], + ['a_+b-1-1-any.pkg.tar.zst', 'a__pb-1-1-any.pkg.tar.zst'], + ['a__p-1-1-any.pkg.tar.zst', 'a__up-1-1-any.pkg.tar.zst'], + ['a_uu-1-1-any.pkg.tar.zst', 'a_uuu-1-1-any.pkg.tar.zst'], + ['@x-1-1-any.pkg.tar.zst', '_ax-1-1-any.pkg.tar.zst'], + ['+x-1-1-any.pkg.tar.zst', '_px-1-1-any.pkg.tar.zst'], + ['x86_64-1-1-any.pkg.tar.zst', 'x86_64-1-1-any.pkg.tar.zst'], + ['lshw-B.02.20-3-x86_64.pkg.tar.zst', hashed('lshw-B.02.20-3-x86_64.pkg.tar.zst')], + ['.hidden-1-1-any.pkg.tar.zst', hashed('.hidden-1-1-any.pkg.tar.zst')], + ['-dash-1-1-any.pkg.tar.zst', hashed('-dash-1-1-any.pkg.tar.zst')], + ['caf%C3%A9-1-1-any.pkg.tar.zst', hashed('caf%C3%A9-1-1-any.pkg.tar.zst')], + ['a!b-1-1-any.pkg.tar.zst', hashed('a!b-1-1-any.pkg.tar.zst')], + [at128, at128], + [at129, hashed(at129)], +]; + +test('tagOf gives the hand-checked tags', async () => { + assert.equal(at128.length, 128); + for (const [name, want] of HAND) assert.equal(await tagOf(name), want, name); +}); + +test('tagOf matches client/pacman-oci --tags for every name in tests/fixtures/filenames.txt', async () => { + const names = readFileSync(FIXTURE, 'utf8').split('\n').filter((l) => l !== ''); + assert.ok(names.length >= 2000, `the fixture has ${names.length} names`); + const all = [...names, ...HAND.map(([n]) => n)]; + const lines = client(['--tags'], `${all.join('\n')}\n`).split('\n').filter((l) => l !== ''); + assert.equal(lines.length, all.length, 'one line out per name in'); + const diffs = []; + for (const [i, line] of lines.entries()) { + const [name, tag] = line.split('\t'); + assert.equal(name, all[i], `line ${i + 1} names the input`); + const got = await tagOf(name); + if (got !== tag) diffs.push(`${name}: client ${tag}, worker ${got}`); + } + assert.deepEqual(diffs, [], `${diffs.length} of ${all.length} names differ`); +}); + +test('tagOf matches client/pacman-oci --tag one name at a time', async () => { + const sample = [ + ...HAND.map(([n]) => n), + ...readFileSync(FIXTURE, 'utf8') + .split('\n') + .filter((l) => /[+:@_]|[A-Z]/.test(l)) + .slice(0, 12), + ]; + for (const name of sample) { + const want = client(['--tag', name]).replace(/\n$/, ''); + assert.equal(await tagOf(name), want, name); + } +}); diff --git a/worker/test/world.js b/worker/test/world.js new file mode 100644 index 0000000..8cf9f8b --- /dev/null +++ b/worker/test/world.js @@ -0,0 +1,293 @@ +// A test world for worker/src/index.js: a GHCR stand-in behind a mock fetch, +// a Cache API stub with a clock, a ctx that collects waitUntil promises, and +// a capture of console.error. No network. +// +// const w = world(); // installs the globals +// const res = await w.get('/x86_64/core/core.db'); +// w.close(); // restores them +// +// Every response body that the stand-in returns is tracked. w.unusedBodies() +// lists the ones the Worker neither read nor cancelled. + +import worker from '../src/index.js'; + +export const HOST = 'ghcr.mirror.example.test'; +export const REGISTRY = 'https://ghcr.io/v2/pkgforge-dev/mirror-archlinux'; +export const BLOB_HOST = 'https://pkg-containers.githubusercontent.com'; +export const T0 = Date.parse('2026-10-14T03:00:00Z'); +export const MANIFEST_TYPE = 'application/vnd.oci.image.manifest.v1+json'; + +export const hex = (c) => c.repeat(64); +export const digest = (c) => `sha256:${hex(c)}`; + +export function manifestJson(layers, extra = {}) { + return JSON.stringify({ + schemaVersion: 2, + mediaType: MANIFEST_TYPE, + config: { mediaType: 'application/vnd.oci.empty.v1+json', digest: digest('4'), size: 2 }, + layers: layers.map(([title, d, size = 100]) => ({ + mediaType: 'application/octet-stream', + digest: d, + size, + annotations: { 'org.opencontainers.image.title': title }, + })), + ...extra, + }); +} + +export async function sha256hex(s) { + const d = new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(s))); + return Array.from(d, (b) => b.toString(16).padStart(2, '0')).join(''); +} + +// A body that records whether it was read or cancelled. highWaterMark 0: the +// source is pulled only when a reader asks for data. +function trackedBody(bytes, track, chunk = 0) { + let off = 0; + return new ReadableStream( + { + pull(c) { + track.read = true; + if (off >= bytes.byteLength) { + c.close(); + return; + } + const end = chunk > 0 ? Math.min(off + chunk, bytes.byteLength) : bytes.byteLength; + c.enqueue(bytes.slice(off, end)); + off = end; + if (off >= bytes.byteLength) c.close(); + }, + cancel() { + track.cancelled = true; + }, + }, + { highWaterMark: 0 }, + ); +} + +class CacheStub { + constructor(clock) { + this.clock = clock; + this.entries = new Map(); + this.matches = 0; + this.puts = []; + this.failMatch = false; + this.failPut = false; + } + + async match(key) { + this.matches += 1; + if (this.failMatch) throw new Error('cache stub: match failed'); + const url = typeof key === 'string' ? key : key.url; + const e = this.entries.get(url); + if (!e) return undefined; + if (e.until <= this.clock.now) { + this.entries.delete(url); + return undefined; + } + return new Response(e.body, { status: e.status, headers: e.headers }); + } + + async put(key, res) { + if (this.failPut) throw new Error('cache stub: put failed'); + if (typeof key === 'string' || key.method !== 'GET') throw new TypeError('cache stub: the key must be a GET Request'); + if (res.status === 206) throw new TypeError('cache stub: 206 cannot be stored'); + const cc = res.headers.get('cache-control') || ''; + const m = /(?:^|,)\s*max-age=(\d+)/.exec(cc); + if (!m || /no-store/.test(cc)) throw new TypeError(`cache stub: no max-age in "${cc}"`); + const ttl = Number(m[1]); + const body = await res.text(); + this.entries.set(key.url, { body, status: res.status, headers: [...res.headers], until: this.clock.now + ttl * 1000 }); + this.puts.push({ url: key.url, ttl, body: JSON.parse(body) }); + } + + // The stored entries whose key path starts with /__cache/v1//. + ofKind(kind) { + return this.puts.filter((p) => new URL(p.url).pathname.startsWith(`/__cache/v1/${kind}/`)); + } +} + +export function world() { + const clock = { now: T0 }; + const cache = new CacheStub(clock); + const saved = { + fetch: globalThis.fetch, + caches: Object.getOwnPropertyDescriptor(globalThis, 'caches'), + now: Date.now, + error: console.error, + timeout: AbortSignal.timeout, + }; + const w = { + clock, + cache, + requests: [], + errors: [], + bodies: [], + timeouts: [], + // GHCR state: manifests by "arch/repo:ref", blobs by "arch/repo@digest". + manifests: new Map(), + blobs: new Map(), + // A function (url, init) => Response | Promise, tried before the maps. + intercept: null, + // Seconds from the request time to the signed URL's se. + seAfter: 400, + }; + + w.response = (status, body = '', headers = {}, opts = {}) => { + const bytes = typeof body === 'string' ? new TextEncoder().encode(body) : body; + const track = { status, read: false, cancelled: false, url: opts.url || '' }; + w.bodies.push(track); + if ([101, 204, 205, 304].includes(status)) { + // A null body status: there is nothing to read or cancel. + track.read = true; + return new Response(null, { status, headers }); + } + return new Response(trackedBody(bytes, track, opts.chunk || 0), { status, headers }); + }; + + w.manifest = (repoRef, layers, extra) => { + const body = manifestJson(layers, extra); + w.manifests.set(repoRef, body); + return body; + }; + + w.signedUrl = (d) => { + const se = new Date(clock.now + w.seAfter * 1000).toISOString().replace(/\.\d{3}Z$/, 'Z'); + return `${BLOB_HOST}/ghcrblobs10/blobs/${d}?se=${encodeURIComponent(se)}&sig=abc%2Bdef%3D&sp=r`; + }; + + globalThis.fetch = async (url, init = {}) => { + w.requests.push({ url: String(url), init }); + if (w.intercept) { + const r = await w.intercept(String(url), init); + if (r !== undefined) return r; + } + const u = new URL(String(url)); + if (!u.href.startsWith(`${REGISTRY}/`)) throw new Error(`test world: unexpected fetch ${url}`); + const rest = u.pathname.slice(new URL(REGISTRY).pathname.length + 1); + let m = /^(.+)\/manifests\/(.+)$/.exec(rest); + if (m) { + const body = w.manifests.get(`${m[1]}:${decodeURIComponent(m[2])}`); + if (body === undefined) { + return w.response(404, '{"errors":[{"code":"MANIFEST_UNKNOWN","message":"manifest unknown"}]}', { + 'content-type': 'application/json', + }, { url }); + } + return w.response(200, body, { 'content-type': MANIFEST_TYPE, 'content-length': String(new TextEncoder().encode(body).byteLength) }, { url }); + } + m = /^(.+)\/blobs\/(sha256:[0-9a-f]{64})$/.exec(rest); + if (m) { + const b = w.blobs.get(`${m[1]}@${m[2]}`); + if (b === undefined) return w.response(404, '{"errors":[{"code":"BLOB_UNKNOWN"}]}', {}, { url }); + if (typeof b === 'function') return b(url, init); + return w.response(307, '', { location: b === true ? w.signedUrl(m[2]) : b, 'content-length': '0' }, { url }); + } + throw new Error(`test world: unexpected GHCR path ${rest}`); + }; + Object.defineProperty(globalThis, 'caches', { value: { default: cache }, configurable: true, writable: true }); + Date.now = () => clock.now; + console.error = (...args) => { + w.errors.push(args); + }; + AbortSignal.timeout = (ms) => { + w.timeouts.push(ms); + return saved.timeout.call(AbortSignal, ms); + }; + + // GET (or METHOD) PATH; waits for every waitUntil promise before it returns. + w.request = async (path, method = 'GET', headers = {}) => { + const pending = []; + const ctx = { + waitUntil: (p) => pending.push(p), + passThroughOnException: () => { + throw new Error('test world: passThroughOnException is not used'); + }, + }; + const req = new Request(`https://${HOST}${path}`, { method, headers }); + const res = await worker.fetch(req, {}, ctx); + while (pending.length > 0) await Promise.all(pending.splice(0)); + return res; + }; + w.get = (path) => w.request(path, 'GET'); + w.head = (path) => w.request(path, 'HEAD'); + + // The requests the Worker made from index FROM on. + w.since = (from) => w.requests.slice(from); + + w.unusedBodies = () => w.bodies.filter((b) => !b.read && !b.cancelled); + + w.advance = (seconds) => { + clock.now += seconds * 1000; + }; + + w.close = () => { + globalThis.fetch = saved.fetch; + if (saved.caches) Object.defineProperty(globalThis, 'caches', saved.caches); + else delete globalThis.caches; + Date.now = saved.now; + console.error = saved.error; + AbortSignal.timeout = saved.timeout; + }; + return w; +} + +// A world with x86_64/core: latest with core.db and core.files, three +// packages with their .sig layers, one package with no .sig layer, and a +// signed URL for every blob. +export const PKG = 'zlib-1:1.3.2-3-x86_64.pkg.tar.zst'; +export const PKG_TAG = 'zlib-1_c1.3.2-3-x86_64.pkg.tar.zst'; +export const PLUS = 'gcc-libs-15.2.1+r22+gc4e96a094636-1-x86_64.pkg.tar.zst'; +export const PLUS_TAG = 'gcc-libs-15.2.1_pr22_pgc4e96a094636-1-x86_64.pkg.tar.zst'; +export const NOSIG = 'nosig-1-1-any.pkg.tar.zst'; +export const UPPER = 'lshw-B.02.20-3-x86_64.pkg.tar.zst'; + +export const D = { + db: digest('a'), + files: digest('b'), + pkg: digest('c'), + pkgSig: digest('d'), + plus: digest('e'), + plusSig: digest('f'), + nosig: digest('1'), + upper: digest('2'), + upperSig: digest('3'), +}; + +export async function seeded() { + const w = world(); + const db = [ + ['core.db', D.db], + ['core.files', D.files], + ]; + w.manifest('x86_64/core:latest', db); + w.manifest(`x86_64/core:${PKG_TAG}`, [ + [PKG, D.pkg], + [`${PKG}.sig`, D.pkgSig], + ]); + w.manifest(`x86_64/core:${PLUS_TAG}`, [ + [PLUS, D.plus], + [`${PLUS}.sig`, D.plusSig], + ]); + w.manifest('x86_64/core:nosig-1-1-any.pkg.tar.zst', [[NOSIG, D.nosig]]); + w.manifest(`x86_64/core:fn-${await sha256hex(UPPER)}`, [ + [UPPER, D.upper], + [`${UPPER}.sig`, D.upperSig], + ]); + for (const d of Object.values(D)) w.blobs.set(`x86_64/core@${d}`, true); + return w; +} + +// Runs FN in a seeded world and restores the globals after it. Every test +// also checks that no GHCR body was left unread and uncancelled. +export async function inWorld(fn) { + const w = await seeded(); + try { + await fn(w); + const unused = w.unusedBodies(); + if (unused.length > 0) { + throw new Error(`GHCR bodies neither read nor cancelled: ${JSON.stringify(unused)}`); + } + } finally { + w.close(); + } +} diff --git a/worker/wrangler.toml b/worker/wrangler.toml new file mode 100644 index 0000000..ece3109 --- /dev/null +++ b/worker/wrangler.toml @@ -0,0 +1,34 @@ +# ghcr-mirror-archlinux-pkgforge: the HTTPS front for the GHCR mirror. +# +# Deploy: .github/workflows/worker.yml does it on main. By hand: +# cd worker && npx wrangler@4.150.0 deploy +# Check the configuration with no deploy: +# cd worker && npx wrangler@4.150.0 deploy --dry-run --outdir "$(mktemp -d)" +name = "ghcr-mirror-archlinux-pkgforge" +main = "src/index.js" +account_id = "2a97c8f21aff7a773bf976c61ace1381" +compatibility_date = "2026-10-13" +# From 2026-08-04 the compatibility date turns on the Node.js APIs. This +# Worker uses only Web APIs (fetch, Cache, Web Crypto), so it opts out. +compatibility_flags = ["no_nodejs_compat", "no_nodejs_compat_v2"] +workers_dev = false +preview_urls = false + +routes = [ + { pattern = "ghcr.mirror.archlinux.pkgforge.dev", custom_domain = true }, +] + +# Each request is at most three cache lookups, two GHCR subrequests and one +# SHA-256 of a manifest of at most 64 KiB. +[limits] +cpu_ms = 50 + +# Every request is sampled, but the invocation log is off. The Worker calls +# console.error only on a 5xx answer or a cache failure, so the logs hold +# every failure and nothing else. +[observability] +enabled = true +head_sampling_rate = 1 + +[observability.logs] +invocation_logs = false From 8f7a126b77cabd15ebac38fb77e0a09c43f3962b Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 11:26:38 +0545 Subject: [PATCH 03/22] status: measure the mirror hourly, render the Pages site and badges, watch the Worker --- .github/workflows/status.yml | 263 +++++ .github/workflows/watchdog.yml | 64 +- scripts/check-staleness | 51 +- scripts/cron-tolerance | 88 +- scripts/status-collect | 539 +++++++++++ scripts/status-render | 1553 ++++++++++++++++++++++++++++++ tests/fixtures/status/bench.json | 20 + tests/fixtures/status/fake-curl | 141 +++ tests/unit/60-status.sh | 563 +++++++++++ 9 files changed, 3269 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/status.yml create mode 100755 scripts/status-collect create mode 100755 scripts/status-render create mode 100644 tests/fixtures/status/bench.json create mode 100755 tests/fixtures/status/fake-curl create mode 100644 tests/unit/60-status.sh diff --git a/.github/workflows/status.yml b/.github/workflows/status.yml new file mode 100644 index 0000000..b2c0596 --- /dev/null +++ b/.github/workflows/status.yml @@ -0,0 +1,263 @@ +# Measure the mirror against upstream every hour, and publish the result as a +# static site: on GitHub Pages, and on the branch status. +# +# decide does this run also measure pacman? Yes on dispatch with +# bench=true, and once a day: the first run from 05:00 to 08:59 +# UTC that finds no bench of the day on the status branch +# bench scripts/bench-consumer; its JSON goes to build as an artifact +# build scripts/status-collect, then scripts/status-render. On main the +# site, history.jsonl, worker.md and bench/.json go to the +# branch status as one orphan commit, and the site to Pages as an +# artifact +# deploy actions/deploy-pages, on main, when Pages takes a workflow build +# +# ⚠ A person enables Pages once, with an admin token. GITHUB_TOKEN cannot: +# gh api -X POST repos/pkgforge-dev/mirror-archlinux/pages -f build_type=workflow +# Until then deploy says so in a notice and deploys nothing. +# +# The status branch holds the history the next run reads. watchdog.yml reads +# status.json and worker.md from it to raise worker-down. +name: Status + +on: + schedule: + - cron: '47 * * * *' + workflow_dispatch: + inputs: + bench: + description: Also measure pacman through each path (scripts/bench-consumer) + type: boolean + default: false + +permissions: {} + +concurrency: + group: status + cancel-in-progress: false + +defaults: + run: + shell: bash + +env: + MIRROR_REGISTRY: ghcr.io + +jobs: + decide: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + outputs: + bench: ${{ steps.decide.outputs.bench }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Decide whether this run measures pacman + id: decide + env: + BENCH_INPUT: ${{ inputs.bench }} + run: | + bench=false + hour="$(date -u +%H)" + today="$(date -u +%F)" + last="" + if [ "$BENCH_INPUT" = true ]; then + bench=true + else + rc=0 + git ls-remote --exit-code --heads origin status > "$RUNNER_TEMP/heads.txt" || rc=$? + case "$rc" in + 0) + git fetch --quiet --depth 1 origin status + if git cat-file -e FETCH_HEAD:bench.json; then + rc=0 + last="$(git show FETCH_HEAD:bench.json | jq -r '.date // empty' | cut -c 1-10)" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "::warning::bench.json on the status branch does not read; a new bench is due" + last="" + fi + fi + ;; + 2) echo "no status branch yet" ;; + *) + echo "::error::git ls-remote origin status ended $rc" + exit "$rc" + ;; + esac + if [ "$((10#$hour))" -ge 5 ] && [ "$((10#$hour))" -le 8 ] && [ "$last" != "$today" ]; then + bench=true + fi + fi + echo "bench: $bench (dispatch input '${BENCH_INPUT:-}', $hour:00 UTC, last bench ${last:-none}, today $today)" + echo "bench=$bench" >> "$GITHUB_OUTPUT" + + bench: + needs: decide + if: ${{ needs.decide.outputs.bench == 'true' }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Measure pacman through each path + env: + CONTAINER_RUNTIME: docker + run: | + mkdir -p "$RUNNER_TEMP/bench" + scripts/bench-consumer "$RUNNER_TEMP/bench/bench.json" + if ! jq -e '.schema == 1 and (.cases | type == "array") and (.date | type == "string")' "$RUNNER_TEMP/bench/bench.json" > "$RUNNER_TEMP/bench.check"; then + echo "::error::scripts/bench-consumer wrote no schema 1 bench.json" + sed -n 1,40p "$RUNNER_TEMP/bench/bench.json" + exit 1 + fi + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: bench + path: ${{ runner.temp }}/bench + retention-days: 7 + if-no-files-found: error + overwrite: true + + build: + needs: [decide, bench] + if: ${{ !cancelled() && needs.decide.result == 'success' }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Read the history and the bench from the status branch + run: | + in="$RUNNER_TEMP/status" + mkdir -p "$in" + rc=0 + git ls-remote --exit-code --heads origin status > "$RUNNER_TEMP/heads.txt" || rc=$? + case "$rc" in + 0) + git clone --quiet --depth 1 --branch status --single-branch \ + "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" "$RUNNER_TEMP/prev" + for f in history.jsonl bench.json; do + if [ -f "$RUNNER_TEMP/prev/$f" ]; then + cp "$RUNNER_TEMP/prev/$f" "$in/$f" + fi + done + if [ -d "$RUNNER_TEMP/prev/bench" ]; then + cp -r "$RUNNER_TEMP/prev/bench" "$in/bench" + fi + ;; + 2) echo "no status branch yet: the history starts with this run" ;; + *) + echo "::error::git ls-remote origin status ended $rc" + exit "$rc" + ;; + esac + ls -la "$in" + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + if: ${{ needs.bench.result == 'success' }} + with: + name: bench + path: ${{ runner.temp }}/bench-new + - name: Take this run's bench + if: ${{ needs.bench.result == 'success' }} + run: | + in="$RUNNER_TEMP/status" + day="$(jq -r '.date' "$RUNNER_TEMP/bench-new/bench.json" | cut -c 1-10)" + if ! [[ "$day" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then + echo "::error::bench.json names no date: '$day'" + exit 1 + fi + mkdir -p "$in/bench" + cp "$RUNNER_TEMP/bench-new/bench.json" "$in/bench.json" + cp "$RUNNER_TEMP/bench-new/bench.json" "$in/bench/$day.json" + - name: Measure the mirror, upstream and the Worker + run: scripts/status-collect "$RUNNER_TEMP/status" + - name: Render the site + run: | + python3 --version + python3 -I scripts/status-render "$RUNNER_TEMP/status" "$RUNNER_TEMP/site" + - name: Push the site and the history to the status branch + if: ${{ github.ref == 'refs/heads/main' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + # One orphan commit each run: the branch holds the current state, + # and history.jsonl is the record of the runs. + branch="$RUNNER_TEMP/branch" + rm -rf "$branch" + mkdir -p "$branch" + cp -r "$RUNNER_TEMP/site/." "$branch/" + cp "$RUNNER_TEMP/status/history.jsonl" "$RUNNER_TEMP/status/worker.md" "$branch/" + if [ -f "$RUNNER_TEMP/status/bench.json" ]; then + cp "$RUNNER_TEMP/status/bench.json" "$branch/bench.json" + fi + if [ -d "$RUNNER_TEMP/status/bench" ]; then + cp -r "$RUNNER_TEMP/status/bench" "$branch/bench" + fi + touch "$branch/.nojekyll" + generated="$(jq -r .generated "$branch/status.json")" + cd "$branch" + git init --quiet --initial-branch status + git add -A + git -c user.name='github-actions[bot]' \ + -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit --quiet -m "status: $generated" + auth="$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 -w 0)" + echo "::add-mask::$auth" + git -c http.extraheader="AUTHORIZATION: basic $auth" \ + push --force --quiet "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" HEAD:refs/heads/status + echo "status branch: $(git rev-parse HEAD), $(git ls-files | awk 'END { print NR }') file(s), measured $generated" + - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + if: ${{ github.ref == 'refs/heads/main' }} + with: + path: ${{ runner.temp }}/site + + deploy: + needs: build + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + concurrency: + group: pages + cancel-in-progress: false + steps: + - name: Check that Pages takes a workflow build + id: pages + env: + GH_TOKEN: ${{ github.token }} + run: | + rc=0 + gh api "repos/$GITHUB_REPOSITORY/pages" > "$RUNNER_TEMP/pages.json" 2> "$RUNNER_TEMP/pages.err" || rc=$? + if [ "$rc" -eq 0 ]; then + type="$(jq -r '.build_type // "unknown"' "$RUNNER_TEMP/pages.json")" + elif grep -q 'HTTP 404' "$RUNNER_TEMP/pages.err"; then + type=disabled + else + cat "$RUNNER_TEMP/pages.err" >&2 + echo "::error::could not read the Pages settings: gh api ended $rc" + echo " reproduce: gh api repos/$GITHUB_REPOSITORY/pages" + exit 1 + fi + if [ "$type" = workflow ]; then + echo "deploy=true" >> "$GITHUB_OUTPUT" + else + echo "::notice::Pages is '$type', not a workflow build, so nothing is deployed. Enable it once: gh api -X POST repos/$GITHUB_REPOSITORY/pages -f build_type=workflow" + echo "deploy=false" >> "$GITHUB_OUTPUT" + fi + - name: Deploy to Pages + id: deploy + if: ${{ steps.pages.outputs.deploy == 'true' }} + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/.github/workflows/watchdog.yml b/.github/workflows/watchdog.yml index fd4ccee..d5bc456 100644 --- a/.github/workflows/watchdog.yml +++ b/.github/workflows/watchdog.yml @@ -1,14 +1,20 @@ # Measure the outcome, not the runs: has each target's latest stopped -# following upstream? And have the schedules stopped firing? +# following upstream? Have the schedules stopped firing? Does the Worker +# still answer? # # ⭐ sync.yml's keepalive job runs the same schedule checks, so each workflow # witnesses the other. A schedule that stops firing leaves no run behind to # fail; this is the run that notices. +# +# Two crons. 41 7 runs every job once a day. 7 * runs only the Worker check, +# each hour, 20 minutes after status.yml records its probes: a Worker that +# fails its probe in 2 status runs in a row opens worker-down within the hour. name: Watchdog on: schedule: - cron: '41 7 * * *' + - cron: '7 * * * *' workflow_dispatch: permissions: {} @@ -23,8 +29,9 @@ defaults: jobs: staleness: + if: ${{ github.event_name != 'schedule' || github.event.schedule == '41 7 * * *' }} runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 20 permissions: contents: read issues: write @@ -32,6 +39,9 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + # The whole history: a target added less than the tolerance ago, + # with nothing promoted yet, is new, not stale. + fetch-depth: 0 - name: Check every target follows upstream env: GH_TOKEN: ${{ github.token }} @@ -59,6 +69,7 @@ jobs: exit "$rc" schedules: + if: ${{ github.event_name != 'schedule' || github.event.schedule == '41 7 * * *' }} runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: @@ -76,3 +87,52 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: scripts/check-schedules-fired + + worker: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Raise worker-down after 2 failed status runs in a row + env: + GH_TOKEN: ${{ github.token }} + run: | + # status.yml probes the Worker each hour and counts the runs in a + # row with a failed probe (scripts/status-collect). This reads its + # last record from the status branch. + rc=0 + git ls-remote --exit-code --heads origin status > "$RUNNER_TEMP/heads.txt" || rc=$? + case "$rc" in + 0) ;; + 2) + echo "::notice::no status branch yet: status.yml has recorded no Worker probe" + exit 0 + ;; + *) + echo "::error::git ls-remote origin status ended $rc" + exit "$rc" + ;; + esac + git fetch --quiet --depth 1 origin status + git show FETCH_HEAD:status.json > "$RUNNER_TEMP/status.json" + git show FETCH_HEAD:worker.md > "$RUNNER_TEMP/worker.md" + read -r generated age failed ok <<< "$(jq -r '[.generated, (now - .generated_epoch | floor), .worker.failed_runs, .worker.ok] | map(tostring) | join(" ")' "$RUNNER_TEMP/status.json")" + echo "status.json from $generated, $age s ago: Worker ok $ok, failed $failed run(s) in a row" + # Three status runs missed: the record says nothing about now. + if [ "$age" -gt 10800 ]; then + echo "::error::the last Worker probe is from $generated, $age s ago; status.yml has stopped recording" + echo " reproduce: git fetch origin status && git show FETCH_HEAD:status.json | jq '{generated, worker}'" + exit 1 + fi + if [ "$ok" = true ]; then + scripts/track-issue clear worker-down "The Worker passes its probes again, measured $generated: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/status/worker.md" + elif [ "$ok" = false ] && [ "$failed" -ge 2 ]; then + scripts/track-issue raise worker-down "The Worker fails its probe" "$RUNNER_TEMP/worker.md" + else + echo "nothing to raise or clear: worker-down opens at 2 failed runs in a row" + fi diff --git a/scripts/check-staleness b/scripts/check-staleness index f736bf7..a60caee 100755 --- a/scripts/check-staleness +++ b/scripts/check-staleness @@ -16,7 +16,13 @@ # run that is green and promotes nothing, a target held run after run by one # bad file: each ends here, as a database that has stopped following upstream. # -# OUT/.state current fresh stale unknown +# A target with no latest at all, whose file git added no more than the +# tolerance ago, is new: the sync has not seeded it yet. Older, it is stale. ⚠ +# When git cannot say when the file was added (no git, a shallow clone, a file +# git does not track), it is stale: an unknown age is never a grace period. +# watchdog.yml checks out the whole history for this. +# +# OUT/.state current fresh new stale unknown # OUT/.md the issue body for a stale or unknown target # # Environment: MIRROR_NOW (seconds) pins the clock, for the test suite only. @@ -53,6 +59,29 @@ run_url="${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:-pkgforge- work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT +# added_date CONF -> YYYY-MM-DD of the last commit that added CONF, or nothing +# when git cannot say; what git said then goes to stderr. +added_date() { + local dir base shallow rc=0 + if ! command -v git > /dev/null; then + log "git is not on PATH, so the date $1 was added is unknown" + return 0 + fi + dir="$(dirname "$1")" + base="$(basename "$1")" + shallow="$(git -C "$dir" rev-parse --is-shallow-repository 2> "$work/git.err")" || rc=$? + if [ "$rc" -ne 0 ]; then + log "git cannot say when $1 was added: $(sed -n 1p "$work/git.err")" + return 0 + fi + if [ "$shallow" != false ]; then + log "the clone is shallow, so the date $1 was added is unknown" \ + "reproduce: git -C $dir rev-parse --is-shallow-repository" + return 0 + fi + git -C "$dir" log --diff-filter=A --format=%cs -- "$base" | awk 'NR == 1' +} + worst=0 for conf in "$@"; do target_check "$conf" @@ -71,8 +100,11 @@ for conf in "$@"; do fi rc=0 REGISTRY_TOKEN='' registry_manifest "$repo_path" latest "$work/$name.json" || rc=$? + latest_absent=0 if [ "$rc" -eq "$REG_STATUS_OK" ]; then fetched="$(jq -r '.annotations["dev.pkgforge.mirror.fetched"] // empty' "$work/$name.json")" + elif [ "$rc" -eq "$REG_STATUS_ABSENT" ]; then + latest_absent=1 fi # What upstream serves now @@ -88,12 +120,20 @@ for conf in "$@"; do done <<< "$(target_all "$conf" db_server)" verdict="$(stale_verdict "$promoted" "$upstream" "$fetched" "$today" "$tolerance")" + added="" + if [ "$verdict" = stale ] && [ "$latest_absent" -eq 1 ]; then + added="$(added_date "$conf")" + if [ -n "$added" ] && [ "$("$REPO_ROOT/scripts/date-age" "$added" "$today")" -le "$tolerance" ]; then + verdict=new + fi + fi printf '%s\n' "$verdict" > "$out/$name.state" - printf '%-24s %-8s promoted %s, fetched %s, upstream %s, tolerance %s day(s)\n' \ - "$name" "$verdict" "${promoted:-none}" "${fetched:-never}" "${upstream:-unreadable}" "$tolerance" + printf '%-24s %-8s promoted %s, fetched %s, upstream %s, tolerance %s day(s)%s\n' \ + "$name" "$verdict" "${promoted:-none}" "${fetched:-never}" "${upstream:-unreadable}" "$tolerance" \ + "${added:+, target added $added}" case "$verdict" in - current | fresh) continue ;; + current | fresh | new) continue ;; esac worst=3 # shellcheck disable=SC2016 # Markdown backticks, not command substitution @@ -105,6 +145,9 @@ for conf in "$@"; do printf '| `latest` serves | `%s` |\n' "${promoted:-nothing readable}" printf '| fetched | %s |\n' "${fetched:-never}" printf '| upstream now | `%s` |\n' "${upstream:-no db_server answered}" + if [ "$latest_absent" -eq 1 ]; then + printf '| target file added | %s |\n' "${added:-unknown to git}" + fi printf '| tolerance | %s day(s), from sync.yml'"'"'s cron |\n\n' "$tolerance" printf 'The latest sync runs and their summaries: %s\n\n' "$run_url" printf 'Each run'"'"'s summary names the state of every target (held, replaced, private, behind, error) and the files at fault.\n\n' diff --git a/scripts/cron-tolerance b/scripts/cron-tolerance index 725d0e3..c6d9d5c 100755 --- a/scripts/cron-tolerance +++ b/scripts/cron-tolerance @@ -3,12 +3,20 @@ # How many days of silence from one workflow's schedule is normal. # # scripts/cron-tolerance .github/workflows/sync.yml +# scripts/cron-tolerance --minutes .github/workflows/sync.yml # # Prints two integers on one line: the largest gap in days the schedule itself # produces, and the tolerance built from it. # # 1 3 a daily cron: it never skips a day, and three days of silence is a stop # +# With --minutes it prints one integer instead: the largest gap in minutes +# between two firings. scripts/status-collect uses it to show the age of a +# snapshot against the sync interval. +# +# 180 '17 */3 * * *' +# 1440 '41 7 * * *' +# # ⛔ The number is derived, never written down. A check that says "fail after 3 # days" stops being right the moment somebody changes a cron from daily to # weekly, and nothing would say so. This reads the cron the workflow actually @@ -26,9 +34,9 @@ # point: the schedule is the contract and the runs are what is measured against # it. # -# ⛔ Day resolution only. The hour and minute fields are read and ignored: a -# published tag is named for a date, so a check finer than a day has nothing to -# compare against. ⚠ GitHub's own delay is measured in hours here, the two runs +# ⛔ Day resolution only, except with --minutes. The hour and minute fields +# are read and ignored for the tolerance: a published tag is named for a date, +# so a check finer than a day has nothing to compare against. ⚠ GitHub's own delay is measured in hours here, the two runs # on record firing at 17:32Z and 16:36Z against a `30 05` cron, and a delay of # hours cannot move a date by more than one day. # @@ -54,8 +62,13 @@ die() { exit 1 } +mode=days +if [ "${1:-}" = --minutes ]; then + mode=minutes + shift +fi wf="${1:-}" -[ -n "$wf" ] || die "usage: cron-tolerance " +[ -n "$wf" ] || die "usage: cron-tolerance [--minutes] " [ -f "$wf" ] || die "no such file: $wf" \ "reproduce: ls .github/workflows/" @@ -98,8 +111,13 @@ fi # # ⛔ No strftime and no mktime. Both are gawk extensions and the runners ship # mawk, which is also why nothing here uses match()'s third argument. The calendar is walked by hand instead. +# +# With --minutes the walk also lists the minutes of each firing day. The +# minute and hour fields do not depend on the day, so each set of crons that +# fire together gives one sorted list, made once and kept. The gap from the +# last firing back round to the first firing of the window counts too. #---------------------------------------------------------------------------# -read -r gap tolerance <<< "$(awk -v crons="$crons" ' +read -r gap tolerance <<< "$(awk -v crons="$crons" -v mode="$mode" ' function leap(y) { return (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 } function mdays(y, m) { @@ -178,18 +196,44 @@ read -r gap tolerance <<< "$(awk -v crons="$crons" ' nc = split(crons, list, "\n") + # The minutes of the day each cron fires at, ascending. + for (i = 1; i <= nc; i++) { + if (split(list[i], f, /[[:space:]]+/) != 5) { + printf "cron-tolerance: cannot parse cron expression: %s\n", list[i] > "/dev/stderr" + exit 2 + } + cnt[i] = 0 + for (h = 0; h < 24; h++) { + a = field(f[2], h, 0, 23) + for (mm = 0; mm < 60; mm++) { + b = field(f[1], mm, 0, 59) + if (a < 0 || b < 0) { + printf "cron-tolerance: cannot parse cron expression: %s\n", list[i] > "/dev/stderr" + exit 2 + } + if (a == 1 && b == 1) mins[i, ++cnt[i]] = h * 60 + mm + } + } + if (cnt[i] == 0) { + printf "cron-tolerance: the cron expression fires at no minute of the day: %s\n", list[i] > "/dev/stderr" + exit 2 + } + } + y = 2024; m = 1; d = 1; dow = 1 # 2024-01-01 was a Monday run = 0; gap = 0; fired = 0 + mgap = 0; mlast = -1; mfirst = -1 for (day = 0; day < 1461; day++) { hit = 0 + key = "" for (i = 1; i <= nc; i++) { r = fires(list[i], m, d, dow) if (r < 0) { printf "cron-tolerance: cannot parse cron expression: %s\n", list[i] > "/dev/stderr" exit 2 } - if (r == 1) { hit = 1 } + if (r == 1) { hit = 1; key = key " " i } } if (hit) { fired++ @@ -199,6 +243,25 @@ read -r gap tolerance <<< "$(awk -v crons="$crons" ' run++ } + if (hit && mode == "minutes") { + if (!(key in daylist)) { + split("", mark) + nk = split(key, ks, " ") + for (j = 1; j <= nk; j++) + for (k = 1; k <= cnt[ks[j]]; k++) mark[mins[ks[j], k]] = 1 + s = "" + for (t = 0; t < 1440; t++) if (t in mark) s = s " " t + daylist[key] = s + } + nt = split(daylist[key], ts, " ") + for (j = 1; j <= nt; j++) { + abs = day * 1440 + ts[j] + if (mlast >= 0 && abs - mlast > mgap) mgap = abs - mlast + if (mfirst < 0) mfirst = abs + mlast = abs + } + } + d++ if (d > mdays(y, m)) { d = 1; m++ } if (m > 12) { m = 1; y++ } @@ -211,6 +274,13 @@ read -r gap tolerance <<< "$(awk -v crons="$crons" ' } if (run > gap) gap = run + if (mode == "minutes") { + # Round from the last firing of the window to the first one. + if (1461 * 1440 - mlast + mfirst > mgap) mgap = 1461 * 1440 - mlast + mfirst + printf "%d %d\n", mgap, 0 + exit 0 + } + # gap counts the silent days between two firings. A daily cron has none, and # its largest gap is one day. gap = gap + 1 @@ -221,4 +291,8 @@ read -r gap tolerance <<< "$(awk -v crons="$crons" ' [ -n "${gap:-}" ] && [ -n "${tolerance:-}" ] || die "the cron walk printed nothing for $wf" \ "reproduce: scripts/cron-tolerance $wf" -printf '%s %s\n' "$gap" "$tolerance" +if [ "$mode" = minutes ]; then + printf '%s\n' "$gap" +else + printf '%s %s\n' "$gap" "$tolerance" +fi diff --git a/scripts/status-collect b/scripts/status-collect new file mode 100755 index 0000000..e87aee9 --- /dev/null +++ b/scripts/status-collect @@ -0,0 +1,539 @@ +#!/usr/bin/env bash +# +# Measure what the mirror serves against what upstream serves, for the status +# page and the watchdog. +# +# scripts/status-collect OUT [TARGET.conf ...] +# +# With no target file, every enabled target in targets/ is measured. For each +# target: +# upstream .db from the first db_server that answers 200, in order +# mirror the latest manifest and the database it names, read with no +# credential (Bearer QQ== on GHCR), the way a consumer reads +# coverage upstream (filename, sha256) pairs that the mirror database also +# names, by count and by bytes; the first 20 missing file names +# age now minus the snapshot's fetched time +# lag upstream newest build date minus the mirror's newest build date +# worker GET ///.db and one package, with no +# redirect followed: 302 to https://*.githubusercontent.com when +# latest exists, 404 when it does not +# ghcr the time of the direct latest manifest GET +# +# Writes: +# OUT/status.json schema 1, below +# OUT/history.jsonl one compact line appended per run; the last +# STATUS_HISTORY_LINES (2000) lines are kept +# OUT/worker.md the worker-down issue body: every probe, and how to +# reproduce the first failure +# OUT/history.jsonl is read first when it exists, so the caller puts the +# previous run's copy there. +# +# status.json, schema 1. Times are ISO 8601 UTC, sizes are bytes, durations +# are seconds, a ratio is 0 to 1, null is "not measured": +# schema 1, generated, generated_epoch, source, site_url, registry, +# namespace, worker_url +# sync {workflow, interval_minutes, tolerance_days}, from sync.yml's cron +# summary {targets, promoted, current, upstream_files, upstream_bytes, +# mirrored_files, mirrored_bytes, coverage_files, coverage_bytes, +# oldest_age_seconds, newest_age_seconds} +# worker {probed, ok, probes, failures, failed_runs, failing_since, +# median_seconds}; failed_runs counts the runs in a row, this one +# included, in which a probe failed +# ghcr {median_manifest_seconds} +# targets[] {target, arch, repo, repository, +# upstream {ok, server, sha256, files, bytes, newest_builddate, tried[] +# {server, result}, error}, +# mirror {state promoted|absent|error, error, manifest_code, +# manifest_seconds, fetched, run, db_sha256, files, bytes, +# newest_builddate}, +# current true when the mirror database is upstream's database, +# coverage {files, of_files, ratio_files, bytes, of_bytes, ratio_bytes, +# missing, missing_names[]} or null when upstream is unread, +# age_seconds, lag_seconds, +# worker {db, package}, each {url, code, expect, seconds, location, ok, +# error} or null} +# +# history.jsonl, one object per line: +# {"t": generated, "w": {"ok", "f": failed_runs, "s": failing_since, "ms"}, +# "g": {"ms"}, "x": {"": {"n": upstream files, "m": mirrored +# files, "b": upstream bytes, "mb": mirrored bytes, "a": age, "l": lag, +# "c": 1 current, 0 not, null unknown}}} +# +# Environment: +# MIRROR_NAMESPACE owner/name, default from GITHUB_REPOSITORY +# MIRROR_REGISTRY default ghcr.io +# STATUS_WORKER_URL default https://ghcr.mirror.archlinux.pkgforge.dev; +# empty: no Worker probe +# STATUS_SITE_URL default https://.github.io// +# STATUS_HISTORY_LINES default 2000 +# MIRROR_NOW seconds; pins the clock, for the test suite only +# +# ⚠ db_records reads the database with GNU tar (--wildcards). busybox tar +# cannot, so this runs on the runner, not in Alpine. +# +# Exit status: 0 measured (a target that cannot be read is a measurement, and +# is in status.json), 1 the collect itself could not run. +set -euo pipefail +SCRIPT_NAME=status-collect +# shellcheck source=lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +need curl jq awk sha256sum tar sort + +out="${1:-}" +[ -n "$out" ] || die "usage: scripts/status-collect OUT [TARGET.conf ...]" +shift +[ -n "$MIRROR_NAMESPACE" ] || die "MIRROR_NAMESPACE is not set" \ + "set it to owner/name, or set GITHUB_REPOSITORY" +mkdir -p "$out" + +# Anonymous, the way a consumer reads: registry_token answers QQ== on GHCR. +REGISTRY_TOKEN='' +WORKER_URL="${STATUS_WORKER_URL-https://ghcr.mirror.archlinux.pkgforge.dev}" +WORKER_URL="${WORKER_URL%/}" +HISTORY_LINES="${STATUS_HISTORY_LINES:-2000}" +[[ "$HISTORY_LINES" =~ ^[1-9][0-9]*$ ]] || die "STATUS_HISTORY_LINES must be a positive integer, not '$HISTORY_LINES'" +owner="${MIRROR_NAMESPACE%%/*}" +name_part="${MIRROR_NAMESPACE#*/}" +SITE_URL="${STATUS_SITE_URL:-https://$owner.github.io/${name_part%%/*}/}" +SOURCE_URL="${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:-$MIRROR_NAMESPACE}" +UA="mirror-archlinux-status (+$SOURCE_URL)" + +if [ "$#" -eq 0 ]; then + for conf in "$TARGETS_DIR"/*.conf; do + [ -e "$conf" ] || continue + if target_enabled "$conf"; then + set -- "$@" "$conf" + fi + done +fi +[ "$#" -gt 0 ] || die "no enabled target" "reproduce: ls $TARGETS_DIR" + +now="$(now_epoch)" +[[ "$now" =~ ^[0-9]+$ ]] || die "the clock reads '$now', not seconds" +generated="$(jq -nr --argjson t "$now" '$t | todate')" + +sync_wf="$REPO_ROOT/.github/workflows/sync.yml" +interval="$("$REPO_ROOT/scripts/cron-tolerance" --minutes "$sync_wf")" +read -r _ tolerance <<< "$("$REPO_ROOT/scripts/cron-tolerance" "$sync_wf")" + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +mkdir -p "$work/t" + +#---------------------------------------------------------------------------# +# get URL OUT [HEADER ...] -> "CODE SECONDS LOCATION" on stdout, curl's status +# as the return. Redirects are not followed: a probe measures the answer it +# gets. The body is kept in OUT, so an error body can be quoted. +#---------------------------------------------------------------------------# +get() { + local url="$1" body="$2" args=() h rc=0 res + shift 2 + for h in "$@"; do + args+=(--header "$h") + done + res="$(curl -q --silent --show-error --connect-timeout 10 --max-time 30 \ + --proto '=http,https' --user-agent "$UA" "${args[@]}" \ + --output "$body" --write-out '%{http_code} %{time_total} %{redirect_url}' "$url")" || rc=$? + printf '%s\n' "$res" + return "$rc" +} + +# A database from upstream: redirects followed, a stall deadline rather than a +# fixed one, as scripts/plan reads it. +get_db() { + local url="$1" body="$2" rc=0 res + res="$(curl -q --location --silent --show-error --max-redirs 5 \ + --proto "$UPSTREAM_PROTO" --proto-redir "$UPSTREAM_PROTO" \ + --connect-timeout 20 --speed-limit 4096 --speed-time 60 \ + --retry 2 --retry-connrefused --retry-delay 2 --user-agent "$UA" \ + --output "$body" --write-out '%{http_code} %{time_total}' "$url")" || rc=$? + printf '%s\n' "$res" + return "$rc" +} + +# num VALUE -> VALUE when it is a JSON number, else null +num() { + if [[ "${1:-}" =~ ^-?[0-9]+([.][0-9]+)?$ ]]; then + printf '%s\n' "$1" + else + printf 'null\n' + fi +} + +# http_code VALUE -> the three digit code as a number, 0 when curl gave none +http_code() { + if [[ "${1:-}" =~ ^[0-9]{3}$ ]]; then + printf '%d\n' "$((10#$1))" + else + printf '0\n' + fi +} + +# records_stats RECORDS.tsv -> "COUNT BYTES NEWEST_BUILDDATE" +records_stats() { + awk -F '\t' ' + { n++; b += $6; if ($7 + 0 > nb) nb = $7 + 0 } + END { printf "%d %.0f %.0f\n", n, b, nb } + ' "$1" +} + +# probe NAME URL EXPECT DIR -> one probe object, as JSON, on stdout +# +# EXPECT is the code the Worker must answer: 302 when latest exists, 404 when +# it does not, empty when the registry could not be read. With no EXPECT only +# no answer or a 5xx fails; any other answer is measured and not judged. +probe() { + local label="$1" url="$2" expect="$3" dir="$4" rc=0 res code secs loc host ok err + res="$(get "$url" "$dir/$label.body")" || rc=$? + read -r code secs loc <<< "$res" + code="$(http_code "${code:-}")" + host="" + if [ -n "${loc:-}" ]; then + host="${loc#*://}" + host="${host%%[/?#]*}" + fi + ok=null + err="" + if [ "$rc" -ne 0 ]; then + err="curl $rc: no answer within the 30 s deadline, or the connection failed" + elif [ -z "$expect" ]; then + if [ "$code" -ge 500 ]; then err="HTTP $code"; fi + elif [ "$code" != "$expect" ]; then + err="HTTP $code, expected $expect" + elif [ "$expect" = 302 ] && ! [[ "${loc:-}" =~ ^https://[^/?#]*\.githubusercontent\.com/ ]]; then + err="302 to '${host:-nothing}', not to https://*.githubusercontent.com" + fi + if [ -n "$err" ]; then + ok=false + elif [ -n "$expect" ]; then + ok=true + fi + [ -z "$err" ] || log "worker probe $url: $err" + jq -nc --arg url "$url" --argjson code "$code" --arg expect "$expect" \ + --argjson secs "$(num "${secs:-}")" --arg host "$host" --argjson ok "$ok" --arg err "$err" \ + '{url: $url, code: $code, expect: (if $expect == "" then null else ($expect | tonumber) end), + seconds: $secs, location: (if $host == "" then null else $host end), ok: $ok, + error: (if $err == "" then null else $err end)}' +} + +#---------------------------------------------------------------------------# +# One target +#---------------------------------------------------------------------------# +collect_target() { + local conf="$1" arch repo name repo_path tw + target_check "$conf" + arch="$(target_get "$conf" arch)" + repo="$(target_get "$conf" repo)" + name="$arch-$repo" + repo_path="$(target_repository "$conf")" + tw="$work/$name" + mkdir -p "$tw" + + #-------------------------------------------------------------------------# + # Upstream, from the first db_server that answers 200 + #-------------------------------------------------------------------------# + local server rc res code up_server="" up_err="" tried="[]" why + while IFS= read -r server; do + [ -n "$server" ] || continue + rc=0 + res="$(get_db "$server/$repo.db" "$tw/up.db")" || rc=$? + read -r code _ <<< "$res" + if [ "$rc" -eq 0 ] && [ "${code:-}" = 200 ]; then + up_server="$server" + break + fi + if [ "$rc" -ne 0 ]; then why="curl $rc"; else why="HTTP ${code:-000}"; fi + log "$name: $server/$repo.db: $why" + tried="$(jq -c --arg s "$server" --arg r "$why" '. + [{server: $s, result: $r}]' <<< "$tried")" + done <<< "$(target_all "$conf" db_server)" + + local up_ok=false up_sha="" up_n="" up_b="" up_nb="" + if [ -z "$up_server" ]; then + up_err="no db_server answered 200 for $repo.db" + else + up_sha="$(sha256sum "$tw/up.db" | cut -d ' ' -f 1)" + rc=0 + db_records "$tw/up.db" "$tw/up.tsv" || rc=$? + if [ "$rc" -ne 0 ]; then + up_err="$up_server/$repo.db does not read as a database (db_records ended $rc)" + log "$name: $up_err" + else + up_ok=true + read -r up_n up_b up_nb <<< "$(records_stats "$tw/up.tsv")" + fi + fi + + #-------------------------------------------------------------------------# + # The mirror's latest, anonymously + #-------------------------------------------------------------------------# + local token="" m_state m_err="" m_code=0 m_secs="" fetched="" run="" m_sha="" m_n="" m_b="" m_nb="" errc d + rc=0 + token="$(registry_token "$repo_path" pull)" || rc=$? + if [ "$rc" -eq 0 ]; then + res="$(get "$REG_URL/v2/$repo_path/manifests/latest" "$tw/latest.json" \ + "Authorization: Bearer $token" \ + 'Accept: application/vnd.oci.image.manifest.v1+json')" || rc=$? + read -r m_code m_secs _ <<< "$res" + m_code="$(http_code "${m_code:-}")" + else + m_secs="" + fi + if [ -z "$token" ]; then + m_state=error + m_err="no pull token for $repo_path" + elif [ "$rc" -ne 0 ]; then + m_state=error + m_err="no answer for $repo_path:latest (curl $rc)" + elif [ "$m_code" = 200 ] && jq -e '.schemaVersion == 2' "$tw/latest.json" > /dev/null; then + m_state=promoted + elif [ "$m_code" = 404 ]; then + errc="$(registry_error_code "$tw/latest.json")" + case "$errc" in + MANIFEST_UNKNOWN | NAME_UNKNOWN) m_state=absent ;; + *) + m_state=error + m_err="$repo_path:latest answered 404 with error code '${errc:-none}'" + ;; + esac + else + m_state=error + m_err="$repo_path:latest answered HTTP $m_code" + fi + + if [ "$m_state" = promoted ]; then + fetched="$(jq -r '.annotations["dev.pkgforge.mirror.fetched"] // empty' "$tw/latest.json")" + run="$(jq -r '.annotations["dev.pkgforge.mirror.run"] // empty' "$tw/latest.json")" + if [ -n "$fetched" ] && ! [[ "$fetched" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then + warn "$name: latest names a fetched time '$fetched' that is not ISO 8601 UTC; the age is not measured" + fetched="" + fi + read -r d _ _ <<< "$(manifest_layer "$tw/latest.json" "$repo.db")" + if [ -z "${d:-}" ]; then + m_state=error + m_err="$repo_path:latest has no single layer titled $repo.db" + else + rc=0 + registry_blob "$repo_path" "$d" "$tw/mirror.db" || rc=$? + if [ "$rc" -ne 0 ]; then + m_state=error + m_err="could not read $repo.db ($d) of $repo_path" + else + m_sha="${d#sha256:}" + rc=0 + db_records "$tw/mirror.db" "$tw/mirror.tsv" || rc=$? + if [ "$rc" -ne 0 ]; then + m_state=error + m_err="$repo.db of $repo_path:latest does not read as a database" + else + read -r m_n m_b m_nb <<< "$(records_stats "$tw/mirror.tsv")" + fi + fi + fi + fi + [ -z "$m_err" ] || log "$name: $m_err" + + #-------------------------------------------------------------------------# + # Coverage: an upstream file counts when the mirror names the same file + # name with the same sha256 + #-------------------------------------------------------------------------# + local cov=null cm cn cmb cnb missing_json + if [ "$up_ok" = true ] && [ "$m_state" != error ]; then + touch "$tw/mirror.tsv" + read -r cn cm cnb cmb <<< "$(awk -F '\t' -v miss="$tw/missing.txt" ' + FILENAME == ARGV[1] { have[$1 SUBSEP $5] = 1; next } + { n++; nb += $6; if (($1 SUBSEP $5) in have) { m++; mb += $6 } else print $1 > miss } + END { close(miss); printf "%d %d %.0f %.0f\n", n, m, nb, mb } + ' "$tw/mirror.tsv" "$tw/up.tsv")" + touch "$tw/missing.txt" + missing_json="$(LC_ALL=C sort "$tw/missing.txt" | awk 'NR <= 20' | jq -R . | jq -sc .)" + cov="$(jq -nc --argjson m "$cm" --argjson n "$cn" --argjson mb "$cmb" --argjson nb "$cnb" \ + --argjson names "$missing_json" \ + '{files: $m, of_files: $n, ratio_files: (if $n > 0 then $m / $n else null end), + bytes: $mb, of_bytes: $nb, ratio_bytes: (if $nb > 0 then $mb / $nb else null end), + missing: ($n - $m), missing_names: $names}')" + fi + + #-------------------------------------------------------------------------# + # The Worker, with no redirect followed + #-------------------------------------------------------------------------# + local w_db=null w_pkg=null expect="" pkg + if [ -n "$WORKER_URL" ]; then + case "$m_state" in + promoted) expect=302 ;; + absent) expect=404 ;; + esac + w_db="$(probe db "$WORKER_URL/$arch/$repo/$repo.db" "$expect" "$tw")" + if [ "$m_state" = promoted ]; then + # One package by a plain name: the first by name, so the probe is the + # same file from run to run while the database stays the same. + pkg="$(awk -F '\t' '$1 ~ /^[a-z0-9][a-z0-9._-]*[.]pkg[.]tar[.][a-z0-9]+$/ { print $1 }' "$tw/mirror.tsv" | + LC_ALL=C sort | awk 'NR == 1')" + if [ -n "$pkg" ]; then + w_pkg="$(probe package "$WORKER_URL/$arch/$repo/$pkg" 302 "$tw")" + fi + fi + fi + + jq -nc \ + --arg target "$name" --arg arch "$arch" --arg repo "$repo" \ + --arg repository "$MIRROR_REGISTRY/$repo_path" \ + --argjson up_ok "$up_ok" --arg up_server "$up_server" --arg up_sha "$up_sha" \ + --argjson up_n "$(num "$up_n")" --argjson up_b "$(num "$up_b")" --argjson up_nb "$(num "$up_nb")" \ + --argjson tried "$tried" --arg up_err "$up_err" \ + --arg m_state "$m_state" --arg m_err "$m_err" --argjson m_code "$m_code" \ + --argjson m_secs "$(num "${m_secs:-}")" --arg fetched "$fetched" --arg run "$run" \ + --arg m_sha "$m_sha" --argjson m_n "$(num "$m_n")" --argjson m_b "$(num "$m_b")" \ + --argjson m_nb "$(num "$m_nb")" --argjson cov "$cov" --argjson now "$now" \ + --argjson w_db "$w_db" --argjson w_pkg "$w_pkg" ' + def nul: if . == "" then null else . end; + { + target: $target, arch: $arch, repo: $repo, repository: $repository, + upstream: {ok: $up_ok, server: ($up_server | nul), sha256: ($up_sha | nul), + files: $up_n, bytes: $up_b, newest_builddate: $up_nb, tried: $tried, error: ($up_err | nul)}, + mirror: {state: $m_state, error: ($m_err | nul), manifest_code: $m_code, manifest_seconds: $m_secs, + fetched: ($fetched | nul), run: ($run | nul), db_sha256: ($m_sha | nul), + files: $m_n, bytes: $m_b, newest_builddate: $m_nb}, + current: (if $up_sha != "" and $m_sha != "" then $up_sha == $m_sha + elif $m_state == "absent" and $up_ok then false else null end), + coverage: $cov, + age_seconds: (if $fetched != "" then $now - ($fetched | fromdateiso8601) else null end), + lag_seconds: (if $up_nb != null and $m_nb != null then $up_nb - $m_nb else null end), + worker: {db: $w_db, package: $w_pkg} + }' > "$work/t/$name.json" + + printf '%-20s upstream %-5s %6s files mirror %-8s coverage %s worker %s\n' "$name" \ + "$up_ok" "${up_n:-?}" "$m_state" \ + "$(jq -r 'if .coverage == null then "unknown" else "\(.coverage.files)/\(.coverage.of_files)" end' "$work/t/$name.json")" \ + "$(jq -r '[.worker.db, .worker.package] | map(select(. != null) | "\(.code)\(if .ok == false then "!" else "" end)") | join(",") | if . == "" then "-" else . end' "$work/t/$name.json")" >&2 +} + +for conf in "$@"; do + collect_target "$conf" +done + +#---------------------------------------------------------------------------# +# The Worker's failure streak, from the previous run's line +#---------------------------------------------------------------------------# +prev_f=0 +prev_s="" +if [ -s "$out/history.jsonl" ]; then + last="$(tail -n 1 "$out/history.jsonl")" + rc=0 + parsed="$(jq -r '[(.w.f // 0 | tostring), (.w.s // "")] | join(" ")' <<< "$last" 2> "$work/jq.err")" || rc=$? + if [ "$rc" -eq 0 ] && [[ "$parsed" =~ ^([0-9]+)\ ?(.*)$ ]]; then + prev_f="${BASH_REMATCH[1]}" + prev_s="${BASH_REMATCH[2]}" + else + warn "the last line of $out/history.jsonl does not read; the Worker failure count starts again" \ + "$(cat "$work/jq.err")" "reproduce: tail -n 1 $out/history.jsonl | jq ." + fi +fi + +jq -s \ + --arg generated "$generated" --argjson now "$now" --arg source "$SOURCE_URL" \ + --arg site "$SITE_URL" --arg registry "$MIRROR_REGISTRY" --arg ns "$MIRROR_NAMESPACE" \ + --arg worker "$WORKER_URL" --argjson interval "$(num "$interval")" \ + --argjson tolerance "$(num "${tolerance:-}")" \ + --argjson prev_f "$prev_f" --arg prev_s "$prev_s" ' + def median: sort | if length == 0 then null elif length % 2 == 1 then .[length / 2 | floor] + else (.[length / 2 - 1] + .[length / 2]) / 2 end; + def sum_of(f): map(f // 0) | add // 0; + . as $t + | ([$t[] | .worker.db, .worker.package | select(. != null)]) as $probes + | ([$probes[] | select(.ok == false)] | length) as $fail + | (if ($probes | length) == 0 then null + elif $fail > 0 then false + elif ([$probes[] | select(.ok == true)] | length) > 0 then true + else null end) as $wok + | ([$t[] | select(.coverage != null)]) as $cov + | ([$t[] | .age_seconds | select(. != null)]) as $ages + | { + schema: 1, + generated: $generated, + generated_epoch: $now, + source: $source, + site_url: $site, + registry: $registry, + namespace: $ns, + worker_url: (if $worker == "" then null else $worker end), + sync: {workflow: ".github/workflows/sync.yml", interval_minutes: $interval, tolerance_days: $tolerance}, + summary: { + targets: ($t | length), + promoted: ([$t[] | select(.mirror.state == "promoted")] | length), + current: ([$t[] | select(.current == true)] | length), + upstream_files: ($cov | sum_of(.coverage.of_files)), + upstream_bytes: ($cov | sum_of(.coverage.of_bytes)), + mirrored_files: ($cov | sum_of(.coverage.files)), + mirrored_bytes: ($cov | sum_of(.coverage.bytes)), + coverage_files: (($cov | sum_of(.coverage.of_files)) as $n + | if $n > 0 then ($cov | sum_of(.coverage.files)) / $n else null end), + coverage_bytes: (($cov | sum_of(.coverage.of_bytes)) as $n + | if $n > 0 then ($cov | sum_of(.coverage.bytes)) / $n else null end), + oldest_age_seconds: ($ages | max), + newest_age_seconds: ($ages | min) + }, + worker: { + probed: ($worker != ""), + ok: $wok, + probes: ($probes | length), + failures: $fail, + failed_runs: (if $wok == false then $prev_f + 1 else 0 end), + failing_since: (if $wok == false then (if $prev_f > 0 and $prev_s != "" then $prev_s else $generated end) else null end), + median_seconds: ([$probes[] | .seconds | select(. != null)] | median) + }, + ghcr: {median_manifest_seconds: ([$t[] | .mirror.manifest_seconds | select(. != null)] | median)}, + targets: ($t | sort_by(.target)) + }' "$work"/t/*.json > "$work/status.json" +mv "$work/status.json" "$out/status.json" + +#---------------------------------------------------------------------------# +# History: one compact line per run, the last HISTORY_LINES kept +#---------------------------------------------------------------------------# +jq -c '{ + t: .generated, + w: {ok: .worker.ok, f: .worker.failed_runs, s: .worker.failing_since, + ms: (if .worker.median_seconds == null then null else (.worker.median_seconds * 1000 | round) end)}, + g: {ms: (if .ghcr.median_manifest_seconds == null then null else (.ghcr.median_manifest_seconds * 1000 | round) end)}, + x: (.targets | map({key: .target, value: { + n: .coverage.of_files, m: .coverage.files, b: .coverage.of_bytes, mb: .coverage.bytes, + a: .age_seconds, l: .lag_seconds, + c: (if .current == true then 1 elif .current == false then 0 else null end)}}) | from_entries) + }' "$out/status.json" > "$work/line.json" +if [ -f "$out/history.jsonl" ]; then + cat "$out/history.jsonl" "$work/line.json" > "$work/history.all" +else + cp "$work/line.json" "$work/history.all" +fi +tail -n "$HISTORY_LINES" "$work/history.all" > "$work/history.jsonl" +mv "$work/history.jsonl" "$out/history.jsonl" + +#---------------------------------------------------------------------------# +# The worker-down issue body +#---------------------------------------------------------------------------# +# shellcheck disable=SC2016 # Markdown backticks inside a jq program +jq -r ' + def cell: if . == null then "-" else tostring end; + "", + (if .worker.ok == false then + "The Worker fails its probe in \(.worker.failed_runs) run(s) in a row, since \(.worker.failing_since)." + elif .worker.ok == true then "The Worker passes every probe." + else "The Worker was not probed, or no probe could be judged." end), + "", + "Worker: \(.worker_url // "not set"). Each probe is a GET with no redirect followed. A target with `latest` must answer 302 to https://*.githubusercontent.com. A target with no `latest` must answer 404.", + "", + "| target | probe | HTTP | expected | seconds | result |", + "| --- | --- | --- | --- | --- | --- |", + (.targets[] as $t | [["db", $t.worker.db], ["package", $t.worker.package]][] + | select(.[1] != null) + | "| \($t.target) | \(.[0]) | \(.[1].code) | \(.[1].expect | cell) | \(.[1].seconds | cell) | \(if .[1].ok == true then "ok" elif .[1].ok == false then .[1].error else "not judged" end) |"), + "", + ([.targets[] | .worker.db, .worker.package | select(. != null and .ok == false)] | first // null + | if . == null then empty else + "reproduce:\n\n```bash\ncurl --connect-timeout 10 --max-time 30 -sS -o /dev/null -w \"%{http_code} %{redirect_url}\\n\" \"\(.url)\"\n```\n" + end), + "This issue closes itself when the Worker passes its probes again." +' "$out/status.json" > "$work/worker.md" +mv "$work/worker.md" "$out/worker.md" + +jq -r '"status-collect: \(.summary.targets) target(s), \(.summary.promoted) promoted, coverage \(if .summary.coverage_files == null then "unknown" else (.summary.coverage_files * 1000 | floor / 10 | tostring) + "%" end), worker \(.worker.ok | if . == null then "not judged" else tostring end), failed runs \(.worker.failed_runs)"' "$out/status.json" >&2 diff --git a/scripts/status-render b/scripts/status-render new file mode 100755 index 0000000..88baaf3 --- /dev/null +++ b/scripts/status-render @@ -0,0 +1,1553 @@ +#!/usr/bin/env python3 +# +# Render the status site from what scripts/status-collect measured. +# +# python3 -I scripts/status-render IN OUT_SITE +# +# IN holds status.json (required), history.jsonl and bench.json (optional). +# OUT_SITE gets: +# index.html one static page: inline CSS, light and dark, +# no JavaScript +# badges/.svg coverage and snapshot age of one target +# badges/.svg the same over the targets of one architecture +# badges/all.svg the same over every target +# badges/speed.svg pacman's time through the oci wrapper and +# through the official mirror, from bench.json +# charts/coverage.svg share of upstream files in the mirror, per +# target, grouped by architecture +# charts/freshness.svg snapshot age per target, against the sync +# interval +# charts/size.svg GiB per target, in the mirror and upstream only +# charts/history-coverage.svg coverage per architecture, last 30 days +# charts/history-age.svg oldest snapshot age per architecture, last 30 +# days +# charts/speed.svg bench.json: seconds per path and operation +# charts/narrow/.svg the same charts laid out 420 px wide, for a +# phone +# status.json, history.json, bench.json the data behind the page +# +# Every SVG has a title, axis labels, a legend and its own style with a dark +# variant. Colours are the validated colour-blind safe categorical slots, and +# an architecture keeps its colour in every chart. A state colour always comes +# with a shape and a word. +# +# Standard library only. Run it with python3 -I, so the script directory and +# the environment cannot change what is imported. Run without -I, the script +# starts itself again with it. +# +# Exit status: 0 rendered, 1 the input could not be read, 2 usage. + +import sys +import os + +if not sys.flags.isolated: + os.execv(sys.executable, [sys.executable, "-I"] + sys.argv) + +import datetime +import html +import json +import math + +PROG = "status-render" + +FONT = ('system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", ' + 'Arial, "Noto Sans", "Liberation Sans", sans-serif') +MONO = 'ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace' + +# The reference categorical slots, light and dark, in their validated order. +SLOTS_LIGHT = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100", + "#e87ba4", "#008300", "#4a3aa7", "#e34948"] +SLOTS_DARK = ["#3987e5", "#d95926", "#199e70", "#c98500", + "#d55181", "#008300", "#9085e9", "#e66767"] +# Slot 0 is the de-emphasis grey. +OTHER_LIGHT = "#a3a29c" +OTHER_DARK = "#6b6a65" +# State colours, never used for a series. Each comes with a shape and a word. +STATE = {"good": "#0ca30c", "warning": "#fab219", + "critical": "#d03b3b", "none": "#898781", "unknown": "#898781"} + +ARCH_ORDER = ["x86_64", "aarch64", "armv7h", "loong64", "riscv64"] +REPO_ORDER = ["core", "extra", "multilib", "alarm", "aur"] +PATH_ORDER = ["official", "worker", "oci", "oci-serial"] +PATH_LABEL = {"official": "official mirror", "worker": "Worker, native", + "oci": "oci wrapper", "oci-serial": "oci wrapper, serial"} +OP_ORDER = ["sy", "sw-small", "sw-large"] + +WIDE = 760 +NARROW = 420 +PAD = 24 + + +def warn(msg): + sys.stderr.write("%s: warning: %s\n" % (PROG, msg)) + + +def die(msg, *more): + sys.stderr.write("%s: %s\n" % (PROG, msg)) + for m in more: + sys.stderr.write(" %s\n" % m) + sys.exit(1) + + +#---------------------------------------------------------------------------# +# Formatting +#---------------------------------------------------------------------------# +def esc(s): + return html.escape("" if s is None else str(s), quote=True) + + +def fmt_int(n): + return "-" if n is None else "{:,}".format(int(n)) + + +def fmt_pct(r): + if r is None: + return "-" + if r >= 1: + return "100%" + if r <= 0: + return "0%" + if r < 0.001: + return "<0.1%" + v = math.floor(r * 1000) / 10 + return ("%.1f" % v).rstrip("0").rstrip(".") + "%" + + +def fmt_age(s): + if s is None: + return "-" + if s < 0: + s = 0 + if s < 3600: + return "%d min" % round(s / 60) + if s < 48 * 3600: + return "%.1f h" % (s / 3600) + return "%.1f d" % (s / 86400) + + +def fmt_interval(s): + return fmt_age(s).replace(".0 h", " h").replace(".0 d", " d") + + +def fmt_gib(b): + if b is None: + return "-" + g = b / 1073741824 + if g == 0: + return "0" + if g < 10: + return "%.2f" % g + if g < 100: + return "%.1f" % g + return "%.0f" % g + + +def fmt_secs(s): + if s is None: + return "-" + if s < 1: + return "%d ms" % round(s * 1000) + if s < 10: + return "%.2f s" % s + return "%.1f s" % s + + +def fmt_hours(h): + """An axis tick in hours: 0, 15 min, 1 h, 1 d, 7 d.""" + if h == 0: + return "0" + if h < 1: + return "%d min" % round(h * 60) + if h < 24 or h % 24: + return ("%.1f" % h).rstrip("0").rstrip(".") + " h" + return "%d d" % (h // 24) + + +def fmt_time(iso): + if not iso: + return "-" + return str(iso).replace("T", " ")[:16] + + +def utc(epoch): + return datetime.datetime.fromtimestamp(epoch, tz=datetime.timezone.utc) + + +def parse_time(s): + if not isinstance(s, str): + return None + try: + d = datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ") + except ValueError: + return None + return d.replace(tzinfo=datetime.timezone.utc).timestamp() + + +def median(values): + v = sorted(x for x in values if x is not None) + if not v: + return None + n = len(v) + if n % 2: + return v[n // 2] + return (v[n // 2 - 1] + v[n // 2]) / 2 + + +def num(x): + return isinstance(x, (int, float)) and not isinstance(x, bool) + + +# Text width, estimated from the proportions of a UI sans. Layout leaves +# room, so an estimate a little wide is safe and one too narrow is not. +NARROW_CHARS = set("il.,:;'|!()[]{}1 ") +WIDE_CHARS = set("mwMW@%") + + +def text_w(s, size, bold=False): + w = 0.0 + for c in str(s): + if c in NARROW_CHARS: + w += 0.30 + elif c in WIDE_CHARS: + w += 0.86 + elif c.isupper(): + w += 0.66 + elif c.isdigit(): + w += 0.58 + else: + w += 0.54 + return w * size * (1.07 if bold else 1.0) + + +# Verdana at 11 px, the width a badge is laid out for. Verdana is wide, so a +# system font fits in the same box. +VERDANA = {" ": 3.9, "%": 12.3, ".": 4.0, ",": 4.0, ":": 4.7, "-": 5.0, + "_": 7.0, "/": 4.9, "·": 4.0, "(": 4.6, ")": 4.6, "<": 9.2, + "a": 6.7, "b": 6.8, "c": 5.8, "d": 6.8, "e": 6.6, "f": 3.9, + "g": 6.8, "h": 7.0, "i": 3.0, "j": 3.8, "k": 6.5, "l": 3.0, + "m": 10.7, "n": 7.0, "o": 6.7, "p": 6.8, "q": 6.8, "r": 4.7, + "s": 5.8, "t": 4.3, "u": 7.0, "v": 6.5, "w": 9.0, "x": 6.5, + "y": 6.5, "z": 5.8} + + +def badge_w(s): + w = 0.0 + for c in s: + if c in VERDANA: + w += VERDANA[c] + elif c.isdigit(): + w += 7.0 + elif c.isupper(): + w += 7.6 + else: + w += 7.0 + return w + + +def nice_step(raw): + if raw <= 0: + return 1 + e = math.floor(math.log10(raw)) + f = raw / 10 ** e + for n in (1, 2, 2.5, 5, 10): + if f <= n + 1e-9: + return n * 10 ** e + return 10 ** (e + 1) + + +def nice_ticks(vmax, n=5): + if vmax <= 0: + vmax = 1 + step = nice_step(vmax / n) + top = math.ceil(vmax / step - 1e-9) * step + ticks = [] + i = 0 + while i * step <= top + step / 1000: + ticks.append(round(i * step, 10)) + i += 1 + return ticks, top + + +def hour_ticks(vmax, n=5): + """Ticks in hours on steps a reader counts in: 1, 2, 3, 6, 12 h, days.""" + steps = [0.25, 0.5, 1, 2, 3, 6, 12, 24, 48, 72, 168, 336, 720] + step = next((s for s in steps if vmax / s <= n), steps[-1]) + top = math.ceil(vmax / step - 1e-9) * step + return [i * step for i in range(int(round(top / step)) + 1)], top + + +#---------------------------------------------------------------------------# +# Identity: an architecture keeps its slot, whatever else is on the page +#---------------------------------------------------------------------------# +def arch_slots(arches): + known = [a for a in ARCH_ORDER if a in arches] + extra = sorted(a for a in arches if a not in ARCH_ORDER) + slots = {} + for i, a in enumerate(ARCH_ORDER): + slots[a] = i + 1 + nxt = len(ARCH_ORDER) + 1 + for a in extra: + slots[a] = nxt if nxt <= len(SLOTS_LIGHT) else 0 + nxt += 1 + return known + extra, slots + + +def repo_key(repo): + if repo in REPO_ORDER: + return (REPO_ORDER.index(repo), repo) + return (len(REPO_ORDER), repo) + + +def target_key(t, order): + arch = t.get("arch", "") + a = order.index(arch) if arch in order else len(order) + return (a, repo_key(t.get("repo", ""))) + + +#---------------------------------------------------------------------------# +# States +#---------------------------------------------------------------------------# +def freshness(t, interval, tolerance): + m = (t.get("mirror") or {}).get("state") + if m == "absent": + return "none", "not mirrored yet" + age = t.get("age_seconds") + if m != "promoted" or age is None: + return "unknown", "unknown" + if t.get("current") is True: + return "good", "matches upstream" + if age <= 2 * interval: + return "good", "within 2 syncs" + if age <= tolerance: + return "warning", "late" + return "critical", "stale" + + +def cov_ratio(t): + return (t.get("coverage") or {}).get("ratio_files") + + +def target_level(t, interval, tolerance): + f, _ = freshness(t, interval, tolerance) + if f in ("none", "unknown"): + return f + r = cov_ratio(t) + if f == "good" and (r is None or r < 1): + return "warning" + return f + + +def target_word(t, interval, tolerance): + lvl = target_level(t, interval, tolerance) + if lvl == "warning" and freshness(t, interval, tolerance)[0] == "good": + return lvl, "partial" + return lvl, {"good": "ok", "warning": "late", "critical": "stale", + "none": "not mirrored yet", "unknown": "unknown"}[lvl] + + +LEVEL_RANK = {"good": 0, "warning": 1, "critical": 2, "unknown": 1, "none": 1} + + +def not_read(t): + """Why a target has no coverage: which side could not be read.""" + if not (t.get("upstream") or {}).get("ok"): + return "upstream not read" + return "registry not read" + + +#---------------------------------------------------------------------------# +# SVG +#---------------------------------------------------------------------------# +def chart_style(): + rules = [] + for i, c in enumerate(SLOTS_LIGHT): + rules.append(".s%d{fill:%s}.l%d{stroke:%s}" % (i + 1, c, i + 1, c)) + rules.append(".s0{fill:%s}.l0{stroke:%s}" % (OTHER_LIGHT, OTHER_LIGHT)) + for k, c in STATE.items(): + rules.append(".st-%s{fill:%s}" % (k, c)) + dark = [] + for i, c in enumerate(SLOTS_DARK): + dark.append(".s%d{fill:%s}.l%d{stroke:%s}" % (i + 1, c, i + 1, c)) + dark.append(".s0{fill:%s}.l0{stroke:%s}" % (OTHER_DARK, OTHER_DARK)) + return ( + ".bg{fill:#fcfcfb;stroke:rgba(11,11,11,.10)}" + "text{font-family:%(f)s}" + ".tt{font-size:16px;font-weight:600;fill:#0b0b0b}" + ".hd{font-size:30px;font-weight:600;fill:#0b0b0b}" + ".sub{font-size:13px;fill:#52514e}" + ".lb{font-size:12px;fill:#0b0b0b}" + ".gr{font-size:12px;font-weight:600;fill:#52514e;paint-order:stroke;stroke:#fcfcfb;" + "stroke-width:5px;stroke-linejoin:round}" + ".vl{font-size:12px;fill:#0b0b0b;font-variant-numeric:tabular-nums}" + ".mu{fill:#6f6e69}" + ".lg{font-size:12px;fill:#52514e}" + ".tk{font-size:11px;fill:#6f6e69;font-variant-numeric:tabular-nums}" + ".ax{font-size:12px;fill:#52514e}" + ".note{font-size:13px;fill:#6f6e69}" + ".grid{stroke:#e9e8e3;stroke-width:1}" + ".base{stroke:#c3c2b7;stroke-width:1}" + ".ref{stroke:#7d7b75;stroke-width:1}" + ".lead{stroke:#c3c2b7;stroke-width:1;fill:none}" + ".track{fill:#ebeae5}" + ".ring{stroke:#fcfcfb;stroke-width:2}" + ".cut{stroke:#fcfcfb;stroke-width:2}" + ".line{fill:none;stroke-width:2;stroke-linejoin:round;stroke-linecap:round}" + ".hollow{fill:none;stroke:#898781;stroke-width:1.5}" + "%(rules)s" + "@media (prefers-color-scheme:dark){" + ".bg{fill:#1a1a19;stroke:rgba(255,255,255,.10)}" + ".tt,.hd,.lb,.vl{fill:#ffffff}" + ".sub,.gr,.ax,.lg{fill:#c3c2b7}" + ".tk,.note,.mu{fill:#9a9890}" + ".grid{stroke:#2a2a28}" + ".base,.lead{stroke:#3d3d3a}" + ".ref{stroke:#9a9890}" + ".track{fill:#2e2e2c}" + ".ring,.cut,.gr{stroke:#1a1a19}" + "%(dark)s}" + ) % {"f": FONT, "rules": "".join(rules), "dark": "".join(dark)} + + +class Svg: + def __init__(self, title, desc, w): + self.title = title + self.desc = desc + self.w = w + self.parts = [] + + def add(self, s): + self.parts.append(s) + + def text(self, x, y, s, cls, anchor=None): + a = ' text-anchor="%s"' % anchor if anchor else "" + self.add('%s' % (x, y, cls, a, esc(s))) + + def render(self, height): + w = self.w + return ( + '\n' + '%s\n%s\n' + '\n' + '\n' + '%s\n\n' + ) % (w, height, w, height, esc(self.title), esc(self.desc), + chart_style(), w - 1, height - 1, "\n".join(self.parts)) + + +def wrap(s, size, width): + lines = [] + cur = "" + for word in s.split(): + nxt = (cur + " " + word).strip() + if cur and text_w(nxt, size) > width: + lines.append(cur) + cur = word + else: + cur = nxt + if cur: + lines.append(cur) + return lines + + +def header(svg, title, headline, caption): + """The title, the headline number and its caption. Returns the next y.""" + svg.text(PAD, 40, title, "tt") + svg.text(PAD, 82, headline, "hd") + y = 82 + x = PAD + text_w(headline, 30, True) + 14 + room = svg.w - PAD - x + lines = wrap(caption, 13, room) if room > 200 else [] + if 0 < len(lines) <= 2: + if len(lines) == 1: + svg.text(x, y - 3, lines[0], "sub") + else: + svg.text(x, y - 18, lines[0], "sub") + svg.text(x, y - 1, lines[1], "sub") + return y + 30 + for ln in wrap(caption, 13, svg.w - 2 * PAD): + y += 19 + svg.text(PAD, y, ln, "sub") + return y + 30 + + +def glyph(svg, level, cx, cy): + if level == "good": + svg.add('' % (cx, cy)) + elif level == "warning": + svg.add('' % (cx, cy - 5.6)) + elif level == "critical": + svg.add('' % (cx, cy - 5)) + else: + svg.add('' % (cx, cy)) + + +def legend(svg, y, items): + """items: (kind, class, label), kind box, line or glyph. Returns the next y.""" + x = PAD + for kind, cls, label in items: + w = 16 + text_w(label, 12) + 18 + if x + w - 18 > svg.w - PAD and x > PAD: + x = PAD + y += 20 + if kind == "box": + svg.add('' + % (x, y - 9, cls)) + elif kind == "line": + svg.add('' % (x - 1, y - 4, cls)) + else: + glyph(svg, cls, x + 5, y - 4) + svg.text(x + 16, y, label, "lg") + x += w + return y + 16 + + +def hbar(x0, y, w, h, r=4, round_end=True): + """A bar from x0 rightwards: square at the baseline, rounded data end.""" + if w <= 0: + return "" + if not round_end or w < 1: + return "M%.1f %.1fh%.1fv%.1fh%.1fz" % (x0, y, w, h, -w) + r = min(r, w, h / 2) + return ("M%.1f %.1fh%.1fa%.1f %.1f 0 0 1 %.1f %.1fv%.1fa%.1f %.1f 0 0 1 %.1f %.1fh%.1fz" + % (x0, y, w - r, r, r, r, r, h - 2 * r, r, r, -r, r, -(w - r))) + + +class Row: + def __init__(self, kind, label, segments=None, value="", more="", short=None, + tip="", level=None): + self.kind = kind # group or bar + self.label = label + self.segments = segments or [] # (value, class); the outer end is rounded + self.value = value # ink, beside the bar + self.more = more # muted, after value; left out when narrow + self.short = short # replaces value when narrow + self.tip = tip + self.level = level + + +def bar_chart(svg, y, rows, vmax, ticks, tick_fmt, axis_label, ref=None, at_tip=False): + """Horizontal bars, one per row, grouped by group rows. A bar longer than + vmax is cut at the end of the scale and marked. The value stands in a + column at the right, or at the tip of its bar. Returns the height.""" + narrow = svg.w < 600 + row_h = 24 + group_h = 28 + bar_h = 12 + + def shown(r): + if narrow: + return (r.short if r.short is not None else r.value), "" + return r.value, r.more + + label_w = max([text_w(r.label, 12) for r in rows if r.kind == "bar"] + [48]) + 22 + glyph_w = 16 if any(r.level for r in rows) else 0 + widths = [] + for r in rows: + if r.kind != "bar": + continue + v, m = shown(r) + widths.append(text_w(v, 12) + (8 + text_w(m, 12) if m else 0)) + value_w = max(widths + [30]) + 12 + glyph_w + x0 = PAD + label_w + x1 = svg.w - PAD - value_w + plot_w = x1 - x0 + top = y + (18 if ref else 0) + + def sx(v): + return x0 + (0 if vmax <= 0 else plot_w * min(max(v, 0), vmax) / vmax) + + yy = top + marks = [] + for r in rows: + if r.kind == "group": + yy += group_h + marks.append('%s' % (PAD, yy - 9, esc(r.label))) + continue + cy = yy + row_h / 2 + marks.append('%s' % (PAD + 8, cy + 4, esc(r.label))) + segs = [] + pos = 0.0 + total = sum(v for v, _ in r.segments if v) + for i, (v, cls) in enumerate(r.segments): + if not v or v <= 0 or pos >= vmax: + continue + a = sx(pos) + b = sx(pos + v) + if pos > 0: + a += 2 # the surface gap between two segments + last = all(not s[0] or s[0] <= 0 for s in r.segments[i + 1:]) or pos + v >= vmax + d = hbar(a, cy - bar_h / 2, b - a, bar_h, round_end=last) + if d: + segs.append('' % (d, cls)) + pos += v + if total > vmax: + # Cut at the end of the scale: two gaps across the bar. + for dx in (16, 10): + segs.append('' + % (x1 - dx, cy + bar_h / 2 + 1, bar_h + 2)) + tip = "%s" % esc(r.tip) if r.tip else "" + marks.append('%s%s' + % (tip, "".join(segs), x0, yy, svg.w - PAD - x0, row_h)) + vx = x1 + 12 + if at_tip: + vx = (sx(min(total, vmax)) if total > 0 else x0) + 8 + if r.level: + gl = Svg("", "", svg.w) + glyph(gl, r.level, vx + 5, cy) + marks.append("".join(gl.parts)) + vx += glyph_w + v, m = shown(r) + more = '%s' % esc(m) if m else "" + marks.append('%s%s' % (vx, cy + 4, esc(v), more)) + yy += row_h + bottom = yy + 8 + for t in ticks: + x = sx(t) + svg.add('' % (x, top + 6, bottom, "base" if t == 0 else "grid")) + svg.text(x, bottom + 16, tick_fmt(t), "tk", "middle") + if ref and ref[0] <= vmax: + x = sx(ref[0]) + svg.add('' % (x, top - 2, bottom)) + svg.text(x + 4, top + 2, ref[1], "tk") + svg.add("\n".join(marks)) + svg.text(x0 + plot_w / 2, bottom + 38, axis_label, "ax", "middle") + return bottom + 38 + 22 + + +#---------------------------------------------------------------------------# +# Charts from status.json +#---------------------------------------------------------------------------# +def grouped(targets, order): + out = [] + for a in order: + ts = [t for t in targets if t.get("arch") == a] + if ts: + out.append((a, sorted(ts, key=lambda t: repo_key(t.get("repo", ""))))) + return out + + +def chart_coverage(ctx, w): + s = ctx["status"]["summary"] + svg = Svg("Coverage by repository", + "Share of each repository's upstream files that the mirror serves, " + "by file name and sha256, grouped by architecture.", w) + cap = ("of %s upstream files are in the mirror, by name and sha256. %d of %d " + "repositories have a snapshot." % (fmt_int(s.get("upstream_files")), + s.get("promoted", 0), s.get("targets", 0))) + y = header(svg, "Coverage by repository", fmt_pct(s.get("coverage_files")), cap) + groups = grouped(ctx["targets"], ctx["order"]) + items = [("box", "s%d" % ctx["slots"][a], a) for a, _ in groups] + y = legend(svg, y, items + [("box", "track", "upstream only")]) + rows = [] + for a, ts in groups: + rows.append(Row("group", a)) + for t in ts: + c = t.get("coverage") + name = t.get("target") + if not c or not c.get("of_files"): + rows.append(Row("bar", t.get("repo"), [], not_read(t), + short="-", tip="%s: %s" % (name, not_read(t)))) + continue + r = c.get("ratio_files") or 0 + rows.append(Row( + "bar", t.get("repo"), + [(r * 100, "s%d" % ctx["slots"][a]), ((1 - r) * 100, "track")], + fmt_pct(r), "%s / %s" % (fmt_int(c["files"]), fmt_int(c["of_files"])), + tip="%s: %s of %s files (%s), %s of %s GiB" % ( + name, fmt_int(c["files"]), fmt_int(c["of_files"]), fmt_pct(r), + fmt_gib(c.get("bytes")), fmt_gib(c.get("of_bytes"))))) + ticks = [0, 25, 50, 75, 100] if w >= 600 else [0, 50, 100] + h = bar_chart(svg, y, rows, 100, ticks, lambda v: "%d%%" % v, + "Share of upstream files in the mirror") + return svg.render(h), h + + +def chart_freshness(ctx, w): + status, interval, tolerance = ctx["status"], ctx["interval"], ctx["tolerance"] + svg = Svg("Snapshot age", + "Hours since each repository's promoted snapshot was fetched from " + "upstream, against the sync interval. A bar past the stale limit is " + "cut at the end of the scale.", w) + ages = [t["age_seconds"] for t in ctx["targets"] + if num(t.get("age_seconds")) and (t.get("mirror") or {}).get("state") == "promoted"] + s = status["summary"] + cap = ("oldest snapshot. Sync runs every %s. %d of %d repositories match " + "upstream now." % (fmt_interval(interval), s.get("current", 0), s.get("targets", 0))) + y = header(svg, "Snapshot age", fmt_age(max(ages)) if ages else "none", cap) + y = legend(svg, y, [ + ("glyph", "good", "matches upstream, or within 2 syncs"), + ("glyph", "warning", "late"), + ("glyph", "critical", "stale, over %s" % fmt_interval(tolerance)), + ("glyph", "none", "not mirrored yet"), + ]) + hours = [a / 3600 for a in ages] + ih = interval / 3600 + vmax = max([min(max(hours + [0]), tolerance / 3600), 2 * ih * 1.2]) + ticks, top = hour_ticks(vmax, 5 if w >= 600 else 3) + rows = [] + for a, ts in grouped(ctx["targets"], ctx["order"]): + rows.append(Row("group", a)) + for t in ts: + lvl, word = freshness(t, interval, tolerance) + age = t.get("age_seconds") + if lvl in ("none", "unknown") or not num(age): + rows.append(Row("bar", t.get("repo"), [], word, short="-", + tip="%s: %s" % (t.get("target"), word), level=lvl)) + continue + rows.append(Row("bar", t.get("repo"), [(age / 3600, "st-%s" % lvl)], + fmt_age(age), word, short=fmt_age(age), + tip="%s: fetched %s, %s ago, %s" % ( + t.get("target"), (t.get("mirror") or {}).get("fetched"), + fmt_age(age), word), level=lvl)) + h = bar_chart(svg, y, rows, top, ticks, fmt_hours, + "Time since the snapshot was fetched from upstream", + ref=(ih, "sync every %s" % fmt_interval(interval))) + return svg.render(h), h + + +def chart_size(ctx, w): + s = ctx["status"]["summary"] + svg = Svg("Size by repository", + "Compressed package bytes per repository: in the mirror, and upstream " + "only.", w) + cap = ("in the mirror, of %s GiB upstream (%s by bytes)." + % (fmt_gib(s.get("upstream_bytes")), fmt_pct(s.get("coverage_bytes")))) + y = header(svg, "Size by repository", "%s GiB" % fmt_gib(s.get("mirrored_bytes")), cap) + groups = grouped(ctx["targets"], ctx["order"]) + items = [("box", "s%d" % ctx["slots"][a], a) for a, _ in groups] + y = legend(svg, y, items + [("box", "track", "upstream only")]) + gib = 1073741824 + vmax = max([((t.get("coverage") or {}).get("of_bytes") or 0) / gib for t in ctx["targets"]] + [0.001]) + ticks, top = nice_ticks(vmax, 5 if w >= 600 else 3) + rows = [] + for a, ts in groups: + rows.append(Row("group", a)) + for t in ts: + c = t.get("coverage") + if not c or c.get("of_bytes") is None: + rows.append(Row("bar", t.get("repo"), [], not_read(t), short="-", + tip="%s: %s" % (t.get("target"), not_read(t)))) + continue + mb = (c.get("bytes") or 0) / gib + nb = (c.get("of_bytes") or 0) / gib + rows.append(Row("bar", t.get("repo"), + [(mb, "s%d" % ctx["slots"][a]), (max(nb - mb, 0), "track")], + "%s GiB" % fmt_gib(c.get("bytes")), + "of %s" % fmt_gib(c.get("of_bytes")), + short=fmt_gib(c.get("of_bytes")), + tip="%s: %s of %s GiB in the mirror" % ( + t.get("target"), fmt_gib(c.get("bytes")), fmt_gib(c.get("of_bytes"))))) + h = bar_chart(svg, y, rows, top, ticks, lambda v: "%g" % v, "GiB of compressed packages", + at_tip=True) + return svg.render(h), h + + +#---------------------------------------------------------------------------# +# History +#---------------------------------------------------------------------------# +def history_series(history, now, days=30): + """{arch: [(t, coverage ratio or None, oldest age or None)]}, last DAYS.""" + start = now - days * 86400 + out = {} + for line in history: + t = parse_time(line.get("t")) + if t is None or t < start or t > now + 3600: + continue + per = {} + for name, x in (line.get("x") or {}).items(): + if not isinstance(x, dict) or "-" not in name: + continue + p = per.setdefault(name.split("-", 1)[0], [0, 0, None, False]) + if num(x.get("n")) and num(x.get("m")): + p[0] += x["n"] + p[1] += x["m"] + p[3] = True + if num(x.get("a")): + p[2] = x["a"] if p[2] is None else max(p[2], x["a"]) + for arch, (n, m, age, have) in per.items(): + out.setdefault(arch, []).append((t, (m / n) if have and n > 0 else None, age)) + for arch in out: + out[arch].sort() + return out + + +def bucket(points, step, how): + """One point per STEP seconds: the last value, or the largest. A point + with no value is left out.""" + if step <= 0: + return [(t, v) for t, v in points if v is not None] + keep = {} + for t, v in points: + if v is None: + continue + k = int(t // step) + if k not in keep: + keep[k] = (t, v) + elif how == "max": + keep[k] = (max(keep[k][0], t), max(keep[k][1], v)) + else: + keep[k] = max(keep[k], (t, v)) + return [keep[k] for k in sorted(keep)] + + +def time_ticks(x0, x1, plot_w): + span = x1 - x0 + most = max(2, int(plot_w // 72)) + steps = [3600 * h for h in (3, 6, 12, 24, 48, 120, 240)] + step = next((s for s in steps if span / s <= most), steps[-1]) + fmt = "%H:%M" if step < 86400 else "%b %d" + t = math.ceil(x0 / step) * step + ticks = [] + while t <= x1: + d = utc(t) + label = d.strftime(fmt) + if fmt == "%H:%M" and d.hour == 0: + label = d.strftime("%b %d") + ticks.append((t, label.replace(" 0", " "))) + t += step + return ticks + + +def line_chart(svg, y, series, slots, y_ticks, ty, y_fmt, y_label, x0, x1, ref=None): + """One line per series. ty maps a value to 0..1 up the plot.""" + items = [("line", "l%d" % slots.get(a, 0), a) for a, _ in series] + if ref: + items.append(("line", "ref", ref[1])) + y = legend(svg, y, items) + end_w = max([text_w("%s %s" % (a, y_fmt(v, True)), 12) + for a, pts in series for _, v in pts[-1:]] + [40]) + plot_l = PAD + 46 + plot_r = svg.w - PAD - end_w - 20 + top = y + 10 + bottom = top + (200 if svg.w >= 600 else 170) + + def sx(t): + return plot_l + (plot_r - plot_l) * (t - x0) / max(x1 - x0, 1) + + def sy(v): + return bottom - (bottom - top) * min(max(ty(v), 0.0), 1.0) + + for v in y_ticks: + svg.add('' % (plot_l, sy(v), plot_r, + "base" if v == y_ticks[0] else "grid")) + svg.text(plot_l - 8, sy(v) + 4, y_fmt(v), "tk", "end") + for t, label in time_ticks(x0, x1, plot_r - plot_l): + svg.add('' % (sx(t), bottom)) + svg.text(sx(t), bottom + 18, label, "tk", "middle") + if ref: + svg.add('' % (plot_l, sy(ref[0]), plot_r)) + + ends = [] + for arch, pts in series: + cls = slots.get(arch, 0) + if not pts: + continue + steps = [b[0] - a[0] for a, b in zip(pts, pts[1:])] + gap = max(4 * (median(steps) or 3600), 2 * 3600) + d = [] + prev = None + for t, v in pts: + d.append("%s%.1f %.1f" % ("M" if prev is None or t - prev > gap else "L", sx(t), sy(v))) + prev = t + svg.add('%s' % ("".join(d), cls, esc(arch))) + lt, lv = pts[-1] + ends.append([sy(lv), arch, lv, sx(lt), sy(lv)]) + # End labels 15 px apart at least, each with a leader line when it moved. + ends.sort() + for i in range(1, len(ends)): + if ends[i][0] < ends[i - 1][0] + 15: + ends[i][0] = ends[i - 1][0] + 15 + if ends and ends[-1][0] > bottom + 4: + shift = ends[-1][0] - bottom - 4 + for e in ends: + e[0] -= shift + lx = plot_r + 16 + for ly, arch, v, ex, ey in ends: + if abs(ly - ey) > 2: + svg.add('' % (ex + 6, ey, lx - 4, ly)) + for ly, arch, v, ex, ey in ends: + svg.add('%s %s' + % (ex, ey, slots.get(arch, 0), esc(arch), esc(y_fmt(v, True)))) + svg.add('%s%s' + % (lx, ly + 4, esc(arch), esc(y_fmt(v, True)))) + if not ends: + svg.text((plot_l + plot_r) / 2, (top + bottom) / 2, "No history yet", "note", "middle") + svg.text((plot_l + plot_r) / 2, bottom + 42, "Time, UTC", "ax", "middle") + svg.add('%s' + % (PAD + 6, (top + bottom) / 2, esc(y_label))) + return bottom + 42 + 22 + + +def history_window(ctx): + now = ctx["now"] + series = ctx["series"] + times = [p[0] for pts in series.values() for p in pts] + x1 = now + x0 = max(min(times), now - 30 * 86400) if times else now - 86400 + if x1 - x0 < 86400: + x0 = x1 - 86400 + if (x1 - x0) / 86400 >= 29.5: + span = "last 30 days" + elif times: + span = "since %s" % utc(min(times)).strftime("%b %d, %H:%M UTC").replace(" 0", " ") + else: + span = "no history yet" + # More than 2 days: one point per 6 hours, so a line reads as a trend. + step = 6 * 3600 if x1 - x0 > 2 * 86400 else 0 + return x0, x1, span, step + + +def chart_history_coverage(ctx, w): + x0, x1, span, step = history_window(ctx) + s = ctx["status"]["summary"] + svg = Svg("Coverage, last 30 days", + "Share of upstream files in the mirror per architecture over time. " + "With more than 2 days of history, one point per 6 hours, the last " + "value of each.", w) + y = header(svg, "Coverage, last 30 days", fmt_pct(s.get("coverage_files")), + "of upstream files in the mirror now. One line per architecture, %s." % span) + series = [] + for a in ctx["hist_order"]: + pts = [(t, None if c is None else c * 100) for t, c, _ in ctx["series"][a]] + series.append((a, bucket(pts, step, "last"))) + series = [(a, p) for a, p in series if p] + + def pct(v, end=False): + return fmt_pct(v / 100) if end else "%d%%" % v + + ticks = [0, 25, 50, 75, 100] if w >= 600 else [0, 50, 100] + h = line_chart(svg, y, series, ctx["slots"], ticks, lambda v: v / 100, pct, + "Share of upstream files", x0, x1) + return svg.render(h), h + + +AGE_TICKS = [0.25, 0.5, 1, 3, 6, 12, 24, 72, 168, 720] + + +def chart_history_age(ctx, w): + x0, x1, span, step = history_window(ctx) + s = ctx["status"]["summary"] + ih = ctx["interval"] / 3600 + svg = Svg("Snapshot age, last 30 days", + "Age of the oldest promoted snapshot per architecture over time, on a " + "log scale, against the sync interval. With more than 2 days of " + "history, one point per 6 hours, the largest of each.", w) + oldest = s.get("oldest_age_seconds") + y = header(svg, "Snapshot age, last 30 days", fmt_age(oldest) if num(oldest) else "none", + "oldest snapshot now. One line per architecture, %s, log scale." % span) + series = [] + for a in ctx["hist_order"]: + pts = [(t, None if g is None else g / 3600) for t, _, g in ctx["series"][a]] + series.append((a, bucket(pts, step, "max"))) + series = [(a, p) for a, p in series if p] + vals = [v for _, pts in series for _, v in pts] + hi_v = max(vals + [2 * ih]) + lo = 0.5 if min(vals + [ih]) >= 0.5 else 0.25 + hi = next((t for t in AGE_TICKS if t >= hi_v), AGE_TICKS[-1]) + ticks = [t for t in AGE_TICKS if lo <= t <= hi] + if w < 600 and len(ticks) > 4: + ticks = [t for i, t in enumerate(ticks) if i % 2 == 0 or t == hi] + + def ty(v): + return (math.log(max(v, lo)) - math.log(lo)) / (math.log(hi) - math.log(lo)) + + def hours(v, end=False): + return fmt_age(v * 3600) if end else fmt_hours(v) + + h = line_chart(svg, y, series, ctx["slots"], ticks, ty, hours, + "Age, log scale", x0, x1, + ref=(ih, "sync every %s" % fmt_interval(ctx["interval"]))) + return svg.render(h), h + + +#---------------------------------------------------------------------------# +# Bench +#---------------------------------------------------------------------------# +def bench_cases(bench): + out = {} + for c in (bench or {}).get("cases") or []: + if isinstance(c, dict): + out[(c.get("path"), c.get("op"))] = c + return out + + +def bench_totals(bench): + cases = bench_cases(bench) + totals = {} + for p in PATH_ORDER: + vals = [] + for op in OP_ORDER: + c = cases.get((p, op)) + if c and c.get("ok") is True and num(c.get("seconds")): + vals.append(c["seconds"]) + else: + vals = None + break + totals[p] = sum(vals) if vals else None + return totals + + +def op_label(op, c): + files = c.get("files") if c else None + n = " %s" % fmt_int(files) if isinstance(files, int) else "" + return {"sy": "pacman -Sy, the databases", + "sw-small": "pacman -Sw, the%s smallest packages" % n, + "sw-large": "pacman -Sw, the%s largest packages" % n}.get(op, op) + + +def chart_speed(ctx, w): + bench = ctx["bench"] + svg = Svg("pacman speed by path", + "Median seconds of pacman -Sy and -Sw through the official mirror, the " + "Worker, and the oci wrapper with and without prefetch.", w) + # The oci wrapper is the path this mirror ships: it has the colour, and + # the other paths are the grey it is measured against. + items = [("box", "s1", "oci wrapper"), ("box", "s0", "official mirror, Worker, oci serial")] + if not bench: + y = header(svg, "pacman speed by path", "no data", + "No benchmark yet. The status workflow runs scripts/bench-consumer once a day.") + y = legend(svg, y, items) + top = y + 10 + bottom = top + 110 + svg.add('' % (PAD + 140, top, bottom)) + svg.add('' % (PAD + 140, bottom, svg.w - PAD - 60)) + svg.text((PAD + 140 + svg.w - PAD - 60) / 2, (top + bottom) / 2 + 4, + "No measurement yet", "note", "middle") + svg.text((PAD + 140 + svg.w - PAD - 60) / 2, bottom + 34, "Seconds, median of the rounds", + "ax", "middle") + h = bottom + 56 + return svg.render(h), h + totals = bench_totals(bench) + cases = bench_cases(bench) + rounds = max([len(c.get("rounds") or []) for c in cases.values()] + [0]) + oci = totals.get("oci") + cap = ("for -Sy and -Sw through the oci wrapper, median of %d rounds. Official " + "mirror: %s. Measured %s on %s." % (rounds, fmt_secs(totals.get("official")), + str(bench.get("date", "?"))[:10], bench.get("host", "?"))) + y = header(svg, "pacman speed by path", fmt_secs(oci) if oci is not None else "failed", cap) + y = legend(svg, y, items) + vals = [c["seconds"] for c in cases.values() if num(c.get("seconds"))] + ticks, top = nice_ticks(max(vals + [0.1]), 5 if w >= 600 else 3) + rows = [] + for op in OP_ORDER: + some = [cases.get((p, op)) for p in PATH_ORDER] + if not any(some): + continue + rows.append(Row("group", op_label(op, next((c for c in some if c), None)))) + for p in PATH_ORDER: + c = cases.get((p, op)) + cls = "s1" if p == "oci" else "s0" + if not c: + rows.append(Row("bar", PATH_LABEL[p], [], "not measured", short="-")) + elif c.get("ok") is not True or not num(c.get("seconds")): + rows.append(Row("bar", PATH_LABEL[p], [], "failed", + tip="%s, %s: failed" % (PATH_LABEL[p], op))) + else: + rows.append(Row("bar", PATH_LABEL[p], [(c["seconds"], cls)], fmt_secs(c["seconds"]), + tip="%s, %s: median %s of %s; %s files, %.1f MiB" % ( + PATH_LABEL[p], op, fmt_secs(c["seconds"]), + ", ".join(fmt_secs(r) for r in (c.get("rounds") or []) if num(r)), + fmt_int(c.get("files")), (c.get("bytes") or 0) / 1048576))) + h = bar_chart(svg, y, rows, top, ticks, lambda v: "%g s" % v, + "Seconds, median of the rounds; shorter is faster", at_tip=True) + return svg.render(h), h + + +#---------------------------------------------------------------------------# +# Badges +#---------------------------------------------------------------------------# +BADGE_LIGHT = {"good": "#1a7f37", "warning": "#9a6700", "critical": "#cf222e", + "none": "#6e7781", "unknown": "#6e7781", "info": "#0969da"} +BADGE_DARK = {"good": "#238636", "warning": "#9e6a03", "critical": "#da3633", + "none": "#6e7681", "unknown": "#6e7681", "info": "#1f6feb"} + + +def badge(label, message, level): + lw = round(badge_w(label) + 12) + mw = round(badge_w(message) + 12) + w = lw + mw + title = "%s: %s" % (label, message) + style = (".lab{fill:#555}.msg{fill:%s}text{font-family:%s;font-size:11px;fill:#fff}" + "@media (prefers-color-scheme:dark){.lab{fill:#3d444d}.msg{fill:%s}}" + % (BADGE_LIGHT[level], FONT, BADGE_DARK[level])) + return ( + '\n%s\n\n' + '\n' + '' + '\n' + '%s\n' + '%s\n\n' + ) % (w, w, esc(title), esc(title), style, w, lw, lw, mw, + lw / 2, esc(label), lw + mw / 2, esc(message)) + + +def target_badge(t, interval, tolerance): + lvl = target_level(t, interval, tolerance) + if lvl == "none": + return badge(t["target"], "not mirrored", "none") + if lvl == "unknown": + return badge(t["target"], "unknown", "unknown") + return badge(t["target"], "%s · %s" % (fmt_pct(cov_ratio(t)), fmt_age(t.get("age_seconds"))), lvl) + + +def group_badge(label, targets, interval, tolerance, count=False): + promoted = [t for t in targets if (t.get("mirror") or {}).get("state") == "promoted"] + if not promoted: + return badge(label, "not mirrored", "none") + n = sum(((t.get("coverage") or {}).get("of_files") or 0) for t in targets) + m = sum(((t.get("coverage") or {}).get("files") or 0) for t in targets) + worst = max((target_level(t, interval, tolerance) for t in targets), key=lambda lv: LEVEL_RANK[lv]) + if worst in ("none", "unknown"): + worst = "warning" + ages = [t["age_seconds"] for t in promoted if num(t.get("age_seconds"))] + if count: + msg = "%s · %d/%d repos" % (fmt_pct(m / n if n else None), len(promoted), len(targets)) + else: + msg = "%s · %s" % (fmt_pct(m / n if n else None), fmt_age(max(ages) if ages else None)) + return badge(label, msg, worst) + + +def speed_badge(bench): + if not bench: + return badge("pacman speed", "no data", "none") + tot = bench_totals(bench) + if tot.get("oci") is None: + return badge("pacman speed", "oci failed", "critical") + return badge("pacman speed", "oci %s · official %s" % (fmt_secs(tot["oci"]), fmt_secs(tot.get("official"))), + "info") + + +#---------------------------------------------------------------------------# +# The page +#---------------------------------------------------------------------------# +PAGE_CSS = """ +:root{color-scheme:light dark;--page:#f6f6f3;--surface:#fcfcfb;--ink:#0b0b0b; +--ink2:#52514e;--muted:#6f6e69;--line:#e6e5df;--ring:rgba(11,11,11,.10); +--code:#efeee9;--link:#1c5cab;--good:#0ca30c;--warning:#fab219;--critical:#d03b3b; +--none:#898781} +@media (prefers-color-scheme:dark){:root{--page:#0d0d0d;--surface:#1a1a19; +--ink:#ffffff;--ink2:#c3c2b7;--muted:#9a9890;--line:#2c2c2a; +--ring:rgba(255,255,255,.10);--code:#262624;--link:#86b6ef}} +*{box-sizing:border-box} +html{-webkit-text-size-adjust:100%} +body{margin:0;background:var(--page);color:var(--ink);font:15px/1.55 FONT} +a{color:var(--link)} +.wrap{max-width:1680px;margin:0 auto;padding:28px 16px 56px} +.top{display:flex;flex-wrap:wrap;gap:12px 24px;align-items:flex-end; +justify-content:space-between;margin-bottom:24px} +h1{font-size:28px;line-height:1.2;margin:0 0 6px;font-weight:650;letter-spacing:-.01em} +h2{font-size:19px;margin:44px 0 14px;font-weight:620} +p{margin:0 0 10px;max-width:76ch} +.lede{color:var(--ink2)} +.meta{color:var(--muted);font-size:13px} +.badges{display:flex;flex-wrap:wrap;gap:8px} +.badges img{display:block;height:20px;width:auto} +.hero{display:grid;grid-template-columns:minmax(0,1fr);gap:20px 32px; +background:var(--surface);border:1px solid var(--ring);border-radius:16px;padding:16px} +@media (min-width:600px){.hero{padding:24px}} +@media (min-width:900px){.hero{grid-template-columns:minmax(0,4fr) minmax(0,7fr);align-items:center}} +.big{font-size:56px;line-height:1;font-weight:650;letter-spacing:-.03em;margin:0} +@media (min-width:600px){.big{font-size:72px}} +.big-cap{color:var(--ink2);margin:12px 0 0} +.tiles{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:10px} +@media (min-width:560px){.tiles{grid-template-columns:repeat(3,minmax(0,1fr))}} +.tile{border:1px solid var(--ring);border-radius:12px;padding:12px 14px;min-width:0} +.tile .k{font-size:13px;color:var(--ink2)} +.tile .v{font-size:22px;font-weight:620;line-height:1.3;overflow-wrap:anywhere} +.tile .s{font-size:12px;color:var(--muted)} +.use{display:grid;grid-template-columns:minmax(0,1fr);gap:16px} +@media (min-width:900px){.use{grid-template-columns:repeat(2,minmax(0,1fr))}} +.use h3{font-size:15px;margin:0 0 6px;font-weight:620} +.charts{display:grid;grid-template-columns:repeat(auto-fit,minmax(min(100%,720px),1fr)); +gap:20px;justify-items:center} +figure{margin:0;width:100%;max-width:860px;min-width:0} +figure img{display:block;width:100%;height:auto} +details{margin:6px 4px 0} +summary{cursor:pointer;color:var(--ink2);font-size:13px;padding:2px 0} +.scroll{overflow-x:auto;-webkit-overflow-scrolling:touch;border:1px solid var(--ring); +border-radius:12px;background:var(--surface)} +table{border-collapse:collapse;width:100%;font-size:13px} +th,td{text-align:left;padding:8px 12px;border-bottom:1px solid var(--line); +white-space:nowrap;vertical-align:baseline} +tr:last-child td{border-bottom:0} +th{font-weight:600;color:var(--ink2)} +td.n,th.n{text-align:right;font-variant-numeric:tabular-nums} +td details{margin:0;display:inline-block} +td summary{padding:0} +.st{display:inline-flex;align-items:center;gap:7px} +.dot{width:9px;height:9px;border-radius:50%;display:inline-block;flex:none} +.dot.good{background:var(--good)} +.dot.warning{width:0;height:0;border-radius:0;border-left:5px solid transparent; +border-right:5px solid transparent;border-bottom:9px solid var(--warning)} +.dot.critical{border-radius:1px;background:var(--critical);transform:rotate(45deg) scale(.82)} +.dot.none,.dot.unknown{border:1.5px solid var(--none)} +code,pre{font-family:MONO;font-size:13px} +code{background:var(--code);padding:1px 5px;border-radius:5px} +pre{background:var(--code);padding:14px 16px;border-radius:12px;overflow-x:auto;margin:0} +pre code{background:none;padding:0} +.missing{white-space:normal;color:var(--ink2);font-size:12px;max-width:60ch;margin-top:4px} +footer{margin-top:48px;color:var(--muted);font-size:13px} +""".replace("FONT", FONT).replace("MONO", MONO) + + +def state_cell(level, word): + return '%s' % ( + esc(level), esc(word)) + + +def table(head, rows, nums=()): + h = "".join('%s' % esc(x) if i in nums else "%s" % esc(x) + for i, x in enumerate(head)) + body = "".join("%s" % "".join('%s' % c if i in nums else "%s" % c + for i, c in enumerate(r)) for r in rows) + return "%s%s
" % (h, body) + + +def page(ctx, charts, badges): + status, targets = ctx["status"], ctx["targets"] + interval, tolerance, bench = ctx["interval"], ctx["tolerance"], ctx["bench"] + s = status["summary"] + wk = status.get("worker") or {} + worker_url = status.get("worker_url") or "" + site = status.get("site_url") or "" + source = status.get("source") or "" + ns = status.get("namespace") or "" + registry = status.get("registry") or "ghcr.io" + + def tile(k, v, sub="", raw=False): + return ('
%s
%s
' + '
%s
' % (esc(k), v if raw else esc(v), esc(sub))) + + oldest = s.get("oldest_age_seconds") + if wk.get("ok") is True: + wv, wl = "ok", "good" + elif wk.get("ok") is False: + wv, wl = "failing", "critical" + else: + wv, wl = "not judged", "unknown" + ghcr_ms = fmt_secs((status.get("ghcr") or {}).get("median_manifest_seconds")) + wsub = "median %s; GHCR manifest %s" % (fmt_secs(wk.get("median_seconds")), ghcr_ms) + if wk.get("failed_runs"): + wsub = "%d run(s) in a row; median %s" % (wk["failed_runs"], fmt_secs(wk.get("median_seconds"))) + totals = bench_totals(bench) if bench else {} + if totals.get("oci") is not None: + last = tile("pacman, oci wrapper", fmt_secs(totals["oci"]), + "official mirror %s" % fmt_secs(totals.get("official"))) + else: + last = tile("GHCR manifest", ghcr_ms, "median, read with no credential") + tiles = "".join([ + tile("Repositories with a snapshot", "%d of %d" % (s.get("promoted", 0), s.get("targets", 0)), + "%d match upstream now" % s.get("current", 0)), + tile("Files in the mirror", fmt_int(s.get("mirrored_files")), + "of %s upstream" % fmt_int(s.get("upstream_files"))), + tile("Size in the mirror", "%s GiB" % fmt_gib(s.get("mirrored_bytes")), + "of %s GiB, %s" % (fmt_gib(s.get("upstream_bytes")), fmt_pct(s.get("coverage_bytes")))), + tile("Oldest snapshot", fmt_age(oldest) if num(oldest) else "none", + "sync runs every %s" % fmt_interval(interval)), + tile("Worker", state_cell(wl, wv), wsub, raw=True), + last, + ]) + + def fig(name, alt, tbl): + _, hw = charts[name]["wide"] + _, hn = charts[name]["narrow"] + return ('
' + '' + '%(alt)s' + '
Table
%(t)s
' + % {"n": name, "nw": NARROW, "nh": hn, "ww": WIDE, "wh": hw, "alt": esc(alt), "t": tbl}) + + cov_rows, size_rows, fresh_rows = [], [], [] + for t in targets: + c = t.get("coverage") or {} + lvl, word = freshness(t, interval, tolerance) + cov_rows.append([esc(t["target"]), esc(fmt_int(c.get("files"))), esc(fmt_int(c.get("of_files"))), + esc(fmt_pct(c.get("ratio_files"))), esc(fmt_int(c.get("missing")))]) + size_rows.append([esc(t["target"]), esc(fmt_gib(c.get("bytes"))), esc(fmt_gib(c.get("of_bytes"))), + esc(fmt_pct(c.get("ratio_bytes")))]) + fresh_rows.append([esc(t["target"]), state_cell(lvl, word), + esc(fmt_time((t.get("mirror") or {}).get("fetched"))), + esc(fmt_age(t.get("age_seconds"))), esc(fmt_age(t.get("lag_seconds")))]) + hist_rows = [] + for a in ctx["hist_order"]: + pts = ctx["series"][a] + covs = [p for p in pts if p[1] is not None] + ages = [p[2] for p in pts if p[2] is not None] + hist_rows.append([esc(a), esc(utc(pts[0][0]).strftime("%Y-%m-%d %H:%M")), + esc(fmt_pct(covs[0][1]) if covs else "-"), esc(fmt_pct(covs[-1][1]) if covs else "-"), + esc(fmt_age(max(ages)) if ages else "-"), esc(fmt_int(len(pts)))]) + hist_table = table(["architecture", "first point", "coverage then", "coverage now", + "largest age", "points"], hist_rows, (2, 3, 4, 5)) + speed_rows = [] + cases = bench_cases(bench) + for op in OP_ORDER: + for p in PATH_ORDER: + c = cases.get((p, op)) + if c: + speed_rows.append([esc(op), esc(PATH_LABEL[p]), + esc(fmt_secs(c.get("seconds")) if c.get("ok") is True else "failed"), + esc(", ".join(fmt_secs(r) for r in (c.get("rounds") or []) if num(r))), + esc(fmt_int(c.get("files"))), + esc("%.1f" % ((c.get("bytes") or 0) / 1048576))]) + figs = "".join([ + fig("coverage", "Coverage by repository: %s of upstream files are in the mirror." + % fmt_pct(s.get("coverage_files")), + table(["repository", "in the mirror", "upstream", "share", "missing"], cov_rows, (1, 2, 3, 4))), + fig("freshness", "Snapshot age per repository, against the sync interval.", + table(["repository", "state", "fetched, UTC", "age", "lag"], fresh_rows, (3, 4))), + fig("history-coverage", "Coverage per architecture over the last 30 days.", hist_table), + fig("history-age", "Oldest snapshot age per architecture over the last 30 days.", hist_table), + fig("size", "Size by repository: %s GiB of %s GiB are in the mirror." + % (fmt_gib(s.get("mirrored_bytes")), fmt_gib(s.get("upstream_bytes"))), + table(["repository", "GiB in the mirror", "GiB upstream", "share"], size_rows, (1, 2, 3))), + fig("speed", "pacman speed through each path, from the daily benchmark.", + table(["operation", "path", "median", "rounds", "files", "MiB"], speed_rows, (2, 4, 5)) + if speed_rows else "

No benchmark yet.

"), + ]) + + repo_rows = [] + for t in targets: + c = t.get("coverage") or {} + u = t.get("upstream") or {} + lvl, word = target_word(t, interval, tolerance) + names = c.get("missing_names") or [] + miss = esc(fmt_int(c.get("missing"))) + if names: + more = (c.get("missing") or 0) - len(names) + miss = ('
%s
%s%s
' + % (miss, ", ".join("%s" % esc(n) for n in names), + esc(", and %s more" % fmt_int(more)) if more > 0 else "")) + probes = [] + for key in ("db", "package"): + p = (t.get("worker") or {}).get(key) + if p: + probes.append("%s %s%s" % (key, p.get("code"), " failed" if p.get("ok") is False else "")) + repo_rows.append([ + "%s" % esc(t["target"]), state_cell(lvl, word), + esc(fmt_pct(c.get("ratio_files"))), esc(fmt_int(c.get("files"))), esc(fmt_int(c.get("of_files"))), + miss, esc(fmt_gib(c.get("of_bytes"))), esc(fmt_age(t.get("age_seconds"))), + esc(fmt_age(t.get("lag_seconds"))), esc(fmt_time((t.get("mirror") or {}).get("fetched"))), + esc(u.get("server") or u.get("error") or "-"), esc(", ".join(probes) or "-"), + '%s badge' % (esc(t["target"]), esc(t["target"])), + ]) + repos = table(["repository", "state", "coverage", "files", "upstream", "missing", "GiB", "age", + "lag", "fetched, UTC", "upstream server", "Worker", "badge"], + repo_rows, (2, 3, 4, 5, 6, 7, 8)) + + probe_rows = [] + for t in targets: + for key in ("db", "package"): + p = (t.get("worker") or {}).get(key) + if not p: + continue + ok = p.get("ok") + lvl = "good" if ok is True else ("critical" if ok is False else "unknown") + probe_rows.append([ + esc(t["target"]), esc(key), "%s" % esc(p.get("url")), + esc(p.get("code")), esc(p.get("expect") if p.get("expect") is not None else "-"), + esc(fmt_secs(p.get("seconds"))), esc(p.get("location") or "-"), + state_cell(lvl, "ok" if ok is True else (p.get("error") or "not judged"))]) + probes_html = (table(["repository", "probe", "URL", "HTTP", "expected", "time", "redirect to", + "result"], probe_rows, (3, 4, 5)) + if probe_rows else "

The Worker was not probed.

") + + badge_rows = [] + for name in badges: + badge_rows.append(['%s badge' % (esc(name), esc(name)), + "%sbadges/%s.svg" % (esc(site), esc(name))]) + + use = "" + if worker_url: + use = ( + '

Use the mirror

' + '

Through the Worker, no client

pacman downloads as usual. The Worker ' + 'answers each request with a redirect to GHCR.

[core]\nServer = %(w)s/$arch/$repo\n'
+            'Include = /etc/pacman.d/mirrorlist
' + '

Through the oci wrapper

pacman reads the registry with ' + 'client/pacman-oci as its XferCommand.

[options]\n'
+            'XferCommand = /usr/local/bin/pacman-oci %%o %%u\n\n[core]\n'
+            'Server = oci://%(r)s/%(ns)s/$arch/$repo\nInclude = /etc/pacman.d/mirrorlist
' + '
' % {"w": esc(worker_url), "r": esc(registry), "ns": esc(ns)}) + + return """ + + + + + +mirror-archlinux status + + + + +
+
+
+

mirror-archlinux status

+

Arch Linux repositories for %(narch)d architectures, mirrored to GitHub Container Registry and served to pacman.

+

Measured %(gen)s. Source: %(source_text)s.

+
+
all repositories badgepacman speed badge
+
+
+
+

%(cov)s

+

of upstream files are in the mirror, counted by file name and sha256 over %(n)d repositories.

+
+
%(tiles)s
+
+%(use)s +

Charts

+
%(figs)s
+

Repositories

+
%(repos)s
+

Worker probes

+

Each probe is one GET with no redirect followed. A repository with a snapshot must answer 302 to https://*.githubusercontent.com. A repository with no snapshot must answer 404. Two failed runs in a row open the worker-down issue.

+
%(probes)s
+

Badges

+
%(badges)s
+
+

Coverage counts an upstream file when the mirror's database names the same file name with the same sha256. Age is the time since the snapshot was fetched from upstream. Lag is the newest upstream build date minus the newest build date in the mirror. A snapshot that matches upstream is current at any age.

+

Data: status.json, history.json, bench.json. Written by scripts/status-collect and scripts/status-render.

+
+
+ + +""" % { + "css": PAGE_CSS, "gen": esc(fmt_time(status.get("generated")) + " UTC"), + "source": esc(source), "source_text": esc(source.replace("https://", "")), + "narch": len(ctx["order"]), "cov": esc(fmt_pct(s.get("coverage_files"))), + "n": s.get("targets", 0), "tiles": tiles, "use": use, "figs": figs, "repos": repos, + "probes": probes_html, "badges": table(["badge", "URL"], badge_rows), + } + + +#---------------------------------------------------------------------------# +# Input +#---------------------------------------------------------------------------# +def read_json(path, what, required): + if not os.path.exists(path): + if required: + die("%s: no such file" % path, "it is %s" % what, "reproduce: ls -l %s" % path) + return None + try: + with open(path, encoding="utf-8") as f: + return json.load(f) + except (OSError, ValueError) as e: + if required: + die("%s does not read as JSON: %s" % (path, e), + "reproduce: python3 -I -m json.tool %s" % path) + warn("%s does not read as JSON, so it is left out: %s" % (path, e)) + return None + + +def read_history(path): + out = [] + if not os.path.exists(path): + return out + with open(path, encoding="utf-8") as f: + for i, line in enumerate(f, 1): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except ValueError as e: + warn("%s line %d is not JSON, so it is left out: %s" % (path, i, e)) + continue + if not isinstance(obj, dict) or parse_time(obj.get("t")) is None: + warn("%s line %d has no time t, so it is left out" % (path, i)) + continue + out.append(obj) + return out + + +def write(path, text): + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = path + ".tmp" + with open(tmp, "w", encoding="utf-8", newline="\n") as f: + f.write(text) + os.replace(tmp, path) + + +def main(argv): + if len(argv) != 3: + sys.stderr.write("usage: python3 -I scripts/status-render IN OUT_SITE\n") + return 2 + src, site = argv[1], argv[2] + status = read_json(os.path.join(src, "status.json"), "scripts/status-collect's output", True) + if not isinstance(status, dict) or status.get("schema") != 1: + die("%s/status.json is not schema 1" % src, + "reproduce: python3 -I -m json.tool %s/status.json" % src) + if not isinstance(status.get("summary"), dict): + die("%s/status.json has no summary" % src) + targets = status.get("targets") + if not isinstance(targets, list) or not targets: + die("%s/status.json names no target" % src) + for t in targets: + if not isinstance(t, dict) or not isinstance(t.get("target"), str) or "-" not in t["target"]: + die("%s/status.json has a target entry with no arch-repo name: %r" % (src, t)) + t.setdefault("arch", t["target"].split("-", 1)[0]) + t.setdefault("repo", t["target"].split("-", 1)[1]) + history = read_history(os.path.join(src, "history.jsonl")) + bench = read_json(os.path.join(src, "bench.json"), "scripts/bench-consumer's output", False) + if bench is not None and (not isinstance(bench, dict) or bench.get("schema") != 1 + or not isinstance(bench.get("cases"), list)): + warn("%s/bench.json is not schema 1 with a cases list, so it is left out" % src) + bench = None + if bench is not None and not bench["cases"]: + bench = None + + sync = status.get("sync") or {} + order, slots = arch_slots({t["arch"] for t in targets}) + now = status.get("generated_epoch") or parse_time(status.get("generated")) or 0 + series = history_series(history, now) + ctx = { + "status": status, "bench": bench, "order": order, "slots": slots, "now": now, + "targets": sorted(targets, key=lambda t: target_key(t, order)), + "interval": (sync.get("interval_minutes") or 180) * 60, + "tolerance": (sync.get("tolerance_days") or 3) * 86400, + "series": series, + "hist_order": [a for a in order if a in series] + sorted(a for a in series if a not in order), + } + + builders = [("coverage", chart_coverage), ("freshness", chart_freshness), + ("size", chart_size), ("history-coverage", chart_history_coverage), + ("history-age", chart_history_age), ("speed", chart_speed)] + charts = {} + for name, build in builders: + charts[name] = {"wide": build(ctx, WIDE), "narrow": build(ctx, NARROW)} + write(os.path.join(site, "charts", name + ".svg"), charts[name]["wide"][0]) + write(os.path.join(site, "charts", "narrow", name + ".svg"), charts[name]["narrow"][0]) + + interval, tolerance = ctx["interval"], ctx["tolerance"] + badges = [] + for t in ctx["targets"]: + write(os.path.join(site, "badges", t["target"] + ".svg"), target_badge(t, interval, tolerance)) + badges.append(t["target"]) + for a in order: + ts = [t for t in ctx["targets"] if t["arch"] == a] + write(os.path.join(site, "badges", a + ".svg"), group_badge(a, ts, interval, tolerance)) + badges.append(a) + write(os.path.join(site, "badges", "all.svg"), + group_badge("mirror", ctx["targets"], interval, tolerance, count=True)) + badges.append("all") + write(os.path.join(site, "badges", "speed.svg"), speed_badge(bench)) + badges.append("speed") + + write(os.path.join(site, "index.html"), page(ctx, charts, badges)) + write(os.path.join(site, "status.json"), json.dumps(status, indent=1) + "\n") + write(os.path.join(site, "history.json"), json.dumps(history, separators=(",", ":")) + "\n") + write(os.path.join(site, "bench.json"), + json.dumps(bench if bench is not None else {"schema": 1, "cases": []}, indent=1) + "\n") + sys.stderr.write("%s: %d chart(s) in 2 widths, %d badge(s), %d history point(s), bench %s, in %s\n" + % (PROG, len(charts), len(badges), len(history), + "present" if bench else "absent", site)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/fixtures/status/bench.json b/tests/fixtures/status/bench.json new file mode 100644 index 0000000..c758335 --- /dev/null +++ b/tests/fixtures/status/bench.json @@ -0,0 +1,20 @@ +{ + "schema": 1, + "date": "2026-10-14T05:52:10Z", + "host": "fixture runner", + "pacman": "7.1.0", + "cases": [ + {"path": "official", "op": "sy", "seconds": 1.42, "rounds": [1.51, 1.42, 1.38], "files": 2, "bytes": 8961000, "ok": true}, + {"path": "official", "op": "sw-small", "seconds": 4.85, "rounds": [5.02, 4.85, 4.71], "files": 40, "bytes": 1203000, "ok": true}, + {"path": "official", "op": "sw-large", "seconds": 9.31, "rounds": [9.88, 9.31, 9.12], "files": 3, "bytes": 512700000, "ok": true}, + {"path": "worker", "op": "sy", "seconds": 1.61, "rounds": [1.70, 1.61, 1.59], "files": 2, "bytes": 8961000, "ok": true}, + {"path": "worker", "op": "sw-small", "seconds": 6.12, "rounds": [6.30, 6.12, 6.04], "files": 40, "bytes": 1203000, "ok": true}, + {"path": "worker", "op": "sw-large", "seconds": 8.74, "rounds": [9.10, 8.74, 8.66], "files": 3, "bytes": 512700000, "ok": true}, + {"path": "oci", "op": "sy", "seconds": 1.18, "rounds": [1.25, 1.18, 1.16], "files": 2, "bytes": 8961000, "ok": true}, + {"path": "oci", "op": "sw-small", "seconds": 3.96, "rounds": [4.10, 3.96, 3.90], "files": 40, "bytes": 1203000, "ok": true}, + {"path": "oci", "op": "sw-large", "seconds": 8.12, "rounds": [8.40, 8.12, 8.05], "files": 3, "bytes": 512700000, "ok": true}, + {"path": "oci-serial", "op": "sy", "seconds": 1.44, "rounds": [1.52, 1.44, 1.40], "files": 2, "bytes": 8961000, "ok": true}, + {"path": "oci-serial", "op": "sw-small", "seconds": 24.6, "rounds": [25.1, 24.6, 24.3], "files": 40, "bytes": 1203000, "ok": true}, + {"path": "oci-serial", "op": "sw-large", "seconds": 9.05, "rounds": [9.40, 9.05, 8.98], "files": 3, "bytes": 512700000, "ok": true} + ] +} diff --git a/tests/fixtures/status/fake-curl b/tests/fixtures/status/fake-curl new file mode 100755 index 0000000..4033b98 --- /dev/null +++ b/tests/fixtures/status/fake-curl @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# +# A stand-in for curl, for tests/unit/60-status.sh. It makes no network +# request: every URL is answered from files under FAKE_CURL_ROOT. +# +# PATH=:$PATH FAKE_CURL_ROOT=DIR scripts/status-collect OUT +# +# For https://HOST/PATH (the query string dropped), with F = ROOT/HOST/PATH: +# F.code the HTTP code to answer; 000 is a connection that fails +# F.location the Location of a 3xx answer +# F.time the seconds %{time_total} reports (default 0.012) +# F the body; a code of 200 when F exists and F.code does not +# ROOT/HOST/.404 the body of a 404 for a file that is not there +# A host with no directory under ROOT does not resolve. +# +# Understood: --output/-o, --write-out/-w, --fail/-f (in a short option group +# too), --location/-L for one 3xx hop. Options that take a value are skipped +# with their value; any other option is skipped alone. -K and --config are +# refused, so a caller that needs them fails loudly here. +# +# FAKE_CURL_LOG, when set, gets one line per request: the URL. +# +# Exit status: 0, or curl's own codes: 6 no such host, 7 no connection, +# 22 --fail and a code of 400 or more, 2 an option this stand-in refuses. +set -euo pipefail + +root="${FAKE_CURL_ROOT:?FAKE_CURL_ROOT is not set}" +outfile="" +fmt="" +url="" +fail=0 +follow=0 +while [ "$#" -gt 0 ]; do + case "$1" in + -o | --output) outfile="$2"; shift 2 ;; + -w | --write-out) fmt="$2"; shift 2 ;; + -f | --fail) fail=1; shift ;; + -L | --location) follow=1; shift ;; + -K | --config) + echo "fake-curl: $1 is not supported" >&2 + exit 2 + ;; + --proto | --proto-redir | --max-redirs | --connect-timeout | --max-time | --speed-limit | \ + --speed-time | --retry | --retry-delay | --retry-max-time | -H | --header | -A | \ + --user-agent | --max-filesize | --data-urlencode | -u | --user | -X | --request | \ + -C | --continue-at | -e | --referer | -D | --dump-header | --parallel-max) + shift 2 + ;; + http://* | https://*) url="$1"; shift ;; + --*) shift ;; + -*) + case "$1" in + *f*) fail=1 ;; + esac + case "$1" in + *L*) follow=1 ;; + esac + shift + ;; + *) shift ;; + esac +done +[ -n "$url" ] || { echo "fake-curl: no URL" >&2; exit 2; } +if [ -n "${FAKE_CURL_LOG:-}" ]; then + printf '%s\n' "$url" >> "$FAKE_CURL_LOG" +fi + +# answer URL -> sets code, body (a file or empty), location, seconds +answer() { + local rest host path f + rest="${1#*://}" + host="${rest%%/*}" + path="/${rest#*/}" + [ "$rest" != "$host" ] || path=/ + path="${path%%\?*}" + if [ ! -d "$root/$host" ]; then + echo "curl: (6) Could not resolve host: $host" >&2 + return 6 + fi + f="$root/$host$path" + body="" + location="" + seconds=0.012 + if [ -f "$f.time" ]; then seconds="$(cat "$f.time")"; fi + if [ -f "$f.code" ]; then + code="$(cat "$f.code")" + if [ -f "$f" ]; then body="$f"; fi + if [ -f "$f.location" ]; then location="$(cat "$f.location")"; fi + elif [ -f "$f" ]; then + code=200 + body="$f" + else + code=404 + if [ -f "$root/$host/.404" ]; then body="$root/$host/.404"; fi + fi + return 0 +} + +rc=0 +answer "$url" || rc=$? +[ "$rc" -eq 0 ] || exit "$rc" +case "$code" in + 3??) + if [ "$follow" -eq 1 ] && [ -n "$location" ]; then + url="$location" + rc=0 + answer "$url" || rc=$? + [ "$rc" -eq 0 ] || exit "$rc" + fi + ;; +esac + +write_out() { + local s="$fmt" redirect="" + case "$code" in + 3??) redirect="$location" ;; + esac + s="${s//%\{http_code\}/$code}" + s="${s//%\{time_total\}/$seconds}" + s="${s//%\{redirect_url\}/$redirect}" + s="${s//%\{url_effective\}/$url}" + s="${s//\\n/$'\n'}" + printf '%s' "$s" +} + +if [ "$code" = 000 ]; then + echo "curl: (7) Failed to connect" >&2 + write_out + exit 7 +fi +if [ "$fail" -eq 1 ] && [ "$code" -ge 400 ]; then + echo "curl: (22) The requested URL returned error: $code" >&2 + write_out + exit 22 +fi +if [ -n "$outfile" ]; then + if [ -n "$body" ]; then cp "$body" "$outfile"; else : > "$outfile"; fi +elif [ -n "$body" ]; then + cat "$body" +fi +write_out diff --git a/tests/unit/60-status.sh b/tests/unit/60-status.sh new file mode 100644 index 0000000..5485170 --- /dev/null +++ b/tests/unit/60-status.sh @@ -0,0 +1,563 @@ +#!/usr/bin/env bash +# +# The status page on fixed inputs, with no network. +# +# scripts/status-collect runs against fixture databases. The upstream, the +# registry and the Worker are answered by tests/fixtures/status/fake-curl, +# first on PATH as curl. scripts/status-render then runs on what was +# collected, with a fixture history and bench file. Every SVG must parse as +# XML, every badge must carry its text, and the page must reference every +# chart and badge. +# +# ⚠ db_records needs GNU tar, and status-render needs python3. A shell that +# lacks one skips that part and says why: busybox has neither, Arch has no +# python3. The runner job has both. +set -euo pipefail +# shellcheck source=../lib/harness.sh +. "$REPO_ROOT/tests/lib/harness.sh" + +work="$(work_dir)" +trap 'rm -rf "$work"' EXIT + +#---------------------------------------------------------------------------# +# The sync interval, from a cron +#---------------------------------------------------------------------------# +cron_file() { + local c + printf 'on:\n schedule:\n' > "$work/cron.yml" + for c in "$@"; do + printf " - cron: '%s'\n" "$c" >> "$work/cron.yml" + done +} +minutes() { + cron_file "$@" + "$REPO_ROOT/scripts/cron-tolerance" --minutes "$work/cron.yml" +} +CT="scripts/cron-tolerance --minutes FILE" +check "interval: every 3 hours is 180 minutes" 180 "$(minutes '17 */3 * * *')" "$CT" +check "interval: hourly is 60 minutes" 60 "$(minutes '47 * * * *')" "$CT" +check "interval: daily is 1440 minutes" 1440 "$(minutes '41 7 * * *')" "$CT" +check "interval: a daily and an hourly cron together give 60 minutes" 60 "$(minutes '41 7 * * *' '7 * * * *')" "$CT" +check "interval: weekly is 10080 minutes" 10080 "$(minutes '0 6 * * 1')" "$CT" +check "interval: 05:30 and 17:30 give 720 minutes" 720 "$(minutes '30 5,17 * * *')" "$CT" +# One firing in the four-year walk: the gap runs round to the same firing. +check "interval: only on 29 February is 1461 days, 2103840 minutes" 2103840 "$(minutes '0 0 29 2 *')" "$CT" +cron_file '17 */3 * * *' +check "the day tolerance is unchanged: every 3 hours is 1 3" "1 3" \ + "$("$REPO_ROOT/scripts/cron-tolerance" "$work/cron.yml")" "scripts/cron-tolerance FILE" + +#---------------------------------------------------------------------------# +# Fixtures: databases, a registry, an upstream and a Worker +#---------------------------------------------------------------------------# +net="$work/net" +bin="$work/bin" +mkdir -p "$bin" "$net/up.test" "$net/bad.test" "$net/ghcr.io" "$net/worker.test" "$work/targets" +cp "$REPO_ROOT/tests/fixtures/status/fake-curl" "$bin/curl" +chmod +x "$bin/curl" +printf '{"errors":[{"code":"MANIFEST_UNKNOWN","message":"manifest unknown"}]}\n' > "$net/ghcr.io/.404" + +# pkg DIR NAME VERSION ARCH CSIZE BUILDDATE [SEED]: one desc entry; SEED +# changes the sha256 and nothing else +pkg() { + local dir="$1" name="$2" ver="$3" arch="$4" csize="$5" built="$6" seed="${7:-}" sha + sha="$(printf '%s-%s%s' "$name" "$ver" "$seed" | sha256sum | cut -d ' ' -f 1)" + mkdir -p "$dir/$name-$ver" + printf '%%FILENAME%%\n%s\n\n%%NAME%%\n%s\n\n%%VERSION%%\n%s\n\n%%CSIZE%%\n%s\n\n%%SHA256SUM%%\n%s\n\n%%BUILDDATE%%\n%s\n\n%%ARCH%%\n%s\n\n' \ + "$name-$ver-$arch.pkg.tar.zst" "$name" "$ver" "$csize" "$sha" "$built" "$arch" > "$dir/$name-$ver/desc" +} +mkdb() { + (cd "$1" && tar -czf "$2" -- *) +} +# mirror ARCH REPO DB FETCHED: the db as the blob latest names +mirror() { + local base="$net/ghcr.io/v2/test/mirror/$1/$2" sha size + sha="$(sha256sum "$3" | cut -d ' ' -f 1)" + size="$(wc -c < "$3" | tr -d ' ')" + mkdir -p "$base/manifests" "$base/blobs" + cp "$3" "$base/blobs/sha256:$sha" + jq -n --arg sha "sha256:$sha" --argjson size "$size" --arg repo "$2" --arg fetched "$4" '{ + schemaVersion: 2, mediaType: "application/vnd.oci.image.manifest.v1+json", + layers: [{mediaType: "application/vnd.pkgforge.archlinux.db.v1", digest: $sha, size: $size, + annotations: {"org.opencontainers.image.title": "\($repo).db"}}], + annotations: {"dev.pkgforge.mirror.fetched": $fetched, "dev.pkgforge.mirror.run": "42"}}' \ + > "$base/manifests/latest" +} +redirect() { + mkdir -p "$(dirname "$net/worker.test/$1")" + printf '%s\n' "${2:-302}" > "$net/worker.test/$1.code" + printf '%s\n' "${3:-https://pkg-containers.githubusercontent.com/ghcrblobs/blobs/sha256:00?se=x}" \ + > "$net/worker.test/$1.location" +} +target() { + local f="$work/targets/$1-$2.conf" + printf 'arch = %s\nrepo = %s\n' "$1" "$2" > "$f" + shift 2 + printf '%s\n' "$@" >> "$f" +} + +B=1790000000 +# x86_64-core: the mirror serves upstream's very database +pkg "$work/up/x86_64-core" acl 2.3.2-1 x86_64 100 "$B" +pkg "$work/up/x86_64-core" bash 5.3.9-1 x86_64 200 "$B" +pkg "$work/up/x86_64-core" zlib 1:1.3.2-3 x86_64 300 "$B" +pkg "$work/up/x86_64-core" coreutils 9.8-1 x86_64 400 "$((B + 60))" +# x86_64-extra: 3 of 5 files match; delta was rebuilt under the same name, +# epsilon moved on +for p in "alpha 1-1 100" "beta 2-1 200" "gamma 1-1 300"; do + read -r n v s <<< "$p" + pkg "$work/up/x86_64-extra" "$n" "$v" x86_64 "$s" "$B" + pkg "$work/mirror/x86_64-extra" "$n" "$v" x86_64 "$s" "$B" +done +pkg "$work/up/x86_64-extra" delta 1-1 x86_64 400 "$B" rebuilt +pkg "$work/mirror/x86_64-extra" delta 1-1 x86_64 400 "$B" +pkg "$work/up/x86_64-extra" epsilon 1-1 x86_64 500 "$((B + 3600))" +pkg "$work/mirror/x86_64-extra" epsilon 0.9-1 x86_64 500 "$B" +# aarch64-core and loong64-core: upstream only +for t in aarch64-core loong64-core; do + pkg "$work/up/$t" linux 6.17-1 "${t%%-*}" 1000 "$B" + pkg "$work/up/$t" glibc 2.42-1 "${t%%-*}" 2000 "$B" +done + +for t in x86_64-core x86_64-extra aarch64-core loong64-core; do + a="${t%%-*}" r="${t#*-}" + mkdir -p "$net/up.test/$a/$r" + mkdb "$work/up/$t" "$net/up.test/$a/$r/$r.db" +done +mkdb "$work/mirror/x86_64-extra" "$work/extra-mirror.db" + +#---------------------------------------------------------------------------# +# Can db_records read a database here? It needs GNU tar. +#---------------------------------------------------------------------------# +# pipefail as in the scripts: without it a tar that fails inside db_records' +# pipeline reads as an empty database. +rc=0 +bash -c 'set -o pipefail; . "$1"; db_records "$2" "$3"' _ "$REPO_ROOT/scripts/lib.sh" \ + "$net/up.test/x86_64/core/core.db" "$work/probe.tsv" > "$work/probe.err" 2>&1 || rc=$? +if [ "$rc" -eq 0 ] && [ "$(awk 'END { print NR }' "$work/probe.tsv")" != 4 ]; then + rc=1 + printf 'db_records read %s record(s), not 4\n' "$(awk 'END { print NR }' "$work/probe.tsv")" >> "$work/probe.err" +fi +if [ "$rc" -ne 0 ]; then + ok "skipped: status-collect, because db_records cannot read a database with this tar ($(sed -n 1p "$work/probe.err"))" + if command -v python3 > /dev/null; then + ok "skipped: status-render, because there is no collected status to render" + fi + summary + exit $? +fi + +mirror x86_64 core "$net/up.test/x86_64/core/core.db" 2026-10-14T00:00:00Z +mirror x86_64 extra "$work/extra-mirror.db" 2026-10-14T00:00:00Z +redirect x86_64/core/core.db +redirect x86_64/core/acl-2.3.2-1-x86_64.pkg.tar.zst +redirect x86_64/extra/extra.db +redirect x86_64/extra/alpha-1-1-x86_64.pkg.tar.zst 503 '' +mkdir -p "$net/bad.test/x86_64/multilib" +printf '500\n' > "$net/bad.test/x86_64/multilib/multilib.db.code" + +# $arch and $repo are the target file's own placeholders, not shell's. +# shellcheck disable=SC2016 +{ + target x86_64 core 'db_server = https://up.test/$arch/$repo' + target x86_64 extra 'db_server = https://up.test/$arch/$repo' + target x86_64 multilib 'db_server = https://bad.test/$arch/$repo' + target aarch64 core 'db_server = https://up.test/$arch/$repo' + target loong64 core 'db_server = https://down.test/$arch/$repo' 'db_server = https://up.test/$arch/$repo' + target riscv64 core 'db_server = https://up.test/$arch/$repo' 'enabled = no' +} + +NOW="$(jq -n '"2026-10-14T03:00:00Z" | fromdateiso8601')" +# collect OUT [SECONDS]: status-collect against the fixtures, its stderr in +# OUT.err; returns its status +collect() { + local rc=0 + mkdir -p "$1" + PATH="$bin:$PATH" FAKE_CURL_ROOT="$net" FAKE_CURL_LOG="$work/requests.log" \ + TARGETS_DIR="$work/targets" MIRROR_NAMESPACE=test/mirror MIRROR_REGISTRY=ghcr.io \ + GITHUB_REPOSITORY=test/mirror STATUS_WORKER_URL=https://worker.test MIRROR_NOW="${2:-$NOW}" \ + "$REPO_ROOT/scripts/status-collect" "$1" > "$1.out" 2> "$1.err" || rc=$? + return "$rc" +} +RP="PATH=:\$PATH FAKE_CURL_ROOT= scripts/status-collect OUT" + +out="$work/run1" +rc=0 +collect "$out" || rc=$? +check "status-collect measures the fixtures and exits 0" 0 "$rc" "$RP; see $(sed -n '$p' "$out.err")" +if [ "$rc" -ne 0 ]; then + diag "$(sed -n 1,40p "$out.err")" +fi +q() { + jq -r "$1" "$out/status.json" +} + +check "status.json is schema 1" 1 "$(q .schema)" "jq .schema OUT/status.json" +check "every enabled target, sorted; a disabled one is left out" \ + "aarch64-core loong64-core x86_64-core x86_64-extra x86_64-multilib" \ + "$(q '[.targets[].target] | join(" ")')" "jq '.targets[].target' OUT/status.json" +check "the sync interval is read from sync.yml's cron" \ + "$("$REPO_ROOT/scripts/cron-tolerance" --minutes "$REPO_ROOT/.github/workflows/sync.yml")" \ + "$(q .sync.interval_minutes)" "jq .sync OUT/status.json" + +T='.targets[] | select(.target == "x86_64-core")' +check "x86_64-core: the mirror database is upstream's, so it is current" true "$(q "$T | .current")" "jq '$T' OUT/status.json" +check "x86_64-core: coverage is 4 of 4 files, ratio 1" "4 4 1 0" \ + "$(q "$T | .coverage | \"\(.files) \(.of_files) \(.ratio_files) \(.missing)\"")" "jq '$T.coverage' OUT/status.json" +check "x86_64-core: age is now minus the fetched time" 10800 "$(q "$T | .age_seconds")" "jq '$T.age_seconds' OUT/status.json" +check "x86_64-core: no lag" 0 "$(q "$T | .lag_seconds")" "jq '$T.lag_seconds' OUT/status.json" +check "x86_64-core: the Worker answers 302 for the database and for one package" \ + "302 true 302 true acl-2.3.2-1-x86_64.pkg.tar.zst" \ + "$(q "$T | .worker | \"\(.db.code) \(.db.ok) \(.package.code) \(.package.ok) \(.package.url | split(\"/\") | last)\"")" \ + "jq '$T.worker' OUT/status.json" +check "x86_64-core: the redirect host is kept, not the signed URL" pkg-containers.githubusercontent.com \ + "$(q "$T | .worker.db.location")" "jq '$T.worker.db' OUT/status.json" + +T='.targets[] | select(.target == "x86_64-extra")' +check "x86_64-extra: a file counts only with the same name and sha256: 3 of 5" "3 5 2" \ + "$(q "$T | .coverage | \"\(.files) \(.of_files) \(.missing)\"")" "jq '$T.coverage' OUT/status.json" +check "x86_64-extra: coverage by bytes is 600 of 1500" "600 1500 0.4" \ + "$(q "$T | .coverage | \"\(.bytes) \(.of_bytes) \(.ratio_bytes)\"")" "jq '$T.coverage' OUT/status.json" +check "x86_64-extra: the missing files are named, sorted" \ + "delta-1-1-x86_64.pkg.tar.zst epsilon-1-1-x86_64.pkg.tar.zst" \ + "$(q "$T | .coverage.missing_names | join(\" \")")" "jq '$T.coverage.missing_names' OUT/status.json" +check "x86_64-extra: lag is upstream's newest build date minus the mirror's" 3600 \ + "$(q "$T | .lag_seconds")" "jq '$T.lag_seconds' OUT/status.json" +check "x86_64-extra: not current" false "$(q "$T | .current")" "jq '$T.current' OUT/status.json" +check "x86_64-extra: a package probe answered 503 fails" "503 false HTTP 503, expected 302" \ + "$(q "$T | .worker.package | \"\(.code) \(.ok) \(.error)\"")" "jq '$T.worker.package' OUT/status.json" + +T='.targets[] | select(.target == "aarch64-core")' +check "aarch64-core: no latest is absent, not an error" absent "$(q "$T | .mirror.state")" "jq '$T.mirror' OUT/status.json" +check "aarch64-core: coverage is 0 of 2" "0 2 0" "$(q "$T | .coverage | \"\(.files) \(.of_files) \(.ratio_files)\"")" \ + "jq '$T.coverage' OUT/status.json" +check "aarch64-core: the Worker must answer 404, and does; no package probe" "404 404 true null" \ + "$(q "$T | .worker | \"\(.db.code) \(.db.expect) \(.db.ok) \(.package)\"")" "jq '$T.worker' OUT/status.json" + +T='.targets[] | select(.target == "loong64-core")' +check "loong64-core: the second db_server answers when the first does not resolve" \ + "https://up.test/loong64/core https://down.test/loong64/core curl 6" \ + "$(q "$T | .upstream | \"\(.server) \(.tried[0].server) \(.tried[0].result)\"")" "jq '$T.upstream' OUT/status.json" + +T='.targets[] | select(.target == "x86_64-multilib")' +check "x86_64-multilib: no db_server answers: upstream not read, coverage unknown" "false HTTP 500 null" \ + "$(q "$T | \"\(.upstream.ok) \(.upstream.tried[0].result) \(.coverage)\"")" "jq '$T' OUT/status.json" + +check "summary: 5 targets, 2 promoted, 1 current, 7 of 13 files" "5 2 1 7 13" \ + "$(q '.summary | "\(.targets) \(.promoted) \(.current) \(.mirrored_files) \(.upstream_files)"')" "jq .summary OUT/status.json" +check "worker: one failed probe fails the run, the first in a row" "false 1 1 2026-10-14T03:00:00Z" \ + "$(q '.worker | "\(.ok) \(.failures) \(.failed_runs) \(.failing_since)"')" "jq .worker OUT/status.json" +check "history.jsonl has one line for one run" 1 "$(awk 'END { print NR }' "$out/history.jsonl")" "wc -l OUT/history.jsonl" +check "history keeps files, mirrored files and age per target" "5 3 10800" \ + "$(jq -r '.x["x86_64-extra"] | "\(.n) \(.m) \(.a)"' "$out/history.jsonl")" "jq -c . OUT/history.jsonl" +if grep -q 'HTTP 503, expected 302' "$out/worker.md" && grep -q 'reproduce:' "$out/worker.md"; then + ok "worker.md names the failed probe and how to reproduce it" +else + fail "worker.md names the failed probe and how to reproduce it" "$(sed -n 1,30p "$out/worker.md")" "cat OUT/worker.md" +fi +bad_hosts="$(awk '{ h = $0; sub(/^[a-z]+:\/\//, "", h); sub(/\/.*/, "", h); if (h !~ /^(up|bad|down|worker)\.test$/ && h != "ghcr.io") print }' "$work/requests.log")" +check "every request went to a fixture host" "" "$bad_hosts" "cat \$work/requests.log" + +# A second run: the package probe now redirects to plain http. The failure +# count goes on, and the first failure's time is kept. +out1="$out" +out="$work/run2" +mkdir -p "$out" +cp "$out1/history.jsonl" "$out/history.jsonl" +redirect x86_64/extra/alpha-1-1-x86_64.pkg.tar.zst 302 http://pkg-containers.githubusercontent.com/x +rc=0 +collect "$out" "$((NOW + 3600))" || rc=$? +check "a second run measures and exits 0" 0 "$rc" "$RP" +check "worker: two failed runs in a row, failing since the first" "false 2 2026-10-14T03:00:00Z" \ + "$(q '.worker | "\(.ok) \(.failed_runs) \(.failing_since)"')" "jq .worker OUT/status.json" +check "worker: a 302 to http is not a redirect to https://*.githubusercontent.com" \ + "302 to 'pkg-containers.githubusercontent.com', not to https://*.githubusercontent.com" \ + "$(q '.targets[] | select(.target == "x86_64-extra") | .worker.package.error')" "jq .targets OUT/status.json" +check "history.jsonl has two lines after two runs" 2 "$(awk 'END { print NR }' "$out/history.jsonl")" "wc -l OUT/history.jsonl" + +# A third run: the Worker passes again, and the count starts from 0. +redirect x86_64/extra/alpha-1-1-x86_64.pkg.tar.zst +rc=0 +collect "$out" "$((NOW + 7200))" || rc=$? +check "worker: a run that passes clears the count" "true 0 null" \ + "$(q '.worker | "\(.ok) \(.failed_runs) \(.failing_since)"')" "jq .worker OUT/status.json" + +# History is cut to the last 2000 lines, the new line last. +out="$work/run3" +mkdir -p "$out" +awk 'BEGIN { for (i = 1; i <= 2005; i++) printf "{\"t\":\"2026-09-01T00:00:00Z\",\"w\":{\"f\":0},\"i\":%d}\n", i }' > "$out/history.jsonl" +rc=0 +collect "$out" || rc=$? +check "history keeps the last 2000 lines" 2000 "$(awk 'END { print NR }' "$out/history.jsonl")" "wc -l OUT/history.jsonl" +check "the oldest lines go first and the new one is last" "7 2026-10-14T03:00:00Z" \ + "$(jq -rs '"\(.[0].i) \(.[-1].t)"' "$out/history.jsonl")" "sed -n '1p;\$p' OUT/history.jsonl" + +# A last line that does not read is named, and the count starts again. +out="$work/run4" +mkdir -p "$out" +printf '{"t":"2026-10-14T02:00:00Z","w":{"f":5,"s":"2026-10-13T00:00:00Z"}}\nnot json\n' > "$out/history.jsonl" +redirect x86_64/extra/alpha-1-1-x86_64.pkg.tar.zst 503 '' +rc=0 +collect "$out" || rc=$? +check "a history line that does not read does not stop the collect" 0 "$rc" "$RP" +check "and the failure count starts again from it" 1 "$(q .worker.failed_runs)" "jq .worker OUT/status.json" +if grep -q 'history.jsonl does not read' "$out.err"; then + ok "the line that does not read is named on stderr" +else + fail "the line that does not read is named on stderr" "$(sed -n 1,20p "$out.err")" "$RP" +fi + +#---------------------------------------------------------------------------# +# check-staleness: a target with no latest is new while git says its file is +# younger than the tolerance, and stale after, or when git cannot say +#---------------------------------------------------------------------------# +if ! command -v git > /dev/null; then + ok "skipped: the new verdict of check-staleness, because git is not on PATH" +else + srepo="$work/stale-repo" + mkdir -p "$srepo/targets" + git -C "$srepo" init --quiet + add_target() { + # shellcheck disable=SC2016 # the target file's own placeholders + printf 'arch = %s\nrepo = core\ndb_server = https://up.test/$arch/$repo\n' "$1" > "$srepo/targets/$1-core.conf" + git -C "$srepo" add "targets/$1-core.conf" + GIT_AUTHOR_DATE="$2T12:00:00Z" GIT_COMMITTER_DATE="$2T12:00:00Z" \ + git -C "$srepo" -c user.name=test -c user.email=test@example.invalid commit --quiet -m "add $1-core" + } + add_target aarch64 2026-10-01 + add_target loong64 2026-10-13 + staleness() { + local rc=0 + PATH="$bin:$PATH" FAKE_CURL_ROOT="$net" TARGETS_DIR="$2" MIRROR_NAMESPACE=test/mirror \ + MIRROR_REGISTRY=ghcr.io MIRROR_NOW="$NOW" \ + "$REPO_ROOT/scripts/check-staleness" "$1" > "$1.out" 2> "$1.err" || rc=$? + return "$rc" + } + RS="PATH=:\$PATH TARGETS_DIR=/targets scripts/check-staleness OUT" + rc=0 + staleness "$work/stale1" "$srepo/targets" || rc=$? + check "check-staleness: one target stale, so exit 3" 3 "$rc" "$RS; $(sed -n '$p' "$work/stale1.err")" + check "check-staleness: no latest, added 13 days ago, tolerance 3: stale" stale \ + "$(cat "$work/stale1/aarch64-core.state")" "$RS" + check "check-staleness: no latest, added 1 day ago, tolerance 3: new" new \ + "$(cat "$work/stale1/loong64-core.state")" "$RS" + if grep -q '| target file added | 2026-10-01 |' "$work/stale1/aarch64-core.md"; then + ok "check-staleness: the issue body says when the target was added" + else + fail "check-staleness: the issue body says when the target was added" \ + "$(sed -n 1,20p "$work/stale1/aarch64-core.md")" "$RS" + fi + git clone --quiet --depth 1 "file://$srepo" "$work/stale-shallow" + rc=0 + staleness "$work/stale2" "$work/stale-shallow/targets" || rc=$? + check "check-staleness: in a shallow clone the age is unknown, so stale, never new" "3 stale" \ + "$rc $(cat "$work/stale2/loong64-core.state")" "git clone --depth 1 ...; $RS" + if grep -q 'shallow' "$work/stale2.err"; then + ok "check-staleness: says that the clone is shallow" + else + fail "check-staleness: says that the clone is shallow" "$(sed -n 1,20p "$work/stale2.err")" "$RS" + fi +fi + +#---------------------------------------------------------------------------# +# status-render +#---------------------------------------------------------------------------# +if ! command -v python3 > /dev/null; then + ok "skipped: status-render, because python3 is not on PATH" + summary + exit $? +fi + +in="$work/render-in" +mkdir -p "$in" +cp "$work/run1/status.json" "$in/status.json" +cp "$REPO_ROOT/tests/fixtures/status/bench.json" "$in/bench.json" +# 30 days of history at 6 hour steps, ending at the collect time +jq -nc --argjson end "$NOW" ' + range(119; -1; -1) as $i | ($end - $i * 21600) as $t + | {t: ($t | todate), w: {ok: true, f: 0, s: null, ms: 140}, g: {ms: 400}, + x: {"x86_64-core": {n: 4, m: 4, b: 1000, mb: 1000, a: (3600 + ($i % 3) * 3600), l: 0, c: 1}, + "x86_64-extra": {n: 5, m: ([3, ((119 - $i) / 30 | floor)] | min), b: 1500, mb: 600, a: 10800, l: 3600, c: 0}, + "aarch64-core": {n: 2, m: 0, b: 3000, mb: 0, a: null, l: null, c: 0}}}' > "$in/history.jsonl" + +cat > "$work/check.py" << 'EOF' +import json, os, sys +import xml.etree.ElementTree as ET +from html.parser import HTMLParser + +site, spec = sys.argv[1], json.loads(sys.argv[2]) + + +def say(ok, desc, detail=""): + print("%s\t%s\t%s" % ("PASS" if ok else "FAIL", desc, detail.replace("\n", " "))) + + +def texts(path): + root = ET.parse(path).getroot() + return " ".join("".join(e.itertext()) for e in root.iter() if e.tag.endswith("}text")) + + +missing = [f for f in spec["files"] if not os.path.isfile(os.path.join(site, f))] +say(not missing, "every page, chart, badge and data file is written", " ".join(missing)) + +bad = [] +svgs = 0 +for d in ("charts", "badges"): + for dirpath, _, names in os.walk(os.path.join(site, d)): + for f in sorted(names): + p = os.path.join(dirpath, f) + rel = os.path.relpath(p, site) + svgs += 1 + try: + root = ET.parse(p).getroot() + except ET.ParseError as e: + bad.append("%s: %s" % (rel, e)) + continue + if not root.tag.endswith("}svg"): + bad.append("%s: the root is %s" % (rel, root.tag)) + if not any(e.tag.endswith("}title") and "".join(e.itertext()).strip() for e in root): + bad.append("%s: no title" % rel) + style = " ".join("".join(e.itertext()) for e in root.iter() if e.tag.endswith("}style")) + if "prefers-color-scheme:dark" not in style.replace(" ", ""): + bad.append("%s: no dark style" % rel) +say(not bad and svgs == spec["svgs"], + "all %d SVG files parse as XML, each with a title and its own dark style" % spec["svgs"], + "found %d; %s" % (svgs, "; ".join(bad))) + +for name, words in sorted(spec["badges"].items()): + p = os.path.join(site, "badges", name + ".svg") + try: + t = texts(p) + except (OSError, ET.ParseError) as e: + say(False, "badge %s carries its text" % name, str(e)) + continue + lost = [w for w in words if w not in t] + say(not lost, "badge %s carries its text: %s" % (name, " | ".join(words)), "text: %s; not found: %s" % (t, lost)) + +for name, words in sorted(spec.get("chart_text", {}).items()): + try: + t = texts(os.path.join(site, "charts", name + ".svg")) + except (OSError, ET.ParseError) as e: + say(False, "chart %s shows %s" % (name, ", ".join(words)), str(e)) + continue + lost = [w for w in words if w not in t] + say(not lost, "chart %s shows %s" % (name, ", ".join(words)), "not found: %s" % lost) + + +class Refs(HTMLParser): + def __init__(self): + super().__init__() + self.refs = set() + + def handle_starttag(self, tag, attrs): + for k, v in attrs: + if k in ("src", "href", "srcset") and v and "://" not in v and not v.startswith("#"): + self.refs.add(v) + + +r = Refs() +with open(os.path.join(site, "index.html"), encoding="utf-8") as f: + r.feed(f.read()) +want = (["charts/%s.svg" % c for c in spec["charts"]] + ["charts/narrow/%s.svg" % c for c in spec["charts"]] + + ["badges/%s.svg" % b for b in spec["badges"]]) +lost = [w for w in want if w not in r.refs] +say(not lost, "index.html references every chart and badge", " ".join(lost)) +broken = sorted(x for x in r.refs if not os.path.isfile(os.path.join(site, x))) +say(not broken, "every file index.html references exists", " ".join(broken)) + +with open(os.path.join(site, "history.json"), encoding="utf-8") as f: + h = json.load(f) +say(isinstance(h, list) and len(h) == spec["history"], "history.json holds %d point(s)" % spec["history"], + "found %s" % (len(h) if isinstance(h, list) else type(h).__name__)) +with open(os.path.join(site, "bench.json"), encoding="utf-8") as f: + b = json.load(f) +say(len(b.get("cases", [])) == spec["bench_cases"], "bench.json holds %d case(s)" % spec["bench_cases"], + "found %d" % len(b.get("cases", []))) +EOF + +# render IN SITE SPEC: status-render, then check.py's verdicts as TAP +render() { + local rc=0 verdict desc detail + python3 -I "$REPO_ROOT/scripts/status-render" "$1" "$2" > "$2.out" 2> "$2.err" || rc=$? + check "status-render $(basename "$2") exits 0" 0 "$rc" "python3 -I scripts/status-render IN SITE; $(sed -n 1p "$2.err")" + [ "$rc" -eq 0 ] || return 0 + rc=0 + python3 -I "$work/check.py" "$2" "$3" > "$2.check" 2>&1 || rc=$? + if [ "$rc" -ne 0 ]; then + fail "the site checks run" "$(sed -n 1,20p "$2.check")" "python3 -I check.py SITE SPEC" + return 0 + fi + while IFS=$'\t' read -r verdict desc detail; do + if [ "$verdict" = PASS ]; then + ok "$(basename "$2"): $desc" + else + fail "$(basename "$2"): $desc" "$detail" "python3 -I scripts/status-render IN SITE" + fi + done < "$2.check" +} + +CHARTS='["coverage","freshness","size","history-coverage","history-age","speed"]' +spec() { + # spec BENCH_CASES HISTORY SPEED_WORDS + jq -nc --argjson charts "$CHARTS" --argjson cases "$1" --argjson history "$2" --argjson speed "$3" '{ + charts: $charts, bench_cases: $cases, history: $history, + svgs: (($charts | length) * 2 + 10), + files: (["index.html", "status.json", "history.json", "bench.json"] + + ($charts | map("charts/\(.).svg")) + ($charts | map("charts/narrow/\(.).svg"))), + badges: { + "x86_64-core": ["x86_64-core", "100%", "3.0 h"], + "x86_64-extra": ["x86_64-extra", "60%", "3.0 h"], + "x86_64-multilib": ["x86_64-multilib", "not mirrored"], + "aarch64-core": ["aarch64-core", "not mirrored"], + "loong64-core": ["loong64-core", "not mirrored"], + "x86_64": ["x86_64", "77.7%"], + "aarch64": ["aarch64", "not mirrored"], + "loong64": ["loong64", "not mirrored"], + "all": ["mirror", "53.8%", "2/5 repos"], + "speed": $speed}, + chart_text: { + coverage: ["Coverage by repository", "53.8%", "x86_64", "aarch64", "loong64", "core", "extra", "multilib", "upstream only"], + freshness: ["Snapshot age", "3.0 h", "sync every 3 h", "not mirrored yet", "matches upstream"], + size: ["Size by repository", "GiB"], + "history-coverage": ["Coverage, last 30 days", "x86_64", "aarch64", "Time, UTC"], + "history-age": ["Snapshot age, last 30 days", "sync every 3 h", "log scale"]}}' +} + +render "$in" "$work/site" "$(spec 12 120 '["pacman speed", "oci 13.3 s", "official 15.6 s"]')" + +# No bench, and a history line that does not read +in2="$work/render-in2" +mkdir -p "$in2" +cp "$in/status.json" "$in2/status.json" +{ + sed -n 1,10p "$in/history.jsonl" + printf 'not json\n' +} > "$in2/history.jsonl" +render "$in2" "$work/site2" "$(spec 0 10 '["pacman speed", "no data"]')" +if grep -q 'history.jsonl line 11 is not JSON' "$work/site2.err"; then + ok "a history line that does not read is named by its number" +else + fail "a history line that does not read is named by its number" "$(cat "$work/site2.err")" "python3 -I scripts/status-render IN SITE" +fi +if grep -q 'No measurement yet' "$work/site2/charts/speed.svg"; then + ok "with no bench.json the speed chart says so" +else + fail "with no bench.json the speed chart says so" "$(sed -n 1,5p "$work/site2/charts/speed.svg")" "python3 -I scripts/status-render IN SITE" +fi + +# The first run: one history line, in which some targets have no age +in3="$work/render-in3" +mkdir -p "$in3" +cp "$work/run1/status.json" "$work/run1/history.jsonl" "$in3/" +render "$in3" "$work/site3" "$(spec 0 1 '["pacman speed", "no data"]')" + +# No status.json: refused, with the path named +rc=0 +python3 -I "$REPO_ROOT/scripts/status-render" "$work/empty" "$work/site4" > "$work/site4.out" 2> "$work/site4.err" || rc=$? +check "status-render with no status.json exits 1" 1 "$rc" "python3 -I scripts/status-render EMPTY SITE" +if grep -q 'status.json: no such file' "$work/site4.err"; then + ok "and names the file it looked for" +else + fail "and names the file it looked for" "$(cat "$work/site4.err")" "python3 -I scripts/status-render EMPTY SITE" +fi + +summary From bbee3d8a6ec2ce744b5ae583e78386cab4b02124 Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 11:33:31 +0545 Subject: [PATCH 04/22] ci: find tests by discovery, pin static curl and bash, check workflows, python and Renovate pins --- .github/renovate.json | 46 +++++- .github/workflows/ci.yml | 201 +++++++++++++++++++------ scripts/debloat-runner | 12 +- scripts/install-test-tools | 233 +++++++++++++++++++++++++---- tests/lib/harness.sh | 149 +++++++++++++++++- tests/run.sh | 116 ++++++++++++-- tests/static/10-shellcheck.sh | 29 ++-- tests/static/20-actionlint.sh | 4 +- tests/static/40-executable-bits.sh | 19 ++- tests/static/50-workflow-policy.sh | 165 +++++++++++++++++++- tests/static/70-python.sh | 58 +++++++ tests/static/72-test-runner.sh | 114 ++++++++++++++ tests/static/75-renovate.sh | 175 ++++++++++++++++++++++ 13 files changed, 1201 insertions(+), 120 deletions(-) create mode 100644 tests/static/70-python.sh create mode 100644 tests/static/72-test-runner.sh create mode 100644 tests/static/75-renovate.sh diff --git a/.github/renovate.json b/.github/renovate.json index af0a956..d84c320 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -1,27 +1,59 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "description": [ - "Renovate proposes pin moves; a person merges them. Nothing here merges on its own: the verify image decides what this mirror accepts, so a change to it is reviewed.", - "Pins stay pins: actions by commit hash with the version in a comment, images by digest under a '# tag: ' marker. The repository requires SHA-pinned actions, and tests/static/50-workflow-policy.sh fails a change that turns one back into a tag." + "Pins stay pins: actions by commit hash with the version in a comment, images by digest under a '# tag: ' marker. The repository requires SHA-pinned actions, and tests/static/50-workflow-policy.sh fails a change that turns one back into a tag.", + "Digest, pin, patch and minor updates of actions and images merge on their own, as a squash, when every check of the pull request is green. Renovate reads the checks itself: platformAutomerge is off, because main has no required checks and GitHub's own auto-merge would merge a red pull request. A major update and a keyring image wait for a person.", + "The verify image of the targets and the Arch image of the tests move in one pull request. So CI runs pacman in the image that the targets will verify with, before that image merges.", + "ignorePaths replaces the preset's list, which ignores **/tests/**: the test images are pins too.", + "tests/static/75-renovate.sh fails an image pin that the regex manager below does not match, and an automerge rule for major updates." ], - "extends": ["config:recommended", ":disableDependencyDashboard"], + "extends": ["config:recommended", ":disableDependencyDashboard", "helpers:pinGitHubActionDigests"], + "ignorePaths": ["**/node_modules/**"], "labels": ["dependencies"], + "platformAutomerge": false, + "automergeType": "pr", + "automergeStrategy": "squash", "packageRules": [ { "description": "Actions move weekly in one pull request, and only to a release at least three days old, so a release that is pulled or hijacked upstream has time to be noticed before it lands here.", "matchManagers": ["github-actions"], "groupName": "github actions", - "schedule": ["before 6am on monday"], + "schedule": ["* 0-5 * * 1"], "minimumReleaseAge": "3 days" + }, + { + "description": "Small updates of actions and images merge when CI is green.", + "matchManagers": ["github-actions", "custom.regex"], + "matchUpdateTypes": ["digest", "pin", "pinDigest", "patch", "minor"], + "automerge": true + }, + { + "description": "The 15 verify_image lines and the tests' Arch image: one pull request.", + "matchManagers": ["custom.regex"], + "matchDepNames": ["docker.io/library/archlinux"], + "groupName": "archlinux base image" + }, + { + "description": "The keyring images of the targets (keyring_image): one pull request, which a person merges. Their keyrings are the trust root of the aarch64, armv7h, loong64 and riscv64 targets, and no CI job runs them.", + "matchManagers": ["custom.regex"], + "matchDepNames": ["ghcr.io/pkgforge-dev/archlinux"], + "groupName": "keyring images", + "automerge": false + }, + { + "description": "The node image of worker.yml and ci.yml: one pull request, so the Worker is tested in the image that deploys it.", + "matchManagers": ["custom.regex"], + "matchDepNames": ["docker.io/library/node"], + "groupName": "node image" } ], "customManagers": [ { "customType": "regex", - "description": "An image pinned by digest under a '# tag: ' marker: the verify image of each target, and the images the tests run in.", - "managerFilePatterns": ["/^targets/[^/]+\\.conf$/", "/^\\.github/workflows/[^/]+\\.ya?ml$/", "/^tests/integration/[^/]+\\.sh$/"], + "description": "An image pinned by digest under a '# tag: ' marker: the verify and keyring images of each target, and the images the workflows, the scripts and the tests run.", + "managerFilePatterns": ["/^targets/[^/]+\\.conf$/", "/^\\.github/workflows/[^/]+\\.ya?ml$/", "/^scripts/[^/]+$/", "/^tests/.+\\.sh$/"], "matchStrings": [ - "#\\s*tag:\\s*(?[A-Za-z0-9._-]+)\\s*\\n(?:[^\\n]*\\n){0,3}?[^\\n]*?(?docker\\.io/[a-z0-9._/-]+?)(?::[A-Za-z0-9._-]+)?@(?sha256:[a-f0-9]{64})" + "#\\s*tag:\\s*(?[A-Za-z0-9._-]+)\\s*\\n(?:[^\\n]*\\n){0,3}?[^\\n]*?(?(?:docker\\.io|ghcr\\.io)/[a-z0-9._/-]+?)(?::[A-Za-z0-9._-]+)?@(?sha256:[a-f0-9]{64})" ], "datasourceTemplate": "docker" } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 077c536..b55ee9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,23 +1,39 @@ -# The test suites, on every push and pull request. +# The test suites, on each push to main and each pull request. # -# static shellcheck, actionlint and the policy checks (tests/static) -# unit the wrapper and the scripts, under three shells and three awks: -# dash and mawk here, bash and gawk in Arch, busybox in Alpine -# integration the whole mirror against a local registry and upstream, and -# pacman installing through the wrapper (tests/integration) -# debloat scripts/debloat-runner on a real runner, so code that a sync -# rarely needs is still seen to work +# static shellcheck, actionlint, py_compile and the policy checks +# (tests/static) +# unit the wrapper and the scripts under three shells and three +# awks, one job each: dash and mawk on the runner, bash and +# gawk in Arch, busybox in Alpine (tests/unit) +# integration one job per file in tests/integration, found when the run +# starts, so a new file gets its own job with no edit here +# worker the Cloudflare Worker's tests in the node image (tests/worker) +# debloat scripts/debloat-runner on a real runner, measured +# +# ⛔ No job installs a package from a distro mirror. scripts/install-test-tools +# downloads pinned static binaries and checks each sha256. The containers +# mount that directory read-only, last on PATH: a tool fills a gap, and the +# image's own tools stay the ones under test. +# +# Push runs only on main, so a pull request from a branch here runs once, not +# twice. The status branch holds a generated site and never runs this. name: CI on: push: + branches: [main] pull_request: + branches-ignore: [status] + workflow_dispatch: permissions: {} +# One group per pull request, or per ref. A newer push to a pull request +# cancels the run in progress; a run on main always finishes, so each commit +# on main gets a verdict. concurrency: - group: ci-${{ github.ref }} - cancel-in-progress: true + group: ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} defaults: run: @@ -34,60 +50,162 @@ jobs: with: persist-credentials: false - name: Static suite + env: + # Here each of these is present, so a test that needs one fails + # rather than skips when it is not. + TESTS_REQUIRE_TOOLS: shellcheck actionlint jq python3 run: | - scripts/install-test-tools "$RUNNER_TEMP/bin" + scripts/install-test-tools "$RUNNER_TEMP/bin" shellcheck actionlint jq + python3 --version + # The runner's own test first, without the runner: a run.sh that + # hides failures would hide this one too. + REPO_ROOT="$PWD" bash tests/static/72-test-runner.sh PATH="$RUNNER_TEMP/bin:$PATH" tests/run.sh static unit: + name: unit (${{ matrix.leg }}) runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: contents: read + strategy: + fail-fast: false + matrix: + include: + - leg: dash-mawk + image: '' + require: jq curl python3 + - leg: bash-gawk + # tag: base + image: docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 + require: jq curl + - leg: busybox + # tag: 3.22 + image: docker.io/library/alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8 + require: jq curl steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Unit suite, dash and mawk + - name: Test tools + run: scripts/install-test-tools "$RUNNER_TEMP/tools" jq curl bash + - name: Unit suite + env: + LEG: ${{ matrix.leg }} + IMAGE: ${{ matrix.image }} + TESTS_REQUIRE_TOOLS: ${{ matrix.require }} run: | - # Ubuntu's awk is gawk once gawk is installed; mawk is still there, - # and put first here, and both are asserted so an image change shows. - mkdir -p "$RUNNER_TEMP/mawk" - ln -sf /usr/bin/mawk "$RUNNER_TEMP/mawk/awk" - export PATH="$RUNNER_TEMP/mawk:$PATH" - if [ "$(readlink -f /bin/sh)" != /usr/bin/dash ]; then - echo "::error::/bin/sh is $(readlink -f /bin/sh), not dash" - exit 1 + # The shell, the awk and the tools each leg runs, asserted, so a + # change in an image shows here and not as a passing run of the + # wrong thing. + cat > "$RUNNER_TEMP/leg.sh" << 'EOF' + set -eu + PATH="$PATH:$TOOLS" + export PATH + sh_is="$(readlink -f /bin/sh)" + awk_is="$(readlink -f "$(command -v awk)")" + echo "$LEG: sh $sh_is, awk $awk_is, bash $(command -v bash), curl $(command -v curl), jq $(command -v jq)" + case "$LEG:$sh_is:$awk_is" in + dash-mawk:/usr/bin/dash:/usr/bin/mawk) ;; + bash-gawk:/usr/bin/bash:/usr/bin/gawk*) ;; + busybox:/bin/busybox:/bin/busybox) ;; + *) + echo "::error::$LEG runs sh $sh_is and awk $awk_is, which is not what this leg tests" + exit 1 + ;; + esac + exec bash tests/run.sh unit + EOF + if [ "$LEG" = dash-mawk ]; then + # Ubuntu's awk is gawk once gawk is installed; mawk is still + # there, and goes first on PATH here. + mkdir -p "$RUNNER_TEMP/mawk" + ln -sf /usr/bin/mawk "$RUNNER_TEMP/mawk/awk" + PATH="$RUNNER_TEMP/mawk:$PATH" TOOLS="$RUNNER_TEMP/tools" sh "$RUNNER_TEMP/leg.sh" + else + docker run --rm -v "$PWD:/work:ro" -v "$RUNNER_TEMP/tools:/tools:ro" \ + -v "$RUNNER_TEMP/leg.sh:/leg.sh:ro" -w /work \ + -e LEG -e TESTS_REQUIRE_TOOLS -e TOOLS=/tools "$IMAGE" sh /leg.sh fi - if [ "$(readlink -f "$(command -v awk)")" != /usr/bin/mawk ]; then - echo "::error::awk is $(readlink -f "$(command -v awk)"), not mawk" + + # The matrix of the next job, from tests/run.sh --list. + integration-files: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + outputs: + files: ${{ steps.list.outputs.files }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: List the integration test files + id: list + run: | + names="$(tests/run.sh --list integration)" + json="" + while IFS= read -r f; do + [ -n "$f" ] || continue + case "$f" in + *[!A-Za-z0-9._-]*) + echo "::error::tests/integration/$f: a test file name is letters, digits, '.', '_' and '-' only" + exit 1 + ;; + esac + json="${json:+$json,}\"$f\"" + done <<< "$names" + if [ -z "$json" ]; then + echo "::error::tests/run.sh --list integration found no test file" exit 1 fi - tests/run.sh unit - - name: Unit suite, bash and gawk (Arch) - run: | - # tag: base - image=docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 - docker run --rm -v "$PWD:/work:ro" -w /work "$image" bash -c 'pacman -Sy --noconfirm --needed jq > /dev/null && bash tests/run.sh unit' - - name: Unit suite, busybox (Alpine) - run: | - # tag: 3.22 - image=docker.io/library/alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8 - docker run --rm -v "$PWD:/work:ro" -w /work "$image" sh -c 'apk add --no-cache bash curl jq > /dev/null && bash tests/run.sh unit' + echo "files=[$json]" | tee -a "$GITHUB_OUTPUT" integration: + needs: integration-files runs-on: ubuntu-24.04 timeout-minutes: 45 permissions: contents: read + strategy: + fail-fast: false + matrix: + file: ${{ fromJSON(needs.integration-files.outputs.files) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Integration suite + - name: Integration test env: CONTAINER_RUNTIME: docker - run: tests/run.sh integration + TEST_FILE: ${{ matrix.file }} + run: tests/run.sh integration "$TEST_FILE" + + worker: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Test tools + run: scripts/install-test-tools "$RUNNER_TEMP/tools" jq curl bash + - name: Worker suite + env: + # The node image of .github/workflows/worker.yml. Renovate moves + # the two in one pull request (group "node image"). + # tag: 24-alpine + IMAGE: docker.io/library/node:24-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1 + run: | + docker run --rm -v "$PWD:/work:ro" -v "$RUNNER_TEMP/tools:/tools:ro" -w /work \ + -e TESTS_REQUIRE_TOOLS='node bash' "$IMAGE" \ + sh -c 'PATH="$PATH:/tools"; export PATH; node --version; exec bash tests/run.sh worker' + # Measured, not judged: a runner image that no longer carries these + # directories frees less, and that is worth a warning, not a red run. + # debloat-runner failing outright still fails the job. debloat: runs-on: ubuntu-24.04 timeout-minutes: 15 @@ -102,10 +220,9 @@ jobs: before="$(df --output=avail -B1 "$RUNNER_TEMP" | awk 'NR == 2 { print $1 }')" scripts/debloat-runner --dir "$RUNNER_TEMP" after="$(df --output=avail -B1 "$RUNNER_TEMP" | awk 'NR == 2 { print $1 }')" - echo "freed $(((after - before) / 1073741824)) GiB" - # Seen to work means it freed something: a runner image that no - # longer carries any of these directories is worth knowing about. - [ "$after" -gt $((before + 1073741824)) ] || { - echo "::error::debloat-runner freed less than 1 GiB; the runner image changed under it" - exit 1 - } + freed=$((after - before)) + echo "freed $((freed / 1048576)) MiB on $ImageOS $ImageVersion" + echo "debloat-runner freed $((freed / 1048576)) MiB on $ImageOS $ImageVersion" >> "$GITHUB_STEP_SUMMARY" + if [ "$freed" -lt 1073741824 ]; then + echo "::warning::debloat-runner freed $((freed / 1048576)) MiB, less than 1 GiB, on $ImageOS $ImageVersion. The runner image changed under it: compare its TARGETS with du -sh of each on this image." + fi diff --git a/scripts/debloat-runner b/scripts/debloat-runner index 4e93f78..668f45a 100755 --- a/scripts/debloat-runner +++ b/scripts/debloat-runner @@ -88,10 +88,14 @@ trap 'rm -rf "$work"' EXIT pids=() names=() +absent=() for t in "${TARGETS[@]}"; do # shellcheck disable=SC2086 # unquoted so a glob such as julia* expands for p in $t; do - [ -e "$p" ] || continue + if [ ! -e "$p" ]; then + absent+=("$p") + continue + fi # shellcheck disable=SC2024 # the log is the caller's file; only rm needs root sudo rm -rf --one-file-system -- "$p" > "$work/$(printf '%s' "$p" | tr '/' '_').log" 2>&1 & pids+=("$!") @@ -117,6 +121,12 @@ after="$(free_bytes "$dir")" printf 'debloat-runner: removed %d of %d directories, %s free %d GiB -> %d GiB (+%d GiB)\n' \ "$removed" "$((removed + failed))" "$dir" "$((before / 1073741824))" "$((after / 1073741824))" \ "$(((after - before) / 1073741824))" +# A path in TARGETS that this image does not have frees nothing. Named here, +# so a run that frees less than before says which directories went away. +if [ "${#absent[@]}" -gt 0 ]; then + printf 'debloat-runner: not on this image (%s %s): %s\n' \ + "${ImageOS:-unknown image}" "${ImageVersion:-}" "${absent[*]}" +fi if [ -n "$need_bytes" ] && [ "$after" -lt "$need_bytes" ]; then echo "::warning::debloat-runner: $dir still has only $((after / 1073741824)) GiB free, $((need_bytes / 1073741824)) GiB needed" fi diff --git a/scripts/install-test-tools b/scripts/install-test-tools index a635148..c1ac8ca 100755 --- a/scripts/install-test-tools +++ b/scripts/install-test-tools @@ -1,15 +1,35 @@ #!/usr/bin/env bash # -# Install the pinned shellcheck, actionlint and jq into a directory, each -# checked against its sha256, for tests/run.sh. +# Install pinned test tools into a directory, each checked against its sha256, +# for tests/run.sh and the CI jobs. # -# scripts/install-test-tools DIR +# scripts/install-test-tools DIR [TOOL ...] # -# ⛔ The versions and checksums are written here. Each checksum was read on -# 2026-10-13 from the release's own checksum list and from GitHub's asset -# digest, and the two agreed. +# TOOL is shellcheck, actionlint, jq, curl or bash. With no TOOL, all five. +# Each one is a static binary for x86_64 or aarch64 Linux. So one DIR serves +# the runner and every container that mounts it (Arch, Alpine, the node +# image), and no job installs a package from a distro mirror. # -# Exit status: 0 all installed, 1 could not. +# ⛔ The versions and checksums are written here. Where each checksum was read: +# ShellCheck, actionlint and jq: 2026-10-13, from the release's own +# checksum list and from GitHub's asset digest. The two agreed. +# curl 2026-10-14, stunnel/static-curl release 8.22.0, the static Linux +# build that curl.se/download.html links. The tarball sum is GitHub's +# asset digest. The binary sum is in the release notes ("Checksums of +# binaries") and in the SHA256SUMS file inside the tarball. All agreed +# with the downloaded files. +# bash 2026-10-14, Ubuntu 24.04 (noble) bash-static 5.2.21-2ubuntu4, from +# the release pocket, which does not change while noble is supported. The +# .deb sum is in noble's universe Packages index and in Launchpad's +# binaryFileUrls API. The two agreed with the downloaded files, from +# Launchpad and from the archive. The binary sum was measured from the +# extracted .deb. +# +# ⚠ The bash .deb is an ar archive with a zstd data member. It is read here +# with dd, head and tail, then zstd. So the host needs zstd for bash, but no +# ar, dpkg or bsdtar. +# +# Exit status: 0 all installed, 1 could not, 2 usage. set -euo pipefail SHELLCHECK_VERSION=0.11.0 @@ -24,6 +44,22 @@ JQ_VERSION=1.8.2 JQ_SHA256_AMD64=b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f JQ_SHA256_ARM64=8b85c817833814ddca00a144c33705546355afccf0cf39b188f3cdb48b852309 +# The musl build: it has no loader or libc of its own to find, so it runs in +# glibc and musl images alike. +CURL_VERSION=8.22.0 +CURL_TAR_SHA256_X86_64=dfb02460ba2abe513087538f12a3cf79b74b64a5ea3787ce8ac0cdb11251f884 +CURL_TAR_SHA256_AARCH64=cf94cbeaae1b3c1944a4a761ef04478f8f0b23e93a324b5ed009b2082eda11f3 +CURL_SHA256_X86_64=b7443db66d622a27ef2152bb2032d6ada9fb29a8917f22b9521a993f8e61cd7f +CURL_SHA256_AARCH64=c392bc0ea0c7951ee170e0eb7874c9d4000397254d788552ffaa4e7ccb75ae07 + +BASH_DEB_VERSION=5.2.21-2ubuntu4 +BASH_DEB_SHA256_AMD64=684691928e746a29d4b97c3b58069666a3cdd73104c90b217501f5c1ae7cbd64 +BASH_DEB_SHA256_ARM64=1fd25dda094189ac7ad22acad78c298fdd7fedc60c2f45bb00512c13708224a2 +BASH_SHA256_AMD64=ea3065d65dd07162e42e6db082103ef7dda0578436f15da76ff17be7b31cf671 +BASH_SHA256_ARM64=923600157c5ec8cbd17c45127cbf34c766ad401722ceb0f0661f2a287538be47 + +ALL_TOOLS="shellcheck actionlint jq curl bash" + die() { echo "install-test-tools: $1" >&2 shift @@ -34,50 +70,189 @@ die() { exit 1 } +usage() { + echo "usage: scripts/install-test-tools DIR [TOOL ...]" >&2 + echo " TOOL: $ALL_TOOLS (default: all)" >&2 + exit 2 +} + dest="${1:-}" -[ -n "$dest" ] || die "usage: scripts/install-test-tools DIR" +[ -n "$dest" ] || usage +case "$dest" in + -*) usage ;; +esac +shift +tools="${*:-$ALL_TOOLS}" +for t in $tools; do + case " $ALL_TOOLS " in + *" $t "*) ;; + *) + echo "install-test-tools: no pinned tool named '$t'" >&2 + usage + ;; + esac +done case "$(uname -m)" in x86_64) sc_arch=x86_64 sc_sum="$SHELLCHECK_SHA256_X86_64" al_arch=amd64 al_sum="$ACTIONLINT_SHA256_AMD64" jq_arch=amd64 jq_sum="$JQ_SHA256_AMD64" + curl_arch=x86_64 curl_tar_sum="$CURL_TAR_SHA256_X86_64" curl_sum="$CURL_SHA256_X86_64" + deb_arch=amd64 bash_deb_sum="$BASH_DEB_SHA256_AMD64" bash_sum="$BASH_SHA256_AMD64" + ubuntu_pool=http://archive.ubuntu.com/ubuntu/pool ;; aarch64 | arm64) sc_arch=aarch64 sc_sum="$SHELLCHECK_SHA256_AARCH64" al_arch=arm64 al_sum="$ACTIONLINT_SHA256_ARM64" jq_arch=arm64 jq_sum="$JQ_SHA256_ARM64" + curl_arch=aarch64 curl_tar_sum="$CURL_TAR_SHA256_AARCH64" curl_sum="$CURL_SHA256_AARCH64" + deb_arch=arm64 bash_deb_sum="$BASH_DEB_SHA256_ARM64" bash_sum="$BASH_SHA256_ARM64" + ubuntu_pool=http://ports.ubuntu.com/ubuntu-ports/pool ;; - *) die "no pinned test tools for $(uname -m)" ;; + *) die "no pinned test tools for $(uname -m)" "pinned: x86_64 and aarch64 Linux" ;; esac +# need TOOL WHY: TOOL is on this host's PATH. +need() { + command -v "$1" > /dev/null || die "$1 is not on PATH, and $2 needs it" "reproduce: command -v $1" +} +need curl "every download" +need sha256sum "every checksum" +need tar "every archive" +for t in $tools; do + case "$t" in + shellcheck | curl) need xz "the $t archive" ;; + bash) + need zstd "the bash .deb" + need dd "the bash .deb" + ;; + esac +done + mkdir -p "$dest" work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT -# fetch URL FILE SHA256 +sum_of() { + sha256sum "$1" | cut -d ' ' -f 1 +} + +# fetch FILE SHA256 URL [URL ...]: the first URL whose bytes hash to SHA256. +# A URL that fails or serves other bytes is reported, and the next one is tried. fetch() { - local rc=0 got - curl -q --fail --location --silent --show-error --connect-timeout 20 --max-time 300 \ - --retry 3 --retry-connrefused --retry-delay 2 --output "$2" "$1" || rc=$? - [ "$rc" -eq 0 ] || die "could not download $1 (curl $rc)" "reproduce: curl -fLO $1" - got="$(sha256sum "$2" | cut -d ' ' -f 1)" - [ "$got" = "$3" ] || die "$(basename "$2") has sha256 $got, pinned $3" "reproduce: curl -fsSL $1 | sha256sum" + local file="$1" want="$2" url rc got tried=() + shift 2 + for url in "$@"; do + rc=0 + curl -q --fail --location --silent --show-error --connect-timeout 20 --max-time 300 \ + --retry 3 --retry-connrefused --retry-delay 2 --output "$file" "$url" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "install-test-tools: could not download $url (curl $rc)" >&2 + tried+=("curl $rc: $url") + continue + fi + got="$(sum_of "$file")" + if [ "$got" = "$want" ]; then + return 0 + fi + echo "install-test-tools: $url has sha256 $got, pinned $want" >&2 + tried+=("sha256 $got: $url") + rm -f "$file" + done + die "no source gave $(basename "$file") with sha256 $want" "${tried[@]}" \ + "reproduce: curl -fsSL '$1' | sha256sum" } -fetch "https://github.com/koalaman/shellcheck/releases/download/v$SHELLCHECK_VERSION/shellcheck-v$SHELLCHECK_VERSION.linux.$sc_arch.tar.xz" \ - "$work/shellcheck.tar.xz" "$sc_sum" -tar -xJf "$work/shellcheck.tar.xz" -C "$work" -install -m 0755 "$work/shellcheck-v$SHELLCHECK_VERSION/shellcheck" "$dest/shellcheck" +# check FILE SHA256 WHAT REPRODUCE: a file taken out of an archive is the +# one pinned. +check() { + local got + got="$(sum_of "$1")" + [ "$got" = "$2" ] || die "$3 has sha256 $got, pinned $2" \ + "the archive matched its own pin, so the member inside it is not the pinned one" \ + "reproduce: $4" +} + +# ar_member ARCHIVE NAME OUT: copy the member NAME of the ar(1) archive +# ARCHIVE to OUT. The format is an 8-byte magic, then for each member a +# 60-byte text header (name 0-15, size 48-57) and the data, padded to even. +ar_member() { + local archive="$1" want="$2" out="$3" off=8 total hdr name size + total="$(wc -c < "$archive")" + [ "$(dd if="$archive" bs=1 count=7 status=none)" = '!' ] || + die "$archive is not an ar archive" "reproduce: od -c $archive | sed -n 1p" + while [ "$((off + 60))" -le "$total" ]; do + hdr="$(dd if="$archive" bs=1 skip="$off" count=60 status=none)" + name="${hdr:0:16}" + name="${name%% *}" + name="${name%/}" + size="${hdr:48:10}" + size="${size%% *}" + case "$size" in + '' | *[!0-9]*) die "$archive: member header at byte $off has size '$size'" "reproduce: od -c -j $off -N 60 $archive" ;; + esac + if [ "$name" = "$want" ]; then + tail -c +"$((off + 61))" "$archive" > "$out.rest" + head -c "$size" "$out.rest" > "$out" + rm -f "$out.rest" + [ "$(wc -c < "$out")" -eq "$size" ] || + die "$archive: member $want is $size bytes, $(wc -c < "$out") were read" "reproduce: od -c -j $off -N 60 $archive" + return 0 + fi + off=$((off + 60 + size + size % 2)) + done + die "$archive has no member $want" "reproduce: bsdtar -tvf $archive" +} + +install_shellcheck() { + fetch "$work/shellcheck.tar.xz" "$sc_sum" \ + "https://github.com/koalaman/shellcheck/releases/download/v$SHELLCHECK_VERSION/shellcheck-v$SHELLCHECK_VERSION.linux.$sc_arch.tar.xz" + tar -xJf "$work/shellcheck.tar.xz" -C "$work" + install -m 0755 "$work/shellcheck-v$SHELLCHECK_VERSION/shellcheck" "$dest/shellcheck" + "$dest/shellcheck" --version | awk '/^version:/ { print "shellcheck " $2 }' +} -fetch "https://github.com/rhysd/actionlint/releases/download/v$ACTIONLINT_VERSION/actionlint_${ACTIONLINT_VERSION}_linux_$al_arch.tar.gz" \ - "$work/actionlint.tar.gz" "$al_sum" -tar -xzf "$work/actionlint.tar.gz" -C "$work" actionlint -install -m 0755 "$work/actionlint" "$dest/actionlint" +install_actionlint() { + fetch "$work/actionlint.tar.gz" "$al_sum" \ + "https://github.com/rhysd/actionlint/releases/download/v$ACTIONLINT_VERSION/actionlint_${ACTIONLINT_VERSION}_linux_$al_arch.tar.gz" + tar -xzf "$work/actionlint.tar.gz" -C "$work" actionlint + install -m 0755 "$work/actionlint" "$dest/actionlint" + "$dest/actionlint" -version | awk 'NR == 1 { print "actionlint " $1 }' +} + +install_jq() { + fetch "$work/jq" "$jq_sum" "https://github.com/jqlang/jq/releases/download/jq-$JQ_VERSION/jq-linux-$jq_arch" + install -m 0755 "$work/jq" "$dest/jq" + "$dest/jq" --version +} -fetch "https://github.com/jqlang/jq/releases/download/jq-$JQ_VERSION/jq-linux-$jq_arch" "$work/jq" "$jq_sum" -install -m 0755 "$work/jq" "$dest/jq" +install_curl() { + local url="https://github.com/stunnel/static-curl/releases/download/$CURL_VERSION/curl-linux-$curl_arch-musl-$CURL_VERSION.tar.xz" + fetch "$work/curl.tar.xz" "$curl_tar_sum" "$url" + mkdir -p "$work/curl" + tar -xJf "$work/curl.tar.xz" -C "$work/curl" curl + check "$work/curl/curl" "$curl_sum" "curl from curl-linux-$curl_arch-musl-$CURL_VERSION.tar.xz" \ + "curl -fsSL $url | tar -xJO curl | sha256sum" + install -m 0755 "$work/curl/curl" "$dest/curl" + "$dest/curl" --version | awk 'NR == 1 { print $1 " " $2 }' +} + +install_bash() { + local deb="bash-static_${BASH_DEB_VERSION}_$deb_arch.deb" + fetch "$work/bash.deb" "$bash_deb_sum" \ + "https://launchpad.net/ubuntu/+archive/primary/+files/$deb" \ + "$ubuntu_pool/universe/b/bash/$deb" + ar_member "$work/bash.deb" data.tar.zst "$work/bash-data.tar.zst" + zstd -q -d -o "$work/bash-data.tar" "$work/bash-data.tar.zst" + mkdir -p "$work/bash" + tar -xf "$work/bash-data.tar" -C "$work/bash" ./usr/bin/bash-static + check "$work/bash/usr/bin/bash-static" "$bash_sum" "usr/bin/bash-static from $deb" \ + "curl -fsSLO https://launchpad.net/ubuntu/+archive/primary/+files/$deb && bsdtar -xOf $deb data.tar.zst | zstd -dc | tar -xO ./usr/bin/bash-static | sha256sum" + install -m 0755 "$work/bash/usr/bin/bash-static" "$dest/bash" + "$dest/bash" --version | awk 'NR == 1 { print "bash " $4 }' +} -"$dest/shellcheck" --version | awk '/^version:/ { print "shellcheck " $2 }' -"$dest/actionlint" -version | awk 'NR == 1 { print "actionlint " $1 }' -"$dest/jq" --version +for t in $tools; do + "install_$t" +done diff --git a/tests/lib/harness.sh b/tests/lib/harness.sh index fd8be2a..2960cb3 100644 --- a/tests/lib/harness.sh +++ b/tests/lib/harness.sh @@ -3,6 +3,12 @@ # Output is TAP. Each test file is one plan. A failing assertion prints the # measurement and the command that reproduces it, so a CI log is enough to act # on without re-deriving anything. +# +# A test that needs a tool which may be absent calls need_tool. It is skipped +# where the tool is absent, with a TAP "# SKIP" line, unless +# TESTS_REQUIRE_TOOLS names the tool: then it fails. CI sets +# TESTS_REQUIRE_TOOLS in each job that has the tool, so a skip never hides +# there. if [ -z "${REPO_ROOT:-}" ]; then echo "harness: REPO_ROOT is not set" >&2 @@ -11,6 +17,7 @@ fi TESTS_RUN=0 TESTS_FAILED=0 +TESTS_SKIPPED=0 ok() { TESTS_RUN=$((TESTS_RUN + 1)) @@ -23,6 +30,13 @@ not_ok() { printf 'not ok %d - %s\n' "$TESTS_RUN" "$1" } +# skip DESC REASON +skip() { + TESTS_RUN=$((TESTS_RUN + 1)) + TESTS_SKIPPED=$((TESTS_SKIPPED + 1)) + printf 'ok %d - %s # SKIP %s\n' "$TESTS_RUN" "$1" "$2" +} + diag() { printf '# %s\n' "$1" } @@ -47,6 +61,24 @@ check() { fi } +# need_tool NAME DESC: 0 when NAME is on PATH. Otherwise the test DESC fails +# when TESTS_REQUIRE_TOOLS names NAME, or is skipped when it does not, and +# the return is 1. +need_tool() { + if command -v "$1" > /dev/null 2>&1; then + return 0 + fi + case " ${TESTS_REQUIRE_TOOLS:-} " in + *" $1 "*) + fail "$2" "$1 is not on PATH, and TESTS_REQUIRE_TOOLS names it" "reproduce: command -v $1" + ;; + *) + skip "$2" "$1 is not on PATH" + ;; + esac + return 1 +} + summary() { printf '1..%d\n' "$TESTS_RUN" if [ "$TESTS_RUN" -eq 0 ]; then @@ -57,7 +89,11 @@ summary() { printf '# failed %d of %d\n' "$TESTS_FAILED" "$TESTS_RUN" return 1 fi - printf '# passed %d of %d\n' "$TESTS_RUN" "$TESTS_RUN" + if [ "$TESTS_SKIPPED" -gt 0 ]; then + printf '# passed %d of %d, %d of them skipped\n' "$TESTS_RUN" "$TESTS_RUN" "$TESTS_SKIPPED" + else + printf '# passed %d of %d\n' "$TESTS_RUN" "$TESTS_RUN" + fi return 0 } @@ -109,9 +145,112 @@ runtime_detect() { export RUNTIME } -# Every script the repository ships, by path relative to REPO_ROOT. The -# scripts carry no extension, so a find by suffix would miss them. +#---------------------------------------------------------------------------# +# Which files are scripts, and in which language. The scripts carry no +# extension, so a find by suffix would miss them; the first line decides. +#---------------------------------------------------------------------------# + +# script_kind FILE -> sh, bash, python or other. +# A #! line names the interpreter, through env(1) or not. +# With no #! line, a .sh file is bash (sourced, like scripts/lib.sh) and a +# .py file is python (imported). Anything else is other. +script_kind() { + local file="$1" first="" interp="" w words=() + first="$(LC_ALL=C sed -n '1p' "$file" | tr -d '\000\r')" + case "$first" in + '#!'*) + read -r -a words <<< "${first#'#!'}" + interp="${words[0]:-}" + interp="${interp##*/}" + if [ "$interp" = env ]; then + interp="" + for w in "${words[@]:1}"; do + case "$w" in + -* | *=*) ;; + *) + interp="${w##*/}" + break + ;; + esac + done + fi + case "$interp" in + sh | dash | ash | ksh | mksh | posh) echo sh ;; + bash) echo bash ;; + python | python3 | python3.*) echo python ;; + *) echo other ;; + esac + ;; + *) + case "$file" in + *.sh) echo bash ;; + *.py) echo python ;; + *) echo other ;; + esac + ;; + esac +} + +# script_has_shebang FILE: 0 when the first line starts with #!. +script_has_shebang() { + [ "$(LC_ALL=C head -c 2 "$1")" = '#!' ] +} + +# Every file the repository ships under client/ and scripts/, by path +# relative to REPO_ROOT. A __pycache__ directory is never shipped. +shipped_files() { + ( + cd "$REPO_ROOT" || exit 2 + find client scripts -type f ! -path '*/__pycache__/*' | LC_ALL=C sort + ) +} + +# _files_of_kind KIND... reads paths on stdin, prints those of a KIND. +_files_of_kind() { + local f k want + while IFS= read -r f; do + [ -n "$f" ] || continue + k="$(script_kind "$REPO_ROOT/$f")" + for want in "$@"; do + if [ "$k" = "$want" ]; then + printf '%s\n' "$f" + break + fi + done + done +} + +# The shipped shell scripts: sh or bash by their first line. shipped_scripts() { - printf '%s\n' client/pacman-oci - find "$REPO_ROOT/scripts" -type f | sed "s|^$REPO_ROOT/||" | sort + shipped_files | _files_of_kind sh bash +} + +# The shipped python scripts, linted by tests/static/70-python.sh. +shipped_python() { + shipped_files | _files_of_kind python +} + +# Shipped files that are neither, which no static test reads. +shipped_other() { + shipped_files | _files_of_kind other +} + +# Every file under tests/, by path relative to REPO_ROOT. +_test_files() { + ( + cd "$REPO_ROOT" || exit 2 + find tests -type f ! -path '*/__pycache__/*' ! -path '*/node_modules/*' | LC_ALL=C sort + ) +} + +# The shell scripts of the test suites: run.sh, this harness and each +# suite's files, by their first line. ⚠ Not yet tests/fixtures: its scripts +# (build-repo) do not pass shellcheck today. +test_scripts() { + _test_files | grep -v '^tests/fixtures/' | _files_of_kind sh bash +} + +# The python scripts under tests/, fixtures included, by their first line. +test_python() { + _test_files | _files_of_kind python } diff --git a/tests/run.sh b/tests/run.sh index e771d6f..d22f6ca 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -2,47 +2,141 @@ # # Test entry point. # -# tests/run.sh static the tree: shellcheck, actionlint, policy -# tests/run.sh unit the wrapper and the scripts, no network -# tests/run.sh integration a local registry, oras and pacman in a container -# tests/run.sh all all three +# tests/run.sh [SUITE [FILE]] +# tests/run.sh --list SUITE # -# static needs shellcheck and actionlint on PATH. integration needs docker or +# SUITE static the tree: shellcheck, actionlint, py_compile, policy +# unit the wrapper and the scripts, no network (the default) +# integration a local registry, oras and pacman in a container +# worker the Cloudflare Worker under node --test +# all the four suites, in that order +# FILE one file of SUITE, by name (20-client.sh, or 20-client) or by path +# (tests/integration/20-client.sh). SUITE all takes no FILE. +# --list prints the name of each file SUITE runs, one per line. CI makes its +# integration matrix from it, so a new file gets a job with no edit. +# +# The files of a suite are tests/SUITE/*.sh, found when the run starts. +# ⚠ A file named live.sh talks to a deployed service. It runs only when +# _LIVE=1 (WORKER_LIVE=1 for tests/worker/live.sh), also when named. +# +# static needs shellcheck, actionlint and jq on PATH (scripts/install-test-tools), +# and python3 when there is python to compile. integration needs docker or # podman (CONTAINER_RUNTIME picks one) and network to pull the pinned images. +# worker needs node 22 or newer. +# +# Exit status: 0 every file passed, 1 a file failed or no file ran, 2 usage. set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" export REPO_ROOT +SUITES="static unit integration worker" + +usage() { + echo "usage: tests/run.sh [static|unit|integration|worker|all [FILE]]" >&2 + echo " tests/run.sh --list static|unit|integration|worker" >&2 + exit 2 +} + +list=0 +if [ "${1:-}" = --list ]; then + list=1 + shift + [ "$#" -eq 1 ] || usage +fi +[ "$#" -le 2 ] || usage suite="${1:-unit}" +file="${2:-}" case "$suite" in - static | unit | integration) suites="$suite" ;; - all) suites="static unit integration" ;; + all) + [ "$list" -eq 0 ] && [ -z "$file" ] || usage + suites="$SUITES" + ;; *) - echo "usage: tests/run.sh [static|unit|integration|all]" >&2 - exit 2 + case " $SUITES " in + *" $suite "*) suites="$suite" ;; + *) usage ;; + esac ;; esac +# gated SUITE NAME: 0 when NAME is a live test whose switch is off. +gated() { + local switch + [ "$2" = live.sh ] || return 1 + switch="$(printf '%s' "$1" | tr '[:lower:]' '[:upper:]')_LIVE" + [ "${!switch:-0}" != 1 ] +} + +# files_of SUITE: the test files of SUITE, by path, sorted. +files_of() { + local t + for t in "$REPO_ROOT/tests/$1"/*.sh; do + [ -e "$t" ] || continue + printf '%s\n' "$t" + done +} + +if [ -n "$file" ]; then + name="$(basename "$file")" + case "$name" in + *.sh) ;; + *) name="$name.sh" ;; + esac + if [ ! -f "$REPO_ROOT/tests/$suite/$name" ]; then + echo "tests/run.sh: no file $name in tests/$suite" >&2 + echo " the files there:" >&2 + files_of "$suite" | sed "s|^$REPO_ROOT/| |" >&2 + exit 2 + fi +fi + +if [ "$list" -eq 1 ]; then + while IFS= read -r t; do + [ -n "$t" ] || continue + if ! gated "$suite" "$(basename "$t")"; then + basename "$t" + fi + done <<< "$(files_of "$suite")" + exit 0 +fi + failed_files=0 total_files=0 +failed_names=() for s in $suites; do + # A glob, not a read loop, so each test keeps the caller's stdin. for t in "$REPO_ROOT/tests/$s"/*.sh; do [ -e "$t" ] || continue + n="$(basename "$t")" + if [ -n "$file" ] && [ "$n" != "$name" ]; then + continue + fi + if gated "$s" "$n"; then + s_upper="$(printf '%s' "$s" | tr '[:lower:]' '[:upper:]')" + printf '\n# ===== %s/%s: not run, it needs %s_LIVE=1 =====\n' "$s" "$n" "$s_upper" + continue + fi total_files=$((total_files + 1)) - printf '\n# ===== %s/%s =====\n' "$s" "$(basename "$t")" + printf '\n# ===== %s/%s =====\n' "$s" "$n" + start="$SECONDS" if ! bash "$t"; then failed_files=$((failed_files + 1)) + failed_names+=("$s/$n") fi + printf '# ----- %s/%s took %d s\n' "$s" "$n" "$((SECONDS - start))" done done printf '\n# ===== summary =====\n' if [ "$total_files" -eq 0 ]; then - printf '# no test files found for suite: %s\n' "$suites" + printf '# no test file ran for suite: %s%s\n' "$suites" "${file:+, file $file}" exit 1 fi printf '# test files: %d, failed: %d\n' "$total_files" "$failed_files" +for n in "${failed_names[@]}"; do + printf '# failed: %s, reproduce: tests/run.sh %s %s\n' "$n" "${n%%/*}" "${n#*/}" +done [ "$failed_files" -eq 0 ] diff --git a/tests/static/10-shellcheck.sh b/tests/static/10-shellcheck.sh index cbeed3d..45c8ac6 100644 --- a/tests/static/10-shellcheck.sh +++ b/tests/static/10-shellcheck.sh @@ -1,10 +1,15 @@ #!/usr/bin/env bash # -# Every shell file passes shellcheck, the wrapper as POSIX sh. +# Every shell file passes shellcheck, in the dialect its first line names: the +# wrapper as POSIX sh, the rest as bash. # # The wrapper runs under whatever /bin/sh a consumer has (bash on Arch, dash # and busybox elsewhere), so a bashism in it is a bug that only shows on # someone else's machine. +# +# The files are found, not listed: every shell script under client/ and +# scripts/, and every one in the test suites (run.sh, the harness and each +# suite's files). tests/lib/harness.sh script_kind decides what is shell. set -euo pipefail # shellcheck source=../lib/harness.sh . "$REPO_ROOT/tests/lib/harness.sh" @@ -18,20 +23,12 @@ fi cd "$REPO_ROOT" files="$( shipped_scripts - printf '%s\n' tests/run.sh tests/lib/harness.sh - for d in tests/static tests/unit tests/integration; do - if [ -d "$d" ]; then - find "$d" -name '*.sh' | sort - fi - done + test_scripts )" while IFS= read -r f; do [ -n "$f" ] || continue - case "$f" in - client/pacman-oci) shell='sh' ;; - *) shell='bash' ;; - esac + shell="$(script_kind "$f")" out="$(shellcheck -x --source-path=SCRIPTDIR -s "$shell" "$f" 2>&1)" && rc=0 || rc=$? if [ "$rc" -eq 0 ]; then ok "shellcheck -s $shell $f" @@ -40,4 +37,14 @@ while IFS= read -r f; do fi done <<< "$files" +# The wrapper and the files every suite depends on are found by the walk +# above. A walk that misses them misses others too. +for f in client/pacman-oci tests/run.sh tests/lib/harness.sh; do + if [[ $'\n'"$files"$'\n' == *$'\n'"$f"$'\n'* ]]; then + ok "the walk finds $f" + else + fail "the walk finds $f" "script_kind says $(script_kind "$f")" "reproduce: . tests/lib/harness.sh; shipped_scripts; test_scripts" + fi +done + summary diff --git a/tests/static/20-actionlint.sh b/tests/static/20-actionlint.sh index 8a94e21..0b6ddc5 100644 --- a/tests/static/20-actionlint.sh +++ b/tests/static/20-actionlint.sh @@ -15,7 +15,7 @@ fi cd "$REPO_ROOT" found=0 -for f in .github/workflows/*.yml; do +for f in .github/workflows/*.yml .github/workflows/*.yaml; do [ -e "$f" ] || continue found=$((found + 1)) out="$(actionlint -no-color "$f" 2>&1)" && rc=0 || rc=$? @@ -26,7 +26,7 @@ for f in .github/workflows/*.yml; do fi done if [ "$found" -eq 0 ]; then - fail "there are workflows to lint" "no .github/workflows/*.yml" "reproduce: ls .github/workflows" + fail "there are workflows to lint" "no .github/workflows/*.yml or *.yaml" "reproduce: ls .github/workflows" fi summary diff --git a/tests/static/40-executable-bits.sh b/tests/static/40-executable-bits.sh index 0580e08..1ef7ef8 100644 --- a/tests/static/40-executable-bits.sh +++ b/tests/static/40-executable-bits.sh @@ -3,6 +3,10 @@ # Every script is committed executable. A workflow step that runs # scripts/plan from a checkout whose index lost the bit fails with 126, and a # consumer who downloads the wrapper from a raw URL gets whatever mode git had. +# +# The rule, for every file under client/ and scripts/: a file with a #! line +# is 100755, a file without one is a library that is sourced or imported +# (scripts/lib.sh) and is 100644. tests/run.sh is 100755. set -euo pipefail # shellcheck source=../lib/harness.sh . "$REPO_ROOT/tests/lib/harness.sh" @@ -16,16 +20,21 @@ fi while IFS= read -r f; do [ -n "$f" ] || continue - case "$f" in - scripts/lib.sh) want=100644 ;; - *) want=100755 ;; - esac + if script_has_shebang "$f"; then + want=100755 + else + want=100644 + fi mode="$(git ls-files -s -- "$f" | awk '{ print $1 }')" if [ -z "$mode" ]; then fail "$f is in the git index" "git ls-files does not list it" "reproduce: git ls-files -s -- $f" else check "$f is mode $want in the index" "$want" "$mode" "git ls-files -s -- $f" fi -done <<< "$(shipped_scripts; printf '%s\n' tests/run.sh)" +done <<< "$(shipped_files; printf '%s\n' tests/run.sh)" + +# The library rule above must still give the one sourced library its mode. +check "scripts/lib.sh has no #! line, so it stays 100644" 1 \ + "$(script_has_shebang scripts/lib.sh && echo 0 || echo 1)" "sed -n 1p scripts/lib.sh" summary diff --git a/tests/static/50-workflow-policy.sh b/tests/static/50-workflow-policy.sh index 84d7160..5e62087 100644 --- a/tests/static/50-workflow-policy.sh +++ b/tests/static/50-workflow-policy.sh @@ -6,9 +6,14 @@ # any branch can ask for packages: write. These rules keep that write narrow # and visible: # - permissions: {} at the top, and each job asks for what it needs +# - pages: write and id-token: write only in status.yml's deploy job, and +# contents: write only in status.yml's build job # - shell: bash by default, so every run: block has -eo pipefail -# - timeout-minutes on every job, so a hang ends in minutes, not six hours -# - actions pinned by commit, and checkouts that keep no token on disk +# - timeout-minutes on every job, so a hang ends in minutes, not six hours. +# A job that calls a reusable workflow cannot have one; the jobs of the +# workflow it calls are held to the rule instead. +# - actions pinned by commit, and checkouts that keep no token on disk. A +# local path (./.github/workflows/x.yml) is the same commit, so it is pinned. # - no ${{ inputs, matrix, needs or event }} inside a run: block, where a # value would become shell code; those go through env: # - no pull_request_target or workflow_run trigger, which run with a write @@ -16,15 +21,56 @@ # - no use of the packages DELETE API anywhere: nothing is ever deleted # - no unquoted `if: !`, which YAML reads as a tag # - nothing needs the jobs that watch the others +# - every workflow with a schedule: is one scripts/keep-schedules-enabled +# enables set -euo pipefail # shellcheck source=../lib/harness.sh . "$REPO_ROOT/tests/lib/harness.sh" cd "$REPO_ROOT" +work="$(work_dir)" +trap 'rm -rf "$work"' EXIT + +# job_permissions WORKFLOW -> "jobscopelevel" for each permission a +# job asks for, in block form or in flow form ({ a: read, b: write }). +job_permissions() { + awk ' + function emit(pair, kv) { + gsub(/[[:space:]]/, "", pair) + if (pair == "") return + split(pair, kv, ":") + print job "\t" kv[1] "\t" kv[2] + } + /^jobs:/ { injobs = 1; next } + injobs && /^[^[:space:]#]/ { injobs = 0 } + !injobs { next } + /^ [A-Za-z0-9_-]+:[[:space:]]*(#.*)?$/ { job = $1; sub(/:$/, "", job); inperm = 0; next } + /^ permissions:/ { + rest = $0 + sub(/^ permissions:[[:space:]]*/, "", rest) + sub(/[[:space:]]*#.*$/, "", rest) + inperm = (rest == "") + if (rest ~ /^\{/) { + gsub(/[{}]/, "", rest) + n = split(rest, pairs, ",") + for (i = 1; i <= n; i++) emit(pairs[i]) + } else if (rest != "") { + print job "\t*\t" rest + } + next + } + inperm && /^ [A-Za-z-]+:/ { line = $0; sub(/[[:space:]]*#.*$/, "", line); emit(line); next } + inperm && /^[[:space:]]*(#.*)?$/ { next } + inperm { inperm = 0 } + ' "$1" +} + found=0 -for wf in .github/workflows/*.yml; do +workflows=() +for wf in .github/workflows/*.yml .github/workflows/*.yaml; do [ -e "$wf" ] || continue found=$((found + 1)) + workflows+=("$wf") if grep -qE '^permissions:[[:space:]]*\{\}[[:space:]]*$' "$wf"; then ok "$wf: top-level permissions: {}" @@ -38,6 +84,27 @@ for wf in .github/workflows/*.yml; do fail "$wf: no blanket write-all or read-all" "$bad" "reproduce: grep -n 'permissions:' $wf" fi + # Pages and the OIDC token deploy the status site; contents: write pushes + # the status branch. Each is one job's, in one workflow. + bad="" + while IFS=$'\t' read -r job scope level; do + [ -n "$job" ] || continue + case "$scope:$level" in + pages:write | id-token:write) allowed=status.yml:deploy ;; + contents:write) allowed=status.yml:build ;; + *) continue ;; + esac + if [ "$(basename "$wf"):$job" != "$allowed" ]; then + bad="${bad}job $job asks for $scope: $level, which only $allowed may; " + fi + done <<< "$(job_permissions "$wf")" + if [ -z "$bad" ]; then + ok "$wf: pages, id-token and contents write only where the status site needs them" + else + fail "$wf: pages, id-token and contents write only where the status site needs them" "$bad" \ + "reproduce: grep -n -A6 'permissions:' $wf" + fi + if awk '/^defaults:/ { d = 1; next } d && /^ run:/ { r = 1; next } d && r && /^ shell: bash[[:space:]]*$/ { found = 1 } /^[a-z]/ { d = 0; r = 0 } END { exit !found }' "$wf"; then ok "$wf: defaults run shell bash" else @@ -47,9 +114,10 @@ for wf in .github/workflows/*.yml; do missing="$(awk ' /^jobs:/ { in_jobs = 1; next } in_jobs && /^[a-z]/ { in_jobs = 0 } - in_jobs && /^ [A-Za-z0-9_-]+:[[:space:]]*$/ { if (job != "" && !t) print job; job = $1; t = 0; next } + in_jobs && /^ [A-Za-z0-9_-]+:[[:space:]]*$/ { if (job != "" && !t && !u) print job; job = $1; t = 0; u = 0; next } in_jobs && /^ timeout-minutes:/ { t = 1 } - END { if (job != "" && !t) print job } + in_jobs && /^ uses:/ { u = 1 } + END { if (job != "" && !t && !u) print job } ' "$wf")" if [ -z "$missing" ]; then ok "$wf: every job has timeout-minutes" @@ -57,7 +125,7 @@ for wf in .github/workflows/*.yml; do fail "$wf: every job has timeout-minutes" "missing: $missing" "reproduce: grep -n timeout-minutes $wf" fi - unpinned="$(grep -nE '^[[:space:]]*(-[[:space:]]*)?uses:' "$wf" | grep -vE 'uses:[[:space:]]*[^@[:space:]]+@[0-9a-f]{40}([[:space:]]|$)' || [ "$?" -eq 1 ])" + unpinned="$(grep -nE '^[[:space:]]*(-[[:space:]]*)?uses:' "$wf" | grep -vE 'uses:[[:space:]]*(\./[^@[:space:]]+|[^@[:space:]]+@[0-9a-f]{40})([[:space:]]|$)' || [ "$?" -eq 1 ])" if [ -z "$unpinned" ]; then ok "$wf: every action is pinned by commit" else @@ -115,7 +183,7 @@ for wf in .github/workflows/*.yml; do fi done if [ "$found" -eq 0 ]; then - fail "there are workflows" "no .github/workflows/*.yml" "reproduce: ls .github/workflows" + fail "there are workflows" "no .github/workflows/*.yml or *.yaml" "reproduce: ls .github/workflows" fi #---------------------------------------------------------------------------# @@ -128,4 +196,87 @@ else fail "no script or workflow deletes a package or a version" "$deleters" "reproduce: grep -rn DELETE scripts .github client" fi +#---------------------------------------------------------------------------# +# Every scheduled workflow is one scripts/keep-schedules-enabled enables. +# +# ⛔ GitHub turns off the schedules of a repository with 60 days of no +# activity. A scheduled workflow that the script does not enable again stops, +# and leaves no run behind to say so. The script runs here against a stub gh: +# the stub lists every workflow in the tree as active, with an id, and records +# each enable call. So the rule holds whether the script names the workflows +# or finds them. +#---------------------------------------------------------------------------# +scheduled=() +for wf in "${workflows[@]}"; do + if awk '/^on:/ { on = 1; next } on && /^[^[:space:]#]/ { on = 0 } on && /^ schedule:/ { s = 1 } END { exit !s }' "$wf"; then + scheduled+=("$wf") + fi +done + +if [ "${#scheduled[@]}" -eq 0 ]; then + ok "no workflow has a schedule, so none needs keep-schedules-enabled" +elif need_tool jq "keep-schedules-enabled enables every scheduled workflow (the script and the stub need jq)"; then + mkdir -p "$work/bin" + : > "$work/enabled.log" + : > "$work/calls.log" + i=0 + for wf in "${workflows[@]}"; do + i=$((i + 1)) + printf '%s\t%s\n' "$((1000 + i))" "$wf" + done > "$work/ids.tsv" + jq -Rn '[inputs | split("\t") | {id: (.[0] | tonumber), name: .[1], path: .[1], state: "active"}] | {total_count: length, workflows: .}' \ + < "$work/ids.tsv" > "$work/workflows.json" + cat > "$work/bin/gh" << 'EOF' +#!/usr/bin/env bash +# A stand-in for gh: answers the workflow list, records enable calls, refuses +# anything else by name. +set -euo pipefail +printf 'gh %s\n' "$*" >> "$STUB_DIR/calls.log" +args="$*" +jq_expr="" +prev="" +for a in "$@"; do + case "$prev" in + --jq | -q) jq_expr="$a" ;; + esac + prev="$a" +done +if [[ $args =~ (-X|--method)[[:space:]=]*PUT[[:space:]].*actions/workflows/([^/[:space:]]+)/enable ]]; then + printf '%s\n' "${BASH_REMATCH[2]}" >> "$STUB_DIR/enabled.log" +elif [[ $args =~ ^workflow[[:space:]]+enable[[:space:]]+([^[:space:]]+) ]]; then + printf '%s\n' "${BASH_REMATCH[1]}" >> "$STUB_DIR/enabled.log" +elif [[ $args =~ ^api[[:space:]].*actions/workflows(\?[^[:space:]]*)?([[:space:]]|$) ]]; then + if [ -n "$jq_expr" ]; then + jq -r "$jq_expr" "$STUB_DIR/workflows.json" + else + cat "$STUB_DIR/workflows.json" + fi +else + echo "stub gh: no answer for: gh $args" >&2 + exit 1 +fi +EOF + chmod +x "$work/bin/gh" + rc=0 + out="$(STUB_DIR="$work" GH_TOKEN=stub GITHUB_REPOSITORY=pkgforge-dev/mirror-archlinux PATH="$work/bin:$PATH" \ + scripts/keep-schedules-enabled 2>&1)" || rc=$? + if [ "$rc" -ne 0 ]; then + fail "keep-schedules-enabled runs against the stub gh" "exit $rc" "$(printf '%s\n' "$out" | sed -n 1,10p)" \ + "calls: $(tr '\n' ';' < "$work/calls.log")" \ + "reproduce: this file's stub, then GH_TOKEN=x GITHUB_REPOSITORY=o/r PATH=stub:\$PATH scripts/keep-schedules-enabled" + else + for wf in "${scheduled[@]}"; do + id="$(awk -F '\t' -v p="$wf" '$2 == p { print $1 }' "$work/ids.tsv")" + if grep -qxF -e "$id" -e "$(basename "$wf")" -e "$wf" "$work/enabled.log"; then + ok "$wf has a schedule, and keep-schedules-enabled enables it" + else + fail "$wf has a schedule, and keep-schedules-enabled enables it" \ + "the stub listed it as id $id; the script enabled: $(tr '\n' ' ' < "$work/enabled.log")" \ + "GitHub turns its schedule off after 60 days of no activity, and nothing turns it on again" \ + "reproduce: grep -n schedule: $wf; read scripts/keep-schedules-enabled" + fi + done + fi +fi + summary diff --git a/tests/static/70-python.sh b/tests/static/70-python.sh new file mode 100644 index 0000000..a96964c --- /dev/null +++ b/tests/static/70-python.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# +# Every python file in the tree compiles, and every file under client/ and +# scripts/ is a script that some static test reads. +# +# The shell tests read only shell: shellcheck, the failure and pipeline rules. +# A shipped python script (scripts/status-render) is compiled here with +# python3 -I -m py_compile, the way it runs: isolated, standard library only. +# A shipped file whose first line names no known language would pass every +# static test unread, so it fails here instead. +set -euo pipefail +# shellcheck source=../lib/harness.sh +. "$REPO_ROOT/tests/lib/harness.sh" + +cd "$REPO_ROOT" +work="$(work_dir)" +trap 'rm -rf "$work"' EXIT + +#---------------------------------------------------------------------------# +# Every shipped file is shell or python. +#---------------------------------------------------------------------------# +other="$(shipped_other)" +if [ -z "$other" ]; then + ok "every file under client/ and scripts/ is a shell or a python script" +else + while IFS= read -r f; do + fail "$f is a shell or a python script" \ + "its first line is: $(LC_ALL=C sed -n 1p "$f" | tr -d '\000\r' | cut -c1-80)" \ + "no static test reads it; give it a #! line naming sh, bash or python3" \ + "reproduce: sed -n 1p $f" + done <<< "$other" +fi + +#---------------------------------------------------------------------------# +# Every python file compiles. -X pycache_prefix keeps the bytecode out of the +# tree: python3 -I ignores PYTHONPYCACHEPREFIX, and a __pycache__ under +# scripts/ would be a shipped file of no known kind. +#---------------------------------------------------------------------------# +files="$( + shipped_python + test_python +)" +if [ -z "$files" ]; then + ok "there is no python in the tree, so there is nothing to compile" +elif need_tool python3 "python3 is on PATH, to compile the python in the tree"; then + ok "python3 is on PATH: $(python3 -I -c 'import sys; print(sys.version.split()[0])')" + while IFS= read -r f; do + [ -n "$f" ] || continue + out="$(python3 -I -X pycache_prefix="$work/pycache" -m py_compile "$f" 2>&1)" && rc=0 || rc=$? + if [ "$rc" -eq 0 ]; then + ok "python3 -I -m py_compile $f" + else + fail "python3 -I -m py_compile $f" "$(printf '%s\n' "$out" | sed -n 1,20p)" "reproduce: python3 -I -m py_compile $f" + fi + done <<< "$files" +fi + +summary diff --git a/tests/static/72-test-runner.sh b/tests/static/72-test-runner.sh new file mode 100644 index 0000000..38fbbcb --- /dev/null +++ b/tests/static/72-test-runner.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# +# tests/run.sh finds the files of a suite, runs one file when named, keeps a +# live test off unless its switch is on, and fails when a file fails or when +# no file ran. +# +# ci.yml makes one integration job per line of `tests/run.sh --list +# integration`. A file the list misses gets no job and is never run, and the +# run is still green. So the list is checked here, in a tree of its own with +# known files. +# +# ⚠ A run.sh that hides failures also hides this file's failure. So ci.yml +# runs this file once with bash directly, before tests/run.sh static. +set -euo pipefail +# shellcheck source=../lib/harness.sh +. "$REPO_ROOT/tests/lib/harness.sh" + +work="$(work_dir)" +trap 'rm -rf "$work"' EXIT + +tree="$work/tree" +mkdir -p "$tree/tests/lib" "$tree/tests/integration" "$tree/tests/worker" "$tree/tests/unit" +cp "$REPO_ROOT/tests/run.sh" "$tree/tests/run.sh" +cp "$REPO_ROOT/tests/lib/harness.sh" "$tree/tests/lib/harness.sh" + +# passing_test FILE: a test file of one passing assertion that logs its name. +passing_test() { + cat > "$1" << 'EOF' +#!/usr/bin/env bash +set -euo pipefail +. "$REPO_ROOT/tests/lib/harness.sh" +printf '%s\n' "$(basename "$0")" >> "$REPO_ROOT/ran.log" +ok "a test that passes" +summary +EOF +} +passing_test "$tree/tests/integration/10-first.sh" +passing_test "$tree/tests/integration/20-second.sh" +passing_test "$tree/tests/worker/10-unit.sh" +passing_test "$tree/tests/worker/live.sh" +: > "$tree/tests/integration/notes.txt" + +# run ARGS...: run.sh in the tree; output in $out, exit code in $rc, the files +# that ran in $ran (space separated). +run() { + rc=0 + : > "$tree/ran.log" + out="$(env -u WORKER_LIVE -u INTEGRATION_LIVE bash "$tree/tests/run.sh" "$@" 2>&1)" || rc=$? + ran="$(tr '\n' ' ' < "$tree/ran.log")" +} + +run --list integration +check "--list integration names each tests/integration/*.sh, in order" "10-first.sh 20-second.sh" \ + "$(printf '%s\n' "$out" | tr '\n' ' ' | sed 's/ $//')" "tests/run.sh --list integration" +check "--list integration exits 0" 0 "$rc" "tests/run.sh --list integration" + +run --list worker +check "--list worker leaves out live.sh while WORKER_LIVE is not 1" "10-unit.sh" \ + "$(printf '%s\n' "$out" | tr '\n' ' ' | sed 's/ $//')" "tests/run.sh --list worker" + +rc=0 +out="$(WORKER_LIVE=1 bash "$tree/tests/run.sh" --list worker 2>&1)" || rc=$? +check "--list worker names live.sh when WORKER_LIVE=1" "10-unit.sh live.sh" \ + "$(printf '%s\n' "$out" | tr '\n' ' ' | sed 's/ $//')" "WORKER_LIVE=1 tests/run.sh --list worker" + +run integration +check "integration runs every file of the suite" "10-first.sh 20-second.sh " "$ran" "tests/run.sh integration" +check "integration exits 0 when every file passes" 0 "$rc" "tests/run.sh integration" + +run integration 20-second.sh +check "a named file runs alone" "20-second.sh " "$ran" "tests/run.sh integration 20-second.sh" +run integration 20-second +check "a named file runs alone, named without .sh" "20-second.sh " "$ran" "tests/run.sh integration 20-second" +run integration tests/integration/10-first.sh +check "a named file runs alone, named by path" "10-first.sh " "$ran" "tests/run.sh integration tests/integration/10-first.sh" + +run integration 30-absent.sh +check "a file that is not in the suite is a usage error" 2 "$rc" "tests/run.sh integration 30-absent.sh" + +run worker +check "worker does not run live.sh while WORKER_LIVE is not 1" "10-unit.sh " "$ran" "tests/run.sh worker" +run worker live.sh +check "a named live.sh does not run either" "" "$ran" "tests/run.sh worker live.sh" +check "and the run fails, since no file ran" 1 "$rc" "tests/run.sh worker live.sh" + +run unit +check "a suite with no file fails" 1 "$rc" "tests/run.sh unit (in a tree with no unit test)" + +cat > "$tree/tests/integration/30-fails.sh" << 'EOF' +#!/usr/bin/env bash +set -euo pipefail +. "$REPO_ROOT/tests/lib/harness.sh" +fail "a test that fails" +summary +EOF +run integration +check "a failing file fails the run" 1 "$rc" "tests/run.sh integration (with a failing file)" +check "and every other file still runs" "10-first.sh 20-second.sh " "$ran" "tests/run.sh integration" +case "$out" in + *"failed: integration/30-fails.sh, reproduce: tests/run.sh integration 30-fails.sh"*) + ok "the summary names the failed file and the command that runs it alone" + ;; + *) + fail "the summary names the failed file and the command that runs it alone" "$(printf '%s\n' "$out" | tail -n 4)" \ + "reproduce: tests/run.sh integration" + ;; +esac + +run all 10-first.sh +check "all takes no FILE" 2 "$rc" "tests/run.sh all 10-first.sh" +run nosuch +check "an unknown suite is a usage error" 2 "$rc" "tests/run.sh nosuch" + +summary diff --git a/tests/static/75-renovate.sh b/tests/static/75-renovate.sh new file mode 100644 index 0000000..08c9f61 --- /dev/null +++ b/tests/static/75-renovate.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +# +# Renovate can move every image pin in the tree, and merges on its own only +# what CI can judge. +# +# An image is pinned as name:tag@sha256:<64 hex>, under a "# tag: " +# marker line at most three lines before it. The regex manager in +# .github/renovate.json reads that shape, in the files its managerFilePatterns +# name. A pin that it does not match never moves, and nothing says so. Each +# pin is checked here three ways: its file is one the manager reads, the +# manager's regex matches it, and the marker's tag is the pin's tag. +# +# The automerge rules are checked too: +# - no rule automerges a major update +# - platformAutomerge is false: main has no required checks, so GitHub's +# own auto-merge would merge a pull request whose checks are red +# - ignorePaths is set and names no tests directory: the preset default +# ignores **/tests/**, and the test pins would never move +set -euo pipefail +# shellcheck source=../lib/harness.sh +. "$REPO_ROOT/tests/lib/harness.sh" + +cd "$REPO_ROOT" +CONFIG=.github/renovate.json + +# Where images are run from: the targets, the workflows, the scripts and the +# test suites. Fixtures are data, and docs are prose. +SCAN_DIRS=(targets .github/workflows scripts client tests) +PIN='(docker\.io|ghcr\.io|quay\.io|gcr\.io|registry\.k8s\.io|public\.ecr\.aws)/[a-z0-9._/-]+:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}' + +if ! need_tool jq "$CONFIG is read with jq"; then + summary + exit 0 +fi + +if out="$(jq -e . "$CONFIG" 2>&1 > /dev/null)"; then + ok "$CONFIG is JSON" +else + fail "$CONFIG is JSON" "$out" "reproduce: jq . $CONFIG" + summary + exit 1 +fi + +#---------------------------------------------------------------------------# +# The automerge rules. +#---------------------------------------------------------------------------# +majors="$(jq -r '.packageRules // [] | to_entries[] | select(.value.automerge == true) | select((.value.matchUpdateTypes // ["(all)"]) | any(. == "major" or . == "(all)")) | "packageRules[\(.key)]"' "$CONFIG")" +check "no rule in $CONFIG automerges a major update" "" "$majors" "jq '.packageRules' $CONFIG" +check "$CONFIG sets platformAutomerge to false" false "$(jq -r '.platformAutomerge' "$CONFIG")" "jq .platformAutomerge $CONFIG" + +# The keyring images are the trust root of the non-x86_64 targets, and no CI +# job runs them. Renovate applies packageRules in order and a later rule +# wins, so a rule that names them with automerge false comes after every +# rule with automerge true. +keyring="$(jq -r ' + (.packageRules // []) | to_entries as $r + | ([$r[] | select(.value.automerge == true) | .key] | max // -1) as $on + | ([$r[] | select((.value.matchDepNames // []) | index("ghcr.io/pkgforge-dev/archlinux")) | select(.value.automerge == false) | .key] | max // -1) as $off + | if $off < 0 then "no rule names them with automerge false" + elif $off < $on then "packageRules[\($on)] comes after packageRules[\($off)] and turns automerge on" + else "a person merges them" end' "$CONFIG")" +check "the keyring images (ghcr.io/pkgforge-dev/archlinux) never merge on their own" "a person merges them" "$keyring" \ + "jq '.packageRules' $CONFIG" +ignore="$(jq -r '.ignorePaths // ["(unset)"] | .[]' "$CONFIG")" +case "$ignore" in + *'(unset)'* | *test*) + fail "$CONFIG sets ignorePaths, and no entry is a tests directory" "ignorePaths: $(printf '%s ' "$ignore")" \ + "config:recommended ignores **/tests/** when ignorePaths is unset" "reproduce: jq .ignorePaths $CONFIG" + ;; + *) ok "$CONFIG sets ignorePaths, and no entry is a tests directory" ;; +esac + +#---------------------------------------------------------------------------# +# Every image pin is one the regex manager moves. +#---------------------------------------------------------------------------# +patterns="$(jq -r '.customManagers // [] | .[] | select(.customType == "regex" and .datasourceTemplate == "docker") | .managerFilePatterns[]' "$CONFIG")" +regexes="$(jq -r '.customManagers // [] | .[] | select(.customType == "regex" and .datasourceTemplate == "docker") | .matchStrings[]' "$CONFIG")" +if [ -z "$patterns" ] || [ -z "$regexes" ]; then + fail "$CONFIG has a docker regex manager with file patterns and match strings" \ + "patterns: ${patterns:-none}" "match strings: ${regexes:-none}" "reproduce: jq .customManagers $CONFIG" + summary + exit 1 +fi + +rc=0 +grep -P 'x' /dev/null || rc=$? +if [ "$rc" -gt 1 ]; then + fail "grep -P works here, to run the manager's regex" "grep -P exited $rc" "reproduce: grep -P x /dev/null" + summary + exit 1 +fi + +# read_by_manager FILE: 0 when a managerFilePatterns entry names FILE. An +# entry is a /regex/ or a glob. +read_by_manager() { + local p re + while IFS= read -r p; do + [ -n "$p" ] || continue + case "$p" in + /*/) + re="${p#/}" + re="${re%/}" + if printf '%s\n' "$1" | grep -Eq -- "$re"; then + return 0 + fi + ;; + *) + # shellcheck disable=SC2053 # a glob, matched as a pattern on purpose + if [[ $1 == $p ]]; then + return 0 + fi + ;; + esac + done <<< "$patterns" + return 1 +} + +files="$(find "${SCAN_DIRS[@]}" -type f ! -path 'tests/fixtures/*' ! -path '*/__pycache__/*' ! -path '*/node_modules/*' | LC_ALL=C sort)" +pinned_files=0 +while IFS= read -r f; do + [ -n "$f" ] || continue + pins="$(grep -nEo "$PIN" "$f" || [ "$?" -eq 1 ])" + [ -n "$pins" ] || continue + pinned_files=$((pinned_files + 1)) + npins="$(printf '%s\n' "$pins" | wc -l)" + + if read_by_manager "$f"; then + ok "$f: the regex manager reads it ($npins pin(s))" + else + fail "$f: the regex manager reads it ($npins pin(s))" "$pins" \ + "no managerFilePatterns entry in $CONFIG names this file" "reproduce: jq '.customManagers[].managerFilePatterns' $CONFIG" + continue + fi + + best=0 + wrong="" + while IFS= read -r re; do + [ -n "$re" ] || continue + n=0 + while IFS= read -r -d '' m; do + n=$((n + 1)) + marker="$(printf '%s\n' "$m" | sed -nE '1s/^.*#[[:space:]]*tag:[[:space:]]*([A-Za-z0-9._-]+).*$/\1/p')" + tag="$(printf '%s\n' "$m" | grep -oE ':[A-Za-z0-9._-]+@sha256:' | sed -nE '$s/^:(.*)@sha256:$/\1/p')" + if [ "$marker" != "$tag" ]; then + wrong="${wrong}marker '# tag: $marker' over a pin tagged '$tag'; " + fi + done < <(grep -Pzo -- "$re" "$f" || [ "$?" -eq 1 ]) + if [ "$n" -gt "$best" ]; then + best="$n" + fi + done <<< "$regexes" + + if [ "$best" -eq "$npins" ]; then + ok "$f: the regex manager matches each of its $npins pin(s)" + else + fail "$f: the regex manager matches each of its $npins pin(s)" "matched $best of $npins" "$pins" \ + "each pin needs a '# tag: ' line at most three lines before it" \ + "reproduce: grep -n -B3 '@sha256:' $f" + fi + if [ -z "$wrong" ]; then + ok "$f: each '# tag:' marker names the tag of its pin" + else + fail "$f: each '# tag:' marker names the tag of its pin" "$wrong" \ + "Renovate reads the version from the marker, so a stale marker moves the pin to the wrong tag" \ + "reproduce: grep -n -A3 '# tag:' $f" + fi +done <<< "$files" + +if [ "$pinned_files" -gt 0 ]; then + ok "$pinned_files file(s) in ${SCAN_DIRS[*]} pin an image by digest" +else + fail "the tree pins images by digest" "no $PIN in ${SCAN_DIRS[*]}" "reproduce: grep -rnE '@sha256:' ${SCAN_DIRS[*]}" +fi + +summary From 6d6cc0ec091066730b8c698dd950b0cb9661df0a Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 11:36:12 +0545 Subject: [PATCH 05/22] pipeline: per-target chains for 15 targets, keyring_image and keyring_from, upstream-refused verify, signature-aware plan --- .github/workflows/sync-target.yml | 273 ++++++++++++++ .github/workflows/sync.yml | 393 ++++++++++---------- scripts/install-oras | 29 +- scripts/lib.sh | 318 +++++++++++++++- scripts/mirror-file | 26 +- scripts/mirror-shard | 17 +- scripts/plan | 273 ++++++++------ scripts/promote | 11 +- scripts/report | 157 ++++++-- scripts/stage | 146 ++++---- scripts/verify-batch | 577 +++++++++++++++++++++++------ targets/aarch64-alarm.conf | 39 ++ targets/aarch64-aur.conf | 39 ++ targets/aarch64-core.conf | 38 ++ targets/aarch64-extra.conf | 43 +++ targets/armv7h-alarm.conf | 39 ++ targets/armv7h-aur.conf | 39 ++ targets/armv7h-core.conf | 38 ++ targets/armv7h-extra.conf | 39 ++ targets/loong64-core.conf | 38 ++ targets/loong64-extra.conf | 45 +++ targets/riscv64-core.conf | 43 +++ targets/riscv64-extra.conf | 49 +++ targets/x86_64-core.conf | 8 +- targets/x86_64-extra.conf | 31 ++ targets/x86_64-multilib.conf | 31 ++ tests/fixtures/build-repo | 182 ++++++++-- tests/integration/10-pipeline.sh | 578 ++++++++++++++++++++++++------ tests/static/60-targets.sh | 97 ++++- tests/unit/40-lib.sh | 211 +++++++++++ 30 files changed, 3153 insertions(+), 694 deletions(-) create mode 100644 .github/workflows/sync-target.yml create mode 100644 targets/aarch64-alarm.conf create mode 100644 targets/aarch64-aur.conf create mode 100644 targets/aarch64-core.conf create mode 100644 targets/aarch64-extra.conf create mode 100644 targets/armv7h-alarm.conf create mode 100644 targets/armv7h-aur.conf create mode 100644 targets/armv7h-core.conf create mode 100644 targets/armv7h-extra.conf create mode 100644 targets/loong64-core.conf create mode 100644 targets/loong64-extra.conf create mode 100644 targets/riscv64-core.conf create mode 100644 targets/riscv64-extra.conf create mode 100644 targets/x86_64-extra.conf create mode 100644 targets/x86_64-multilib.conf diff --git a/.github/workflows/sync-target.yml b/.github/workflows/sync-target.yml new file mode 100644 index 0000000..25ef43a --- /dev/null +++ b/.github/workflows/sync-target.yml @@ -0,0 +1,273 @@ +# One target's chain, called by sync.yml once per pending target: mirror its +# shards, stage its database, verify it in batches with pacman through the +# wrapper, promote it (main only), and leave a small summary for the report. +# +# ⛔ The chain reads only its own target's artifacts, and every artifact it +# writes carries the target's name: chains of different targets never share a +# file, and one target's failure never holds another. +name: Sync one target + +on: + workflow_call: + inputs: + target: + description: The target, arch-repo + type: string + required: true + shards: + description: The target's shard numbers, a JSON list; [] when it has no file to mirror + type: string + required: true + max_shards: + description: Mirror shards at once + type: number + default: 3 + max_batches: + description: Verify batches at once + type: number + default: 3 + parallel: + description: Files mirrored at once in each shard + type: number + default: 6 + debloat: + description: Free runner disk before mirroring (auto, always, never) + type: string + default: auto + +permissions: {} + +defaults: + run: + shell: bash + +env: + MIRROR_REGISTRY: ghcr.io + +jobs: + mirror: + if: ${{ inputs.shards != '[]' }} + runs-on: ubuntu-24.04 + timeout-minutes: 120 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + max-parallel: ${{ inputs.max_shards }} + matrix: + shard: ${{ fromJSON(inputs.shards) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: plan-${{ inputs.target }} + path: ${{ runner.temp }}/plan/${{ inputs.target }} + # The step ends before the job does, so the results of the files that + # ended (results//lines) are still uploaded below. + - name: Mirror the shard + timeout-minutes: 110 + env: + TARGET: ${{ inputs.target }} + SHARD_INDEX: ${{ matrix.shard }} + PARALLEL: ${{ inputs.parallel }} + DEBLOAT: ${{ inputs.debloat }} + REGISTRY_USER: ${{ github.actor }} + REGISTRY_TOKEN: ${{ github.token }} + run: | + # The disk this runner really has, kept in the log: GitHub guarantees + # 14 GB and is moving runners to a single disk. + df -h / "$RUNNER_TEMP" + scripts/mirror-shard "$RUNNER_TEMP/plan/$TARGET/shards/$SHARD_INDEX.tsv" "$RUNNER_TEMP/results/$SHARD_INDEX" + - name: Upload the results + if: ${{ always() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: results-${{ inputs.target }}-${{ matrix.shard }} + path: ${{ runner.temp }}/results + retention-days: 7 + if-no-files-found: warn + overwrite: true + + stage: + needs: mirror + if: ${{ !cancelled() }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + packages: write + outputs: + verify_matrix: ${{ steps.stage.outputs.verify_matrix }} + has_batches: ${{ steps.stage.outputs.has_batches }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: plan-${{ inputs.target }} + path: ${{ runner.temp }}/plan/${{ inputs.target }} + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: results-${{ inputs.target }}-* + path: ${{ runner.temp }}/results + merge-multiple: false + - name: Stage + id: stage + env: + TARGET: ${{ inputs.target }} + REGISTRY_USER: ${{ github.actor }} + REGISTRY_TOKEN: ${{ github.token }} + run: | + mkdir -p "$RUNNER_TEMP/results" + if [ "$GITHUB_REF" = refs/heads/main ]; then + export STAGING_TAG=staging + else + slug="$(printf '%s' "$GITHUB_REF_NAME" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9\n' '-' | sed 's/--*/-/g; s/^-//; s/-$//' | cut -c1-40)" + export STAGING_TAG="staging-${slug:-branch}" + fi + scripts/stage "$RUNNER_TEMP/plan" "$RUNNER_TEMP/results" "$RUNNER_TEMP/stage" "$TARGET" + - name: Upload the stage + if: ${{ always() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: stage-${{ inputs.target }} + path: ${{ runner.temp }}/stage + retention-days: 7 + if-no-files-found: warn + overwrite: true + + verify: + needs: stage + if: ${{ !cancelled() && needs.stage.result == 'success' && needs.stage.outputs.has_batches == 'true' }} + runs-on: ubuntu-24.04 + timeout-minutes: 95 + permissions: + contents: read + strategy: + fail-fast: false + max-parallel: ${{ inputs.max_batches }} + matrix: ${{ fromJSON(needs.stage.outputs.verify_matrix) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: stage-${{ inputs.target }} + path: ${{ runner.temp }}/stage + # Budgets nest: verify-batch's own VERIFY_BUDGET (4800 s, the pulls and + # the container together) ends inside this step's 85 minutes, which end + # inside the job's 95, so the verdict is always uploaded. + - name: Verify with pacman through the wrapper + timeout-minutes: 85 + env: + TARGET: ${{ matrix.target }} + BATCH: ${{ matrix.batch }} + VERIFY_BUDGET: '4800' + run: scripts/verify-batch "$RUNNER_TEMP/stage" "$TARGET" "$BATCH" "$RUNNER_TEMP/verify" + - name: Upload the verdict + if: ${{ always() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: verify-${{ matrix.target }}-${{ matrix.batch }} + path: ${{ runner.temp }}/verify + retention-days: 7 + if-no-files-found: warn + overwrite: true + + promote: + needs: [stage, verify] + if: ${{ !cancelled() && needs.stage.result == 'success' && github.ref == 'refs/heads/main' }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: stage-${{ inputs.target }} + path: ${{ runner.temp }}/stage + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: verify-${{ inputs.target }}-* + path: ${{ runner.temp }}/verify + merge-multiple: false + - name: Promote + env: + TARGET: ${{ inputs.target }} + REGISTRY_USER: ${{ github.actor }} + REGISTRY_TOKEN: ${{ github.token }} + run: | + mkdir -p "$RUNNER_TEMP/verify" "$RUNNER_TEMP/promote" + rc=0 + scripts/promote "$RUNNER_TEMP/stage" "$RUNNER_TEMP/verify" "$TARGET" 2> "$RUNNER_TEMP/promote/$TARGET.err" || rc=$? + cat "$RUNNER_TEMP/promote/$TARGET.err" >&2 + if [ "$rc" -eq 0 ]; then + if [ -f "$RUNNER_TEMP/stage/$TARGET/promoted.tsv" ]; then + printf '%s\tpromoted\t%s\n' "$TARGET" "$(cut -f 3,4 "$RUNNER_TEMP/stage/$TARGET/promoted.tsv" | tr '\t' ' ')" > "$RUNNER_TEMP/promote/result.tsv" + fi + else + printf '%s\tfailed\t%s\n' "$TARGET" "$(grep -m 1 'promote:' "$RUNNER_TEMP/promote/$TARGET.err" | tr '\t' ' ')" > "$RUNNER_TEMP/promote/result.tsv" + fi + exit "$rc" + - name: Upload the promotion + if: ${{ always() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: promote-${{ inputs.target }} + path: ${{ runner.temp }}/promote + retention-days: 30 + if-no-files-found: warn + overwrite: true + + # What the report needs of this chain, a few KiB, so the report downloads + # no target's plan, stage or verdicts. + summary: + needs: [mirror, stage, verify, promote] + if: ${{ always() }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: stage-${{ inputs.target }} + path: ${{ runner.temp }}/stage + merge-multiple: true + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: verify-${{ inputs.target }}-* + path: ${{ runner.temp }}/verify + merge-multiple: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: promote-${{ inputs.target }} + path: ${{ runner.temp }}/promote + merge-multiple: true + - name: Collect + env: + TARGET: ${{ inputs.target }} + run: | + mkdir -p "$RUNNER_TEMP/stage" "$RUNNER_TEMP/verify" "$RUNNER_TEMP/promote" "$RUNNER_TEMP/summary" + scripts/report collect "$TARGET" "$RUNNER_TEMP/stage" "$RUNNER_TEMP/verify" "$RUNNER_TEMP/promote/result.tsv" "$RUNNER_TEMP/summary/$TARGET" + - name: Upload the summary + if: ${{ always() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: summary-${{ inputs.target }} + path: ${{ runner.temp }}/summary + retention-days: 30 + if-no-files-found: error + overwrite: true diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 11cb747..314f02c 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -1,14 +1,17 @@ -# Mirror every enabled target to GHCR: plan, mirror in shards, stage, verify -# with pacman through the wrapper, promote, report. +# Mirror every enabled target to GHCR: plan, then one chain per pending target +# (.github/workflows/sync-target.yml: mirror in shards, stage, verify with +# pacman through the wrapper, promote), then one report. # # ⛔ A database reaches latest only after every file it names is on the # registry and every file new since the promoted database has passed pacman's -# own checks through the wrapper. Targets are independent: one held target -# never holds another. HISTORY/design.md has the whole design. +# own checks through the wrapper. Targets are independent: each runs its own +# chain, and a slow or held target never delays or holds another. +# HISTORY/design.md has the whole design. # # promote runs only on main. A run from any other branch does everything else, # writes only write-once package tags and its own staging- tag, and -# never moves latest. +# never moves latest. Runs from every branch share one concurrency group, so +# two runs never write the same tags at once. name: Sync on: @@ -20,14 +23,22 @@ on: description: Targets (arch-repo, space separated); empty for every enabled one type: string default: '' - max_parallel: - description: Most matrix jobs at once; the organisation shares 20 - type: string - default: '8' + max_targets: + description: Target chains at once; the organisation shares 20 jobs + type: number + default: 4 + max_shards: + description: Mirror shards at once in each target's chain + type: number + default: 3 + max_batches: + description: Verify batches at once in each target's chain + type: number + default: 3 parallel: description: Files mirrored at once in each shard - type: string - default: '4' + type: number + default: 6 debloat: description: Free runner disk before mirroring type: choice @@ -41,7 +52,7 @@ on: permissions: {} concurrency: - group: sync-${{ github.ref }} + group: sync cancel-in-progress: false defaults: @@ -52,19 +63,18 @@ env: MIRROR_REGISTRY: ghcr.io SHARD_BYTES: '536870912' SHARD_FILES: '400' - MAX_SHARDS: '16' + MAX_SHARDS: '32' jobs: plan: runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 30 permissions: contents: read packages: read outputs: - matrix: ${{ steps.plan.outputs.matrix }} - has_files: ${{ steps.plan.outputs.has_files }} - has_work: ${{ steps.plan.outputs.has_work }} + targets: ${{ steps.plan.outputs.targets }} + has_targets: ${{ steps.plan.outputs.has_targets }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -88,239 +98,226 @@ jobs: args+=("targets/$t.conf") done scripts/plan "$RUNNER_TEMP/plan" "${args[@]}" - - name: Upload the plan + mkdir -p "$RUNNER_TEMP/plan-summary" + cp "$RUNNER_TEMP/plan/summary.tsv" "$RUNNER_TEMP/plan/targets.json" "$RUNNER_TEMP/plan/pending.json" "$RUNNER_TEMP/plan-summary/" + - name: Upload the plan's summary, for the report uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: plan - path: ${{ runner.temp }}/plan + name: plan-summary + path: ${{ runner.temp }}/plan-summary retention-days: 7 if-no-files-found: error overwrite: true - - mirror: - needs: plan - if: ${{ needs.plan.outputs.has_files == 'true' }} - runs-on: ubuntu-24.04 - timeout-minutes: 120 - permissions: - contents: read - packages: write - strategy: - fail-fast: false - max-parallel: ${{ fromJSON(inputs.max_parallel || '8') }} - matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # One artifact per pending target, plan-, so a chain downloads its + # own target's plan and nothing else. The plan names a target per slot; + # tests/static/60-targets.sh checks there is a slot for every enabled + # target. + - name: Upload the plan of slot 0 + if: ${{ steps.plan.outputs.slot0 != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + name: plan-${{ steps.plan.outputs.slot0 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot0 }} + retention-days: 7 + if-no-files-found: error + overwrite: true + - name: Upload the plan of slot 1 + if: ${{ steps.plan.outputs.slot1 != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: plan - path: ${{ runner.temp }}/plan - - name: Mirror the shard - env: - SHARD_INDEX: ${{ matrix.shard }} - PARALLEL: ${{ inputs.parallel || '4' }} - DEBLOAT: ${{ inputs.debloat || 'auto' }} - REGISTRY_USER: ${{ github.actor }} - REGISTRY_TOKEN: ${{ github.token }} - run: | - # The disk this runner really has, kept in the log: GitHub guarantees - # 14 GB and is moving runners to a single disk. - df -h / "$RUNNER_TEMP" - scripts/mirror-shard "$RUNNER_TEMP/plan/shards/$SHARD_INDEX.tsv" "$RUNNER_TEMP/results/$SHARD_INDEX" - - name: Upload the results - if: ${{ always() }} + name: plan-${{ steps.plan.outputs.slot1 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot1 }} + retention-days: 7 + if-no-files-found: error + overwrite: true + - name: Upload the plan of slot 2 + if: ${{ steps.plan.outputs.slot2 != '' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: results-${{ matrix.shard }} - path: ${{ runner.temp }}/results + name: plan-${{ steps.plan.outputs.slot2 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot2 }} retention-days: 7 - if-no-files-found: warn + if-no-files-found: error overwrite: true - - stage: - needs: [plan, mirror] - if: ${{ !cancelled() && needs.plan.result == 'success' && needs.plan.outputs.has_work == 'true' }} - runs-on: ubuntu-24.04 - timeout-minutes: 20 - permissions: - contents: read - packages: write - outputs: - verify_matrix: ${{ steps.stage.outputs.verify_matrix }} - has_batches: ${{ steps.stage.outputs.has_batches }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Upload the plan of slot 3 + if: ${{ steps.plan.outputs.slot3 != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + name: plan-${{ steps.plan.outputs.slot3 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot3 }} + retention-days: 7 + if-no-files-found: error + overwrite: true + - name: Upload the plan of slot 4 + if: ${{ steps.plan.outputs.slot4 != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: plan - path: ${{ runner.temp }}/plan - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + name: plan-${{ steps.plan.outputs.slot4 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot4 }} + retention-days: 7 + if-no-files-found: error + overwrite: true + - name: Upload the plan of slot 5 + if: ${{ steps.plan.outputs.slot5 != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - pattern: results-* - path: ${{ runner.temp }}/results - merge-multiple: false - - name: Stage - id: stage - env: - REGISTRY_USER: ${{ github.actor }} - REGISTRY_TOKEN: ${{ github.token }} - run: | - mkdir -p "$RUNNER_TEMP/results" - if [ "$GITHUB_REF" = refs/heads/main ]; then - export STAGING_TAG=staging - else - slug="$(printf '%s' "$GITHUB_REF_NAME" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9\n' '-' | sed 's/--*/-/g; s/^-//; s/-$//' | cut -c1-40)" - export STAGING_TAG="staging-${slug:-branch}" - fi - scripts/stage "$RUNNER_TEMP/plan" "$RUNNER_TEMP/results" "$RUNNER_TEMP/stage" - - name: Upload the stage - if: ${{ always() }} + name: plan-${{ steps.plan.outputs.slot5 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot5 }} + retention-days: 7 + if-no-files-found: error + overwrite: true + - name: Upload the plan of slot 6 + if: ${{ steps.plan.outputs.slot6 != '' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: stage - path: ${{ runner.temp }}/stage + name: plan-${{ steps.plan.outputs.slot6 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot6 }} retention-days: 7 - if-no-files-found: warn + if-no-files-found: error overwrite: true - - verify: - needs: stage - if: ${{ !cancelled() && needs.stage.result == 'success' && needs.stage.outputs.has_batches == 'true' }} - runs-on: ubuntu-24.04 - timeout-minutes: 90 - permissions: - contents: read - strategy: - fail-fast: false - max-parallel: ${{ fromJSON(inputs.max_parallel || '8') }} - matrix: ${{ fromJSON(needs.stage.outputs.verify_matrix) }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Upload the plan of slot 7 + if: ${{ steps.plan.outputs.slot7 != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + name: plan-${{ steps.plan.outputs.slot7 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot7 }} + retention-days: 7 + if-no-files-found: error + overwrite: true + - name: Upload the plan of slot 8 + if: ${{ steps.plan.outputs.slot8 != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: stage - path: ${{ runner.temp }}/stage - - name: Verify with pacman through the wrapper - env: - TARGET: ${{ matrix.target }} - BATCH: ${{ matrix.batch }} - run: scripts/verify-batch "$RUNNER_TEMP/stage" "$TARGET" "$BATCH" "$RUNNER_TEMP/verify" - - name: Upload the verdict - if: ${{ always() }} + name: plan-${{ steps.plan.outputs.slot8 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot8 }} + retention-days: 7 + if-no-files-found: error + overwrite: true + - name: Upload the plan of slot 9 + if: ${{ steps.plan.outputs.slot9 != '' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: verify-${{ matrix.target }}-${{ matrix.batch }} - path: ${{ runner.temp }}/verify + name: plan-${{ steps.plan.outputs.slot9 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot9 }} retention-days: 7 - if-no-files-found: warn + if-no-files-found: error overwrite: true - - promote: - needs: [stage, verify] - if: ${{ !cancelled() && needs.stage.result == 'success' && github.ref == 'refs/heads/main' }} - runs-on: ubuntu-24.04 - timeout-minutes: 15 - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Upload the plan of slot 10 + if: ${{ steps.plan.outputs.slot10 != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + name: plan-${{ steps.plan.outputs.slot10 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot10 }} + retention-days: 7 + if-no-files-found: error + overwrite: true + - name: Upload the plan of slot 11 + if: ${{ steps.plan.outputs.slot11 != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: stage - path: ${{ runner.temp }}/stage - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + name: plan-${{ steps.plan.outputs.slot11 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot11 }} + retention-days: 7 + if-no-files-found: error + overwrite: true + - name: Upload the plan of slot 12 + if: ${{ steps.plan.outputs.slot12 != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - pattern: verify-* - path: ${{ runner.temp }}/verify - merge-multiple: false - - name: Promote each verified target - env: - REGISTRY_USER: ${{ github.actor }} - REGISTRY_TOKEN: ${{ github.token }} - run: | - # Each target decides alone: one that is refused never holds another. - mkdir -p "$RUNNER_TEMP/verify" "$RUNNER_TEMP/promote" - : > "$RUNNER_TEMP/promote/results.tsv" - for d in "$RUNNER_TEMP"/stage/*/; do - d="${d%/}" - [ -f "$d/state" ] || continue - [ "$(cat "$d/state")" = staged ] || continue - t="$(basename "$d")" - rc=0 - scripts/promote "$RUNNER_TEMP/stage" "$RUNNER_TEMP/verify" "$t" 2> "$RUNNER_TEMP/promote/$t.err" || rc=$? - cat "$RUNNER_TEMP/promote/$t.err" >&2 - if [ "$rc" -eq 0 ]; then - printf '%s\tpromoted\t%s\n' "$t" "$(cut -f 3,4 "$d/promoted.tsv" | tr '\t' ' ')" >> "$RUNNER_TEMP/promote/results.tsv" - else - printf '%s\tfailed\t%s\n' "$t" "$(grep -m 1 'promote:' "$RUNNER_TEMP/promote/$t.err" | tr '\t' ' ')" >> "$RUNNER_TEMP/promote/results.tsv" - fi - done - - name: Upload the promotions - if: ${{ always() }} + name: plan-${{ steps.plan.outputs.slot12 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot12 }} + retention-days: 7 + if-no-files-found: error + overwrite: true + - name: Upload the plan of slot 13 + if: ${{ steps.plan.outputs.slot13 != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: plan-${{ steps.plan.outputs.slot13 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot13 }} + retention-days: 7 + if-no-files-found: error + overwrite: true + - name: Upload the plan of slot 14 + if: ${{ steps.plan.outputs.slot14 != '' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: promote - path: ${{ runner.temp }}/promote - retention-days: 30 - if-no-files-found: warn + name: plan-${{ steps.plan.outputs.slot14 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot14 }} + retention-days: 7 + if-no-files-found: error + overwrite: true + - name: Upload the plan of slot 15 + if: ${{ steps.plan.outputs.slot15 != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: plan-${{ steps.plan.outputs.slot15 }} + path: ${{ runner.temp }}/plan/${{ steps.plan.outputs.slot15 }} + retention-days: 7 + if-no-files-found: error overwrite: true + # One chain per pending target, each on its own: a target's promote waits + # for its own shards and batches only. A job that calls a reusable workflow + # takes no timeout-minutes (GitHub refuses the key there); every job of + # sync-target.yml has its own. + target: + needs: plan + if: ${{ needs.plan.outputs.has_targets == 'true' }} + permissions: + contents: read + packages: write + strategy: + fail-fast: false + max-parallel: ${{ inputs.max_targets || 4 }} + matrix: ${{ fromJSON(needs.plan.outputs.targets) }} + uses: ./.github/workflows/sync-target.yml + with: + target: ${{ matrix.target }} + shards: ${{ toJSON(matrix.shards) }} + max_shards: ${{ inputs.max_shards || 3 }} + max_batches: ${{ inputs.max_batches || 3 }} + parallel: ${{ inputs.parallel || 6 }} + debloat: ${{ inputs.debloat || 'auto' }} + report: - needs: [plan, mirror, stage, verify, promote] + needs: [plan, target] if: ${{ always() }} runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: + actions: write contents: read issues: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - # Each artifact into a directory of its own, by a pattern so one that a - # skipped job never made is not an error. Whether a single match lands in - # DIR or in DIR/ has changed between versions of the action; the - # step below accepts either. + # By a pattern, so an artifact that a skipped job never made is not an + # error. Each summary- holds /..., so merged they land + # side by side whichever way the action lays out a single match. - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - pattern: plan + pattern: plan-summary path: ${{ runner.temp }}/dl-plan + merge-multiple: true - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - pattern: stage - path: ${{ runner.temp }}/dl-stage - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - pattern: promote - path: ${{ runner.temp }}/dl-promote + pattern: summary-* + path: ${{ runner.temp }}/summaries + merge-multiple: true - name: Report env: GH_TOKEN: ${{ github.token }} TRACK_ISSUES: ${{ github.ref == 'refs/heads/main' && '1' || '0' }} run: | - # dl-NAME/NAME when the action made a subdirectory, dl-NAME when not - pick() { - if [ -d "$RUNNER_TEMP/dl-$1/$1" ]; then - echo "$RUNNER_TEMP/dl-$1/$1" - else - mkdir -p "$RUNNER_TEMP/dl-$1" - echo "$RUNNER_TEMP/dl-$1" - fi - } - plan="$(pick plan)" - stage="$(pick stage)" - promote="$(pick promote)" - touch "$promote/results.tsv" - scripts/report "$plan" "$stage" "$promote/results.tsv" + mkdir -p "$RUNNER_TEMP/dl-plan" "$RUNNER_TEMP/summaries" + scripts/report "$RUNNER_TEMP/dl-plan" "$RUNNER_TEMP/summaries" + - name: Start the status workflow + if: ${{ always() && github.ref == 'refs/heads/main' }} + env: + GH_TOKEN: ${{ github.token }} + run: gh workflow run status.yml --ref main --repo "$GITHUB_REPOSITORY" # No needs, in either direction: this runs whatever the sync does, and the # watchdog workflow runs the same two checks, so each witnesses the other. diff --git a/scripts/install-oras b/scripts/install-oras index 78dd269..ef7d48f 100755 --- a/scripts/install-oras +++ b/scripts/install-oras @@ -14,6 +14,11 @@ # ⭐ oras is stable and nothing here follows its releases. A newer version is a # deliberate edit of the three lines below and nothing else. # +# Environment: ORAS_CACHE, a directory that keeps the release archive between +# calls, so a test suite that installs oras for every step downloads it once. +# The archive in it is checked against the pinned sha256 on every use, the +# same as a fresh download, and one that differs is not used. +# # Exit status: 0 installed, 1 could not. set -euo pipefail @@ -54,15 +59,31 @@ trap 'rm -rf "$work"' EXIT asset="oras_${ORAS_VERSION}_linux_${arch}.tar.gz" url="https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/${asset}" -rc=0 -curl -q --fail --location --silent --show-error --connect-timeout 20 --max-time 300 \ - --retry 3 --retry-connrefused --retry-delay 2 --output "$work/$asset" "$url" || rc=$? -[ "$rc" -eq 0 ] || die "could not download $url (curl $rc)" "reproduce: curl -fLO $url" +cached="" +if [ -n "${ORAS_CACHE:-}" ] && [ -f "$ORAS_CACHE/$asset" ]; then + if [ "$(sha256sum "$ORAS_CACHE/$asset" | cut -d ' ' -f 1)" = "$want" ]; then + cp "$ORAS_CACHE/$asset" "$work/$asset" + cached=1 + else + echo "install-oras: $ORAS_CACHE/$asset does not hash to the pinned $want; downloading again" >&2 + fi +fi +if [ -z "$cached" ]; then + rc=0 + curl -q --fail --location --silent --show-error --connect-timeout 20 --max-time 300 \ + --retry 3 --retry-connrefused --retry-delay 2 --output "$work/$asset" "$url" || rc=$? + [ "$rc" -eq 0 ] || die "could not download $url (curl $rc)" "reproduce: curl -fLO $url" +fi got="$(sha256sum "$work/$asset" | cut -d ' ' -f 1)" [ "$got" = "$want" ] || die "$asset has sha256 $got, pinned $want" \ "a release asset that changed after it was published is not installed" \ "reproduce: curl -fsSL $url | sha256sum" +if [ -n "${ORAS_CACHE:-}" ] && [ -z "$cached" ]; then + mkdir -p "$ORAS_CACHE" + cp "$work/$asset" "$ORAS_CACHE/$asset.tmp.$$" + mv -f "$ORAS_CACHE/$asset.tmp.$$" "$ORAS_CACHE/$asset" +fi tar -xzf "$work/$asset" -C "$work" oras install -m 0755 "$work/oras" "$dest/oras" diff --git a/scripts/lib.sh b/scripts/lib.sh index 0369c57..c459fa7 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -72,9 +72,22 @@ need() { # targets/-.conf: `key = value` lines, # comments, a key may repeat # where the list below says so. An unknown key is an error: a typo that is # ignored is a setting that silently does nothing. +# +# The keys that set the trust root of verify (scripts/verify-batch): +# keyring the keyring packages, space separated. verify installs +# them from the snapshot, then populates again. +# keyring_from a target - of the same arch whose promoted +# latest holds the keyring packages, when the target's own +# repository does not. +# keyring_image an image name:tag@sha256:<64 hex> whose +# /usr/share/pacman/keyrings/* is copied into the verify +# container before the first populate. Nothing in it runs. +# keyring_platform the platform of keyring_image, e.g. linux/arm64. #---------------------------------------------------------------------------# -TARGET_KEYS_ONCE="arch repo verify_image verify_platform keyring enabled db_signed" +TARGET_KEYS_ONCE="arch repo verify_image verify_platform keyring keyring_from keyring_image keyring_platform enabled db_signed" TARGET_KEYS_MANY="server db_server" +# An image pinned by digest, with a tag Renovate can follow. +IMAGE_PIN_RE='^[a-z0-9.-]+(:[0-9]+)?(/[a-z0-9._-]+)+:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}$' # target_check CONF -> exit 0, or die naming the line target_check() { @@ -96,7 +109,7 @@ target_check() { printf "%s:%d: unknown key %s\n", f, NR, k; bad = 1 } END { exit bad } - ' "$conf" >&2 || die "$conf is not a valid target file" "reproduce: scripts/target check $conf" + ' "$conf" >&2 || die "$conf is not a valid target file" "reproduce: bash -c '. scripts/lib.sh; target_check $conf'" local arch repo arch="$(target_get "$conf" arch)" repo="$(target_get "$conf" repo)" @@ -108,6 +121,42 @@ target_check() { '' | yes | no) ;; *) die "$conf: db_signed must be yes or no" ;; esac + + # The trust root of verify. read -a splits without glob expansion. + local keyrings k image platform from + keyrings="$(target_get "$conf" keyring)" + local names=() + read -r -a names <<< "$keyrings" + for k in "${names[@]}"; do + [[ "$k" =~ ^[a-z0-9@_+][a-z0-9@._+-]*$ ]] || die "$conf: keyring '$k' is not a package name" + done + image="$(target_get "$conf" keyring_image)" + platform="$(target_get "$conf" keyring_platform)" + if [ -n "$image" ]; then + [[ "$image" =~ $IMAGE_PIN_RE ]] || die "$conf: keyring_image '$image' is not name:tag@sha256:<64 hex>" + [ -n "$platform" ] || die "$conf: keyring_image needs keyring_platform, the platform to pull it for" + fi + if [ -n "$platform" ]; then + [ -n "$image" ] || die "$conf: keyring_platform is set and keyring_image is not" + [[ "$platform" =~ ^linux/[a-z0-9]+(/v[0-9]+)?$ ]] || die "$conf: keyring_platform '$platform' is not linux/[/v]" + fi + from="$(target_get "$conf" keyring_from)" + if [ -n "$from" ]; then + [ "${#names[@]}" -gt 0 ] || die "$conf: keyring_from is set and keyring names no package" + [[ "$from" =~ ^[a-z0-9]+(_[a-z0-9]+)*-[a-z0-9]+([.-][a-z0-9]+)*$ ]] || die "$conf: keyring_from '$from' is not a target name -" + [ "$from" != "$arch-$repo" ] || die "$conf: keyring_from names the target itself; leave it out instead" + [ "${from%%-*}" = "$arch" ] || die "$conf: keyring_from '$from' is another arch than $arch" + [ -f "$(dirname "$conf")/$from.conf" ] || die "$conf: keyring_from names $from, and there is no $(dirname "$conf")/$from.conf" + fi +} + +# target_keyrings CONF -> the keyring packages, one per line +target_keyrings() { + local names=() k + read -r -a names <<< "$(target_get "$1" keyring)" + for k in "${names[@]}"; do + printf '%s\n' "$k" + done } # target_sources CONF -> every server package bytes may come from: the @@ -294,6 +343,76 @@ db_validate() { ' "$1" } +# db_check DB RECORDS -> 0 when DB is a whole, well-formed database, with its +# records in RECORDS (db_records) and the package directories it cannot read +# in RECORDS.nul. Otherwise 1, and one line on stdout says why. +# +# Every package directory must give exactly one record, named after it. The +# one exception is a desc member that holds nothing but NUL bytes, or nothing: +# it carries no field at all, pacman lists it with every field None and cannot +# download it, so nothing is lost by passing over it. Measured 2026-10-14: +# findnewest-0.3-4/desc in Arch Linux ARM aarch64 extra (1271 NUL bytes) and +# ntk-1.3.1001-4/desc in Arch Linux RISC-V extra (576), both GNU sparse +# members. Any other unreadable member refuses the database, as before. +DB_MAX_NUL_MEMBERS=16 +db_check() { + local db="$1" rec="$2" records members unread nonul n f + : > "$rec.nul" + if ! db_records "$db" "$rec"; then + echo "it does not read as a tar archive" + return 1 + fi + records="$(awk 'END { print NR + 0 }' "$rec")" + if [ "$records" -eq 0 ]; then + echo "it lists no package" + return 1 + fi + if ! db_validate "$rec" > "$rec.bad"; then + echo "$(awk 'END { print NR + 0 }' "$rec.bad") record(s) fail their grammar, first: $(sed -n 1p "$rec.bad")" + return 1 + fi + if ! tar -tf "$db" > "$rec.members"; then + echo "its member list does not read" + return 1 + fi + # The package directories with a desc, and those no record is named after. + awk -F / 'NF >= 2 && $2 == "desc" { print $1 }' "$rec.members" > "$rec.dirs" + members="$(awk 'END { print NR + 0 }' "$rec.dirs")" + awk -F '\t' 'FILENAME == ARGV[1] { have[$2 "-" $3] = 1; next } !($0 in have)' "$rec" "$rec.dirs" > "$rec.unread" + unread="$(awk 'END { print NR + 0 }' "$rec.unread")" + if [ "$unread" -gt 0 ]; then + if [ "$unread" -gt "$DB_MAX_NUL_MEMBERS" ]; then + echo "$unread package directories give no record, more than the $DB_MAX_NUL_MEMBERS accepted as NUL-only, first: $(sed -n 1p "$rec.unread")" + return 1 + fi + if LC_ALL=C awk '!/^[A-Za-z0-9@_+][A-Za-z0-9@._+:~-]*$/ { bad = 1 } END { exit !bad }' "$rec.unread"; then + echo "a package directory has a name no package has: $(LC_ALL=C awk '!/^[A-Za-z0-9@_+][A-Za-z0-9@._+:~-]*$/' "$rec.unread" | sed -n 1p)" + return 1 + fi + sed 's|$|/desc|' "$rec.unread" > "$rec.unread-desc" + # One read for all of them; the bytes left once NULs are dropped. + nonul="$(tar -xOf "$db" -T "$rec.unread-desc" | tr -d '\000' | wc -c | tr -d ' ')" || nonul=-1 + if [ "$nonul" != 0 ]; then + n=0 + while IFS= read -r f; do + n="$(tar -xOf "$db" "$f/desc" | tr -d '\000' | wc -c | tr -d ' ')" || n=-1 + if [ "$n" != 0 ]; then + echo "$f/desc gives no record named after its directory, and holds $n byte(s) that are not NUL" + return 1 + fi + done < "$rec.unread" + echo "the members that give no record do not read ($nonul)" + return 1 + fi + cp "$rec.unread" "$rec.nul" + fi + if [ "$((records + unread))" -ne "$members" ]; then + echo "it holds $members package directories, $records readable records and $unread NUL-only member(s)" + return 1 + fi + return 0 +} + # behind A.tsv B.tsv -> "NEWER OLDER": how many packages of B were built # after, and before, the same package in A. B is behind A when OLDER > 0. # Measured on Arch's archived core.db, 2026-10-01 to 10-12: in every pair of @@ -453,16 +572,42 @@ reconcile_results() { # # REGISTRY_USER and REGISTRY_TOKEN, when set, are exchanged for a scoped token. # Without them the reads are anonymous, the way any consumer reads. +# +# REGISTRY_TOKEN_CACHE, when set, is a directory that keeps each scoped token +# in a file of its own. Every process that reads it uses the same token, so a +# shard asks the token endpoint once, not once per read. A 401 or a 403 drops +# the file and asks again, once. +# +# On GHCR a manifest read tries the dummy token Bearer QQ== first, which every +# public package accepts. Only its 200 is taken as the answer: GHCR answers QQ== +# with 404 MANIFEST_UNKNOWN for a repository it does not show to anonymous +# readers (measured 2026-10-14), so with REGISTRY_TOKEN set any other answer is +# asked again with the real token before it decides anything. #---------------------------------------------------------------------------# REG_STATUS_OK=0 REG_STATUS_ERROR=1 REG_STATUS_ABSENT=4 +# registry_token_file REPO SCOPE -> the cache file for that token, or nothing +registry_token_file() { + [ -n "${REGISTRY_TOKEN_CACHE:-}" ] || return 0 + printf '%s/%s\n' "$REGISTRY_TOKEN_CACHE" "$(printf '%s %s %s' "$MIRROR_REGISTRY" "$1" "$2" | sha256sum | cut -d ' ' -f 1)" +} + +# registry_token_drop REPO SCOPE: forget a cached token the registry refused +registry_token_drop() { + local f + f="$(registry_token_file "$1" "$2")" + if [ -n "$f" ]; then + rm -f -- "$f" + fi +} + # registry_token REPO SCOPE -> a bearer token on stdout. SCOPE is pull or # pull,push. Anonymous pulls on GHCR use the dummy token every public package -# accepts; anything else asks the token endpoint. +# accepts; anything else asks the token endpoint, or the cache. registry_token() { - local repo="$1" scope="$2" out rc=0 tok + local repo="$1" scope="$2" out rc=0 tok cache tmp if [ "$MIRROR_REGISTRY_AUTH" = none ]; then printf 'none\n' return 0 @@ -471,6 +616,11 @@ registry_token() { printf 'QQ==\n' return 0 fi + cache="$(registry_token_file "$repo" "$scope")" + if [ -n "$cache" ] && [ -s "$cache" ]; then + cat -- "$cache" + return 0 + fi out="$(mktemp)" if [ -n "${REGISTRY_TOKEN:-}" ]; then curl -q --silent --show-error --fail --connect-timeout 20 --max-time 60 \ @@ -492,6 +642,11 @@ registry_token() { tok="$(jq -r '.token // .access_token // empty' "$out")" rm -f "$out" [ -n "$tok" ] || return 1 + if [ -n "$cache" ]; then + tmp="$(mktemp "$REGISTRY_TOKEN_CACHE/.token.XXXXXX")" + printf '%s\n' "$tok" > "$tmp" + mv -f -- "$tmp" "$cache" + fi printf '%s\n' "$tok" } @@ -500,17 +655,37 @@ registry_error_code() { jq -s -R -r '(fromjson? // {}) | .errors[0].code? // empty' "$1" | tr -d '\r' } +# registry_get_manifest URL TOKEN OUT -> the HTTP status on stdout, curl's +# status as the return value +registry_get_manifest() { + curl -q --silent --show-error --connect-timeout 20 --max-time 60 \ + --retry 3 --retry-connrefused --retry-delay 2 --max-filesize 4194304 \ + --header "Authorization: Bearer $2" \ + --header 'Accept: application/vnd.oci.image.manifest.v1+json' \ + --output "$3" --write-out '%{http_code}' "$1" +} + # registry_manifest REPO REF OUT -> the manifest bytes in OUT # returns 0 read, 4 the registry has no such manifest, 1 could not read registry_manifest() { - local repo="$1" ref="$2" outfile="$3" token code rc=0 err - token="$(registry_token "$repo" pull)" || return "$REG_STATUS_ERROR" - code="$(curl -q --silent --show-error --connect-timeout 20 --max-time 60 \ - --retry 3 --retry-connrefused --retry-delay 2 --max-filesize 4194304 \ - --header "Authorization: Bearer $token" \ - --header 'Accept: application/vnd.oci.image.manifest.v1+json' \ - --output "$outfile" --write-out '%{http_code}' \ - "$REG_URL/v2/$repo/manifests/$ref")" || rc=$? + local repo="$1" ref="$2" outfile="$3" token code="" rc=0 err url + url="$REG_URL/v2/$repo/manifests/$ref" + if [ "$MIRROR_REGISTRY_AUTH" != none ] && [ "$MIRROR_REGISTRY" = ghcr.io ] && [ -n "${REGISTRY_TOKEN:-}" ]; then + code="$(registry_get_manifest "$url" 'QQ==' "$outfile")" || rc=$? + if [ "$rc" -ne 0 ] || [ "$code" != 200 ]; then + code="" + rc=0 + fi + fi + if [ -z "$code" ]; then + token="$(registry_token "$repo" pull)" || return "$REG_STATUS_ERROR" + code="$(registry_get_manifest "$url" "$token" "$outfile")" || rc=$? + if [ "$rc" -eq 0 ] && { [ "$code" = 401 ] || [ "$code" = 403 ]; } && [ -n "$(registry_token_file "$repo" pull)" ]; then + registry_token_drop "$repo" pull + token="$(registry_token "$repo" pull)" || return "$REG_STATUS_ERROR" + code="$(registry_get_manifest "$url" "$token" "$outfile")" || rc=$? + fi + fi if [ "$rc" -ne 0 ]; then warn "no answer for $repo:$ref (curl $rc)" return "$REG_STATUS_ERROR" @@ -576,13 +751,19 @@ manifest_layer() { ' "$1" } -# fetch_upstream URL OUT -> 0 fetched, 4 the server says it has no such file, -# 1 anything else. Large files get a stall deadline rather than a fixed one. +# fetch_upstream URL OUT [MAX_SECONDS] -> 0 fetched, 4 the server says it has +# no such file, 1 anything else. Every fetch has a stall deadline; a database +# or a signature also has a whole-transfer one (MAX_SECONDS), so a server that +# trickles moves the caller on to the next server. A package file has no fixed +# deadline: its size makes any fixed one wrong. fetch_upstream() { - local url="$1" outfile="$2" code rc=0 + local url="$1" outfile="$2" code rc=0 max=() + if [ -n "${3:-}" ]; then + max=(--max-time "$3") + fi code="$(curl -q --location --silent --show-error --max-redirs 5 \ --proto "$UPSTREAM_PROTO" --proto-redir "$UPSTREAM_PROTO" \ - --connect-timeout 20 --speed-limit 4096 --speed-time 60 \ + --connect-timeout 20 --speed-limit 4096 --speed-time 60 "${max[@]}" \ --retry 2 --retry-connrefused --retry-delay 2 \ --user-agent "mirror-archlinux (+https://github.com/${GITHUB_REPOSITORY:-pkgforge-dev/mirror-archlinux})" \ --output "$outfile" --write-out '%{http_code}' "$url")" || rc=$? @@ -607,3 +788,108 @@ fetch_upstream() { now_epoch() { printf '%s\n' "${MIRROR_NOW:-$(date -u +%s)}" } + +# iso_of_epoch SECONDS -> that time as ISO 8601 UTC +iso_of_epoch() { + case "$1" in + '' | *[!0-9]*) return 1 ;; + esac + date -u -d "@$1" +%Y-%m-%dT%H:%M:%SZ +} + +#---------------------------------------------------------------------------# +# How work is split +# +# shard_count FILES BYTES SHARD_FILES SHARD_BYTES MAX -> "K WANTED" +# WANTED = max(ceil(BYTES / SHARD_BYTES), ceil(FILES / SHARD_FILES)), at +# least 1; K is WANTED clamped to MAX and to FILES. No file: "0 0". +# +# batch_count FILES BYTES BATCH_FILES BATCH_BYTES MAX -> "K FILES_LIMIT BYTES_LIMIT WANTED" +# As shard_count with the limits given, but when that gives more than MAX +# batches, both limits grow by an eighth at a time until it does not: every +# file is still verified, in fewer, larger batches. +#---------------------------------------------------------------------------# +shard_count() { + awk -v f="$1" -v b="$2" -v sf="$3" -v sb="$4" -v m="$5" 'BEGIN { + if (f + 0 <= 0) { print "0 0"; exit } + kb = int((b + sb - 1) / sb); kf = int((f + sf - 1) / sf) + k = (kb > kf) ? kb : kf + if (k < 1) k = 1 + w = k + if (k > m + 0) k = m + 0 + if (k > f + 0) k = f + 0 + printf "%d %d\n", k, w + }' +} + +batch_count() { + awk -v f="$1" -v b="$2" -v bf="$3" -v bb="$4" -v m="$5" 'BEGIN { + w = 0 + for (;;) { + kb = int((b + bb - 1) / bb); kf = int((f + bf - 1) / bf) + k = (kb > kf) ? kb : kf + if (k < 1) k = 1 + if (w == 0) w = k + if (k <= m + 0) break + bb = bb + int(bb / 8) + 1 + bf = bf + int(bf / 8) + 1 + } + printf "%d %d %.0f %d\n", k, bf, bb, w + }' +} + +#---------------------------------------------------------------------------# +# Why pacman refused one package +# +# refusal_reason LOG PKG_RC SIG_RC -> "CLASSLINE", from pacman's output +# for that one package (pacman -Swdd, pacman 7.1 wording), and from the exit +# status of the wrapper asked for the package and for its .sig alone: +# sig-missing no signature could be read: pacman says a required one is +# missing or the .sig did not download, or the wrapper serves +# the package and no .sig +# sig-refused pacman read a signature and refused it: an expired or unknown +# key, a trust it does not accept, or a signature that is not +# valid for the bytes. pacman 7.1 takes a key whose primary has +# expired for a missing one: it asks a keyserver, then says +# "required key missing from keyring" (measured 2026-10-14). +# other anything else: the wrapper does not serve the package, a +# checksum, an architecture, a name pacman does not know +# A line of the other class wins over any signature line: a package with two +# faults is not excused for the one that is upstream's. The wrapper's own lines +# are not read: they are its wording, which may change, not pacman's verdict. +#---------------------------------------------------------------------------# +refusal_reason() { + awk -v pkg="${2:-}" -v sig="${3:-}" ' + /^pacman-oci:/ { next } + { + line = $0 + if (match(line, /failed retrieving file '\''[^'\'']*'\''/)) { + name = substr(line, RSTART, RLENGTH) + if (name ~ /\.sig'\''$/) { if (missing == "") missing = line } + else if (other == "") other = line + next + } + if (line ~ /invalid or corrupted package \(checksum\)|does not have a valid architecture|package architecture is not valid/) { + if (other == "") other = line + next + } + if (line ~ /missing required signature/) { if (missing == "") missing = line; next } + if (line ~ /signature from ".*" is (expired|invalid|unknown trust|marginal trust|never trusted)/ || + line ~ /key ".*" (is expired|is unknown|is disabled|could not be looked up remotely)/ || + line ~ /required key missing from keyring|signature format error|unsupported signature format/ || + line ~ /invalid or corrupted package \(PGP signature\)/) { + if (refused == "") refused = line + next + } + if (line ~ /^error: / && line !~ /^error: failed to commit transaction/ && other == "") other = line + } + END { + if (pkg != "0") print "other\tthe wrapper does not serve the package itself (exit " pkg ")" + else if (other != "") print "other\t" other + else if (refused != "") print "sig-refused\t" refused + else if (missing != "") print "sig-missing\t" missing + else if (sig != "0") print "sig-missing\tthe wrapper serves the package and no .sig (exit " sig ")" + else print "other\tpacman named no reason" + } + ' "$1" +} diff --git a/scripts/mirror-file b/scripts/mirror-file index f60d586..633921c 100755 --- a/scripts/mirror-file +++ b/scripts/mirror-file @@ -7,10 +7,12 @@ # scripts/mirror-file SHARD.tsv LINE # # LINE is a line number of SHARD.tsv, a work line from scripts/plan: -# target filename tag sha256 csize name action pgpsig +# target filename tag sha256 csize name action builddate pgpsig # -# One result line is written to $RESULTS/LINE.tsv whatever happens, by the EXIT -# trap when nothing else wrote it, so an item that is killed is still counted: +# One result line is written to $RESULTS/LINE.tsv as soon as the item ends, +# whatever happens, by the EXIT trap when nothing else wrote it, so an item +# that is killed is still counted, and a shard that runs out of time keeps +# the results of every item that ended: # target filename status seconds detail # # pushed pushed now, and read back holding this file and its signature @@ -31,9 +33,15 @@ # is absent only when every server answers 404 for it, and then the file is # mirrored unsigned, as upstream ships it, and counts as present from then on. # +# ⭐ The manifest is the same bytes whoever pushes it: its created annotation is +# the package's %BUILDDATE%, and no annotation names the server the bytes came +# from (the result line does). Two writers of one file push one manifest. +# # Environment: RESULTS, WORK, ORAS, ORAS_CONFIG, SHARD_INDEX, MIRROR_NAMESPACE, -# REGISTRY_USER, REGISTRY_TOKEN, MIRROR_MIN_RATE (bytes per second a push may -# not fall below; default 2097152). +# REGISTRY_USER, REGISTRY_TOKEN, REGISTRY_TOKEN_CACHE (lib.sh), MIRROR_MIN_RATE +# (bytes per second a push may not fall below; default 2097152). A push +# attempt gets 300 s plus the time at that rate, and never more than 600 s, +# GHCR's limit for one upload. # # Exit status: 0 pushed or present, 1 anything else. set -euo pipefail @@ -71,8 +79,9 @@ on_exit() { } trap on_exit EXIT -IFS=$'\t' read -r target file tag sha256 csize name _ pgpsig < <(sed -n "${line}p" "$shard") +IFS=$'\t' read -r target file tag sha256 csize name _ builddate pgpsig < <(sed -n "${line}p" "$shard") [ -n "${target:-}" ] && [ -n "${file:-}" ] || die "line $line of $shard is empty" +created="$(iso_of_epoch "${builddate:-}")" || die "line $line of $shard has build date '${builddate:-}', not seconds since the epoch" conf="$TARGETS_DIR/$target.conf" target_check "$conf" @@ -220,18 +229,21 @@ fi # the media type at the last colon, after any : in the version. #---------------------------------------------------------------------------# deadline=$((300 + csize / MIN_RATE)) +if [ "$deadline" -gt 600 ]; then + deadline=600 +fi source_url="${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:-pkgforge-dev/mirror-archlinux}" mapfile -t oargs < <(oras_args) push_args=( push "${oargs[@]}" --artifact-type application/vnd.pkgforge.archlinux.package.v1 + --annotation "org.opencontainers.image.created=$created" --annotation "org.opencontainers.image.source=$source_url" --annotation "org.opencontainers.image.title=$file" --annotation "dev.pkgforge.mirror.arch=$arch" --annotation "dev.pkgforge.mirror.repo=$repo" --annotation "dev.pkgforge.mirror.pkgname=$name" --annotation "dev.pkgforge.mirror.sha256=$sha256" - --annotation "dev.pkgforge.mirror.upstream=$got_from" "$ref" "./$file:$PKG_MEDIA_TYPE" ) diff --git a/scripts/mirror-shard b/scripts/mirror-shard index 5cfd090..0b79136 100755 --- a/scripts/mirror-shard +++ b/scripts/mirror-shard @@ -13,13 +13,21 @@ # counted as failed, so an item that died without writing one, or one xargs # never started because it gave up early, is not silently missing. # +# Each item writes its result to OUT/lines/LINE.tsv the moment it ends. A shard +# killed before results.tsv is written (the job ran out of time) still leaves +# those, and scripts/stage reads them: what was pushed counts. +# +# Every mirror-file of the shard shares one pull token, kept in a file in the +# work directory (lib.sh REGISTRY_TOKEN_CACHE). +# # The work goes in the directory with the most free space among $RUNNER_TEMP, # /mnt and /tmp, counting each filesystem once. scripts/debloat-runner runs # only when the PARALLEL largest files of the shard would not fit; if they # still do not fit after it, PARALLEL comes down until they do. # # Environment: PARALLEL (default 4), SHARD_INDEX, DEBLOAT (auto, always or -# never; default auto), MIRROR_NAMESPACE, REGISTRY_USER, REGISTRY_TOKEN. +# never; default auto), MIRROR_NAMESPACE, REGISTRY_USER, REGISTRY_TOKEN, +# ORAS_CACHE (scripts/install-oras). # # Exit status: 0 every file is on the registry, 1 otherwise. set -euo pipefail @@ -102,8 +110,11 @@ fi log "$lines file(s), $PARALLEL at a time, in $best_dir ($((best_free / 1073741824)) GiB free)" WORK="$(mktemp -d "$best_dir/mirror.XXXXXX")" -RESULTS="$WORK/results" +RESULTS="$out/lines" +rm -rf "$RESULTS" mkdir -p "$RESULTS" "$WORK/items" "$WORK/bin" +REGISTRY_TOKEN_CACHE="$WORK/tokens" +mkdir -m 0700 "$REGISTRY_TOKEN_CACHE" cleanup() { rm -rf "$WORK" } @@ -116,7 +127,7 @@ trap cleanup EXIT ORAS="$("$REPO_ROOT/scripts/install-oras" "$WORK/bin")" ORAS_CONFIG="$WORK/oras-auth.json" oras_login "$ORAS" "$ORAS_CONFIG" -export RESULTS WORK ORAS ORAS_CONFIG SHARD_INDEX MIRROR_NAMESPACE +export RESULTS WORK ORAS ORAS_CONFIG SHARD_INDEX MIRROR_NAMESPACE REGISTRY_TOKEN_CACHE rc=0 seq 1 "$lines" | xargs -d '\n' -n 1 -P "$PARALLEL" "$REPO_ROOT/scripts/mirror-file" "$shard" || rc=$? diff --git a/scripts/plan b/scripts/plan index a14ddee..5e062d7 100755 --- a/scripts/plan +++ b/scripts/plan @@ -1,14 +1,18 @@ #!/usr/bin/env bash # -# Decide what one sync run has to do, per target, and split it into shards. +# Decide what one sync run has to do, per target, and split each target's work +# into shards of its own. # # scripts/plan OUT [TARGET.conf ...] # # No TARGET means every enabled target in $TARGETS_DIR. For each target: # -# 1. Read the database from every db_server, the distribution's own hosts. +# 1. Read the database from every db_server, the distribution's own hosts, +# each fetch with a deadline; a db_server that fails gives way to the next. # Refuse one that is not whole and well formed: every field checked -# against its grammar, as many records as the archive has members. +# against its grammar, one record per package directory (lib.sh db_check; +# a directory whose desc holds only NUL bytes is counted and named, not +# refused). # 2. Read the promoted database the way a consumer does: anonymously, through # client/pacman-oci. None on a first run. A package that answers only with # the workflow's token is private, and is planned with that token. @@ -21,25 +25,41 @@ # upstream downgrade. # 4. The same bytes as the promoted database: the target is current. # 5. Otherwise every file is skip, when the promoted database names it with -# the same sha256 (no request made), or todo. scripts/mirror-file reads -# each todo file's tag before writing it, and never overwrites one. +# the same sha256 and the same %PGPSIG% (no request made), or todo. +# scripts/mirror-file reads each todo file's tag before writing it: it +# never overwrites a package layer, and it repairs a signature layer that +# upstream signed again. +# 6. The todo files of the target go into its own shards: k = the larger of +# ceil(bytes / SHARD_BYTES) and ceil(files / SHARD_FILES), at least 1, at +# most MAX_SHARDS and the file count; largest file first onto the lightest +# shard. A target never waits for another target's shards. # # OUT//state current pending behind held error # OUT//why one line saying what was measured # OUT//upstream.db the chosen database, byte for byte, and .files and # any .sig the upstream publishes beside them # OUT//records.tsv its records (lib.sh db_records) +# OUT//nul.txt its package directories whose desc is NUL bytes only # OUT//todo.tsv the files this run mirrors +# OUT//shards/.tsv the work lines of shard n, for scripts/mirror-shard # OUT//meta.env KEY=VALUE facts stage writes into the manifest -# OUT/shards/.tsv work lines for scripts/mirror-shard -# OUT/matrix.json the shard list for the workflow's matrix +# OUT/targets.json {"include":[{"target":T,"shards":[0,1,..]},..]}: the +# pending targets with files to mirror +# OUT/pending.json the same shape, "shards":[], for the pending targets +# with no file to mirror (a database that only +# dropped packages): they still stage and verify # OUT/summary.tsv target, state, files, MiB, why # -# Work line: target filename tag sha256 csize name action pgpsig, tab separated. +# Work line: target filename tag sha256 csize name action builddate pgpsig, tab +# separated. pgpsig is last because it may be empty. +# +# In Actions, GITHUB_OUTPUT gets targets (targets.json and pending.json as one +# matrix), has_targets, and slotN=TARGET for each pending target, which the +# workflow's upload steps read to upload OUT/ as plan-. # # Environment: SHARD_BYTES (default 536870912), SHARD_FILES (400), MAX_SHARDS -# (16), ALLOW_OLDER, MIRROR_NAMESPACE, REGISTRY_USER and REGISTRY_TOKEN (used -# only to tell a private package from an absent one). +# (32, per target), ALLOW_OLDER, MIRROR_NAMESPACE, REGISTRY_USER and +# REGISTRY_TOKEN (used only to tell a private package from an absent one). # # Exit status: 0 planned, whatever state each target is in; 1 no plan could be # made at all. @@ -57,12 +77,17 @@ shift SHARD_BYTES="${SHARD_BYTES:-536870912}" SHARD_FILES="${SHARD_FILES:-400}" -MAX_SHARDS="${MAX_SHARDS:-16}" +MAX_SHARDS="${MAX_SHARDS:-32}" for v in SHARD_BYTES SHARD_FILES MAX_SHARDS; do case "${!v}" in '' | *[!0-9]* | 0) die "$v must be a positive integer, not '${!v}'" ;; esac done +# Whole-transfer deadlines for what plan reads from upstream, in seconds. The +# largest measured (2026-10-14): x86_64 extra.files, 51452041 bytes. +DB_MAX_TIME=300 +FILES_MAX_TIME=900 +SIG_MAX_TIME=60 if [ "$#" -eq 0 ]; then for conf in "$TARGETS_DIR"/*.conf; do @@ -74,7 +99,7 @@ if [ "$#" -eq 0 ]; then fi [ "$#" -gt 0 ] || die "no enabled target" "reproduce: ls $TARGETS_DIR" -mkdir -p "$out/shards" +mkdir -p "$out" fetched="$(iso_now)" # set_state DIR STATE WHY @@ -84,30 +109,6 @@ set_state() { log "$(basename "$1"): $2: $3" } -# check_db DB RECORDS -> 0 a whole, well-formed database; prints why not -check_db() { - local db="$1" rec="$2" members records - if ! db_records "$db" "$rec"; then - echo "it does not read as a tar archive" - return 1 - fi - records="$(awk 'END { print NR + 0 }' "$rec")" - members="$(db_members "$db")" - if [ "$records" -eq 0 ]; then - echo "it lists no package" - return 1 - fi - if [ "$records" -ne "$members" ]; then - echo "it holds $members package directories and $records readable records" - return 1 - fi - if ! db_validate "$rec" > "$rec.bad"; then - echo "$(awk 'END { print NR + 0 }' "$rec.bad") record(s) fail their grammar, first: $(sed -n 1p "$rec.bad")" - return 1 - fi - return 0 -} - allowed_older() { case ",${ALLOW_OLDER:-}," in *",$1,"*) return 0 ;; @@ -115,6 +116,50 @@ allowed_older() { esac } +# nul_note DIR -> " (N package directory(ies) hold only NUL bytes: a b)", or nothing +nul_note() { + local n + n="$(awk 'END { print NR + 0 }' "$1/nul.txt")" + [ "$n" -gt 0 ] || return 0 + printf '; %s package directory(ies) whose desc holds only NUL bytes, passed over: %s' "$n" "$(tr '\n' ' ' < "$1/nul.txt" | sed 's/ $//')" +} + +# shard_target DIR NAME: OUT//shards/.tsv from its todo.tsv +shard_target() { + local dir="$1" name="$2" files bytes k wanted + mkdir -p "$dir/shards" + files="$(awk 'END { print NR + 0 }' "$dir/todo.tsv")" + bytes="$(awk -F '\t' '{ s += $5 } END { printf "%.0f\n", s }' "$dir/todo.tsv")" + read -r k wanted <<< "$(shard_count "$files" "$bytes" "$SHARD_FILES" "$SHARD_BYTES" "$MAX_SHARDS")" + if [ "$wanted" -gt "$MAX_SHARDS" ] && [ "$k" -gt 0 ]; then + log "$name: $files file(s) and $((bytes / 1048576)) MiB want $wanted shard(s); MAX_SHARDS=$MAX_SHARDS caps them at $k: $(awk -v b="$bytes" -v k="$k" 'BEGIN { printf "%.2f", b / k / 1073741824 }') GiB and $(((files + k - 1) / k)) file(s) per shard" + fi + [ "$k" -gt 0 ] || return 0 + sort -t $'\t' -k5,5nr -k2,2 "$dir/todo.tsv" | awk -F '\t' -v k="$k" -v d="$dir/shards" ' + { + best = 0 + for (i = 1; i < k; i++) if (load[i] < load[best]) best = i + load[best] += $5 + print > (d "/" best ".tsv") + } + ' +} + +# prune DIR: drop what only plan reads, so the target's artifact holds what +# stage and mirror read and nothing else +prune() { + local dir="$1" f + for f in "$dir"/cand/*.db "$dir"/cand/*.tsv "$dir"/cand/*.tsv.*; do + [ -e "$f" ] || continue + case "$f" in + */cand/list.tsv | */cand/kept.tsv) ;; + *) rm -f -- "$f" ;; + esac + done + rm -f -- "$dir/promoted.db" "$dir/promoted.db.part" "$dir/promoted.tsv" "$dir/pairs.tsv" \ + "$dir/files.tsv" "$dir"/files.tsv.* "$dir/latest.json" +} + plan_target() { local conf="$1" arch repo name dir repo_path server rc why i target_check "$conf" @@ -127,6 +172,7 @@ plan_target() { mkdir -p "$dir/cand" : > "$dir/todo.tsv" : > "$dir/meta.env" + : > "$dir/nul.txt" #-------------------------------------------------------------------------# # 1. Candidates, from the distribution's own servers only @@ -137,17 +183,21 @@ plan_target() { [ -n "$server" ] || continue i=$((i + 1)) rc=0 - fetch_upstream "$server/$repo.db" "$dir/cand/$i.db" || rc=$? + fetch_upstream "$server/$repo.db" "$dir/cand/$i.db" "$DB_MAX_TIME" || rc=$? if [ "$rc" -ne 0 ]; then - log "$name: $server/$repo.db did not answer (status $rc)" + log "$name: $server/$repo.db did not answer within ${DB_MAX_TIME}s (status $rc); the next db_server is tried" continue fi rc=0 - why="$(check_db "$dir/cand/$i.db" "$dir/cand/$i.tsv")" || rc=$? + why="$(db_check "$dir/cand/$i.db" "$dir/cand/$i.tsv")" || rc=$? if [ "$rc" -ne 0 ]; then - warn "$name: refused the database from $server: $why" + warn "$name: refused the database from $server: $why" \ + "reproduce: curl -fsSLO $server/$repo.db && bash -c '. scripts/lib.sh; db_check $repo.db /tmp/records.tsv'" continue fi + if [ -s "$dir/cand/$i.tsv.nul" ]; then + log "$name: $server/$repo.db: $(awk 'END { print NR + 0 }' "$dir/cand/$i.tsv.nul") package directory(ies) whose desc holds only NUL bytes: $(tr '\n' ' ' < "$dir/cand/$i.tsv.nul")" + fi printf '%s\t%s\t%s\n' "$i" "$server" "$(sha256sum "$dir/cand/$i.db" | cut -d ' ' -f 1)" >> "$dir/cand/list.tsv" done <<< "$(target_all "$conf" db_server)" @@ -234,6 +284,7 @@ plan_target() { done < "$dir/cand/list.tsv" [ "$other" -eq 0 ] || continue kept=$((kept + 1)) + printf '%s\t%s\t%s\n' "$idx" "$srv" "$sha" >> "$dir/cand/kept.tsv" if [ -n "$promoted_sha" ] && [ "$sha" = "$promoted_sha" ] && [ -z "$promoted_kept" ]; then promoted_kept="$idx"$'\t'"$srv"$'\t'"$sha" fi @@ -264,6 +315,7 @@ plan_target() { IFS=$'\t' read -r best chosen_server chosen_sha <<< "$chosen" cp "$dir/cand/$best.db" "$dir/upstream.db" cp "$dir/cand/$best.tsv" "$dir/records.tsv" + cp "$dir/cand/$best.tsv.nul" "$dir/nul.txt" #-------------------------------------------------------------------------# # 4. Current? @@ -276,24 +328,29 @@ plan_target() { printf 'TARGET=%s\nARCH=%s\nREPO=%s\nREPOSITORY=%s\n' "$name" "$arch" "$repo" "$repo_path" printf 'UPSTREAM=%s\nDB_SHA256=%s\nPROMOTED_DB_SHA256=%s\n' "$chosen_server" "$chosen_sha" "$promoted_sha" printf 'PACKAGES=%s\nBYTES=%s\nNEWEST_BUILDDATE=%s\nFETCHED=%s\nPRIVATE=%s\n' "$packages" "$bytes" "$newest" "$fetched" "$private" + printf 'NUL_MEMBERS=%s\n' "$(awk 'END { print NR + 0 }' "$dir/nul.txt")" } > "$dir/meta.env" if [ "$chosen_sha" = "$promoted_sha" ]; then - set_state "$dir" current "the promoted $repo.db is upstream's ($chosen_sha)" + set_state "$dir" current "the promoted $repo.db is upstream's ($chosen_sha)$(nul_note "$dir")" return 0 fi #-------------------------------------------------------------------------# # The files database and any signatures, from a db_server that served these # very database bytes. The files database must name the same packages with - # the same sha256, or pacman -F would describe other packages. + # the same sha256, or pacman -F would describe other packages. A server that + # fails gives way to the next one with the same bytes. #-------------------------------------------------------------------------# - local files_ok=0 sig_server="" + local files_ok=0 s sig_rc sig_done while IFS=$'\t' read -r idx srv sha; do [ "$sha" = "$chosen_sha" ] || continue rc=0 - fetch_upstream "$srv/$repo.files" "$dir/upstream.files" || rc=$? - [ "$rc" -eq 0 ] || continue + fetch_upstream "$srv/$repo.files" "$dir/upstream.files" "$FILES_MAX_TIME" || rc=$? + if [ "$rc" -ne 0 ]; then + log "$name: $srv/$repo.files did not answer within ${FILES_MAX_TIME}s (status $rc)" + continue + fi if ! db_records "$dir/upstream.files" "$dir/files.tsv"; then warn "$name: $srv/$repo.files does not read as a tar archive" continue @@ -302,87 +359,89 @@ plan_target() { warn "$name: $srv/$repo.files names other packages than $repo.db" continue fi + # A signature is absent only when the server says so with a 404; any other + # failure would otherwise turn a signed database into an unsigned one. + sig_done=1 + for s in db files; do + rm -f "$dir/upstream.$s.sig" + sig_rc=0 + fetch_upstream "$srv/$repo.$s.sig" "$dir/upstream.$s.sig" "$SIG_MAX_TIME" || sig_rc=$? + case "$sig_rc" in + 0) log "$name: upstream signs $repo.$s" ;; + 4) ;; + *) + log "$name: $srv/$repo.$s.sig did not answer (status $sig_rc)" + sig_done=0 + ;; + esac + done + [ "$sig_done" -eq 1 ] || continue files_ok=1 - sig_server="$srv" break done < "$dir/cand/list.tsv" if [ "$files_ok" -ne 1 ]; then - set_state "$dir" error "no db_server that served $repo.db $chosen_sha also served a $repo.files that matches it" + set_state "$dir" error "no db_server that served $repo.db $chosen_sha also served a $repo.files that matches it, and an answer for its signatures" return 0 fi - - # A signature is absent only when the server says so with a 404; any other - # failure would otherwise turn a signed database into an unsigned one. - local s - for s in db files; do - rc=0 - fetch_upstream "$sig_server/$repo.$s.sig" "$dir/upstream.$s.sig" || rc=$? - case "$rc" in - 0) log "$name: upstream signs $repo.$s" ;; - 4) - if [ "$(target_get "$conf" db_signed)" = yes ]; then - set_state "$dir" error "db_signed = yes and $sig_server/$repo.$s.sig answered 404" - return 0 - fi - ;; - *) - set_state "$dir" error "$sig_server/$repo.$s.sig did not answer" + if [ "$(target_get "$conf" db_signed)" = yes ]; then + for s in db files; do + if [ ! -f "$dir/upstream.$s.sig" ]; then + set_state "$dir" error "db_signed = yes and $repo.$s.sig answered 404" return 0 - ;; - esac - done + fi + done + fi #-------------------------------------------------------------------------# # 5. Skip or todo #-------------------------------------------------------------------------# cut -f 1 "$dir/records.tsv" | "$WRAPPER" --tags > "$dir/pairs.tsv" awk -F '\t' -v OFS='\t' -v target="$name" ' - FILENAME == ARGV[1] { promoted[$1] = $5; next } + FILENAME == ARGV[1] { promoted[$1] = $5; psig[$1] = $8; next } FILENAME == ARGV[2] { tag[$1] = $2; next } - ($1 in promoted) && promoted[$1] == $5 { next } - { print target, $1, tag[$1], $5, $6, $2, "todo", $8 } + ($1 in promoted) && promoted[$1] == $5 && psig[$1] == $8 { next } + { print target, $1, tag[$1], $5, $6, $2, "todo", $7, $8 } ' "$dir/promoted.tsv" "$dir/pairs.tsv" "$dir/records.tsv" > "$dir/todo.tsv" + local resigned + resigned="$(awk -F '\t' 'FILENAME == ARGV[1] { p[$1] = $5; s[$1] = $8; next } ($2 in p) && p[$2] == $4 && s[$2] != $9 { n++ } END { print n + 0 }' \ + "$dir/promoted.tsv" "$dir/todo.tsv")" + if [ "$resigned" -gt 0 ]; then + log "$name: $resigned file(s) keep their bytes and carry a new %PGPSIG%; mirror-file repairs their signature layers" + fi + + #-------------------------------------------------------------------------# + # 6. This target's shards + #-------------------------------------------------------------------------# + shard_target "$dir" "$name" - set_state "$dir" pending "$(awk 'END { print NR + 0 }' "$dir/todo.tsv") file(s) to mirror, $(awk -F '\t' '{ s += $5 } END { printf "%.0f", s / 1048576 }' "$dir/todo.tsv") MiB, database from $chosen_server" + set_state "$dir" pending "$(awk 'END { print NR + 0 }' "$dir/todo.tsv") file(s) to mirror, $(awk -F '\t' '{ s += $5 } END { printf "%.0f", s / 1048576 }' "$dir/todo.tsv") MiB, in $(find "$dir/shards" -name '*.tsv' | awk 'END { print NR + 0 }') shard(s), database from $chosen_server$(nul_note "$dir")" } for conf in "$@"; do plan_target "$conf" + prune "$out/$(target_get "$conf" arch)-$(target_get "$conf" repo)" done #---------------------------------------------------------------------------# -# Shards: largest file first onto the lightest shard, which keeps the shards -# within one large file of each other. +# The matrices: one entry per pending target, with its shards #---------------------------------------------------------------------------# -cat "$out"/*/todo.tsv > "$out/all-todo.tsv" -files="$(awk 'END { print NR + 0 }' "$out/all-todo.tsv")" -bytes="$(awk -F '\t' '{ s += $5 } END { printf "%.0f\n", s }' "$out/all-todo.tsv")" -if [ "$files" -eq 0 ]; then - k=0 -else - k="$(awk -v b="$bytes" -v f="$files" -v sb="$SHARD_BYTES" -v sf="$SHARD_FILES" -v m="$MAX_SHARDS" 'BEGIN { - kb = int((b + sb - 1) / sb); kf = int((f + sf - 1) / sf) - k = (kb > kf) ? kb : kf - if (k < 1) k = 1 - if (k > m) k = m - if (k > f) k = f - print k - }')" -fi - -rm -f "$out"/shards/*.tsv -if [ "$k" -gt 0 ]; then - sort -t $'\t' -k5,5nr -k2,2 "$out/all-todo.tsv" | awk -F '\t' -v k="$k" -v dir="$out/shards" ' - { - best = 0 - for (i = 1; i < k; i++) if (load[i] < load[best]) best = i - load[best] += $5 - print > (dir "/" best ".tsv") - } - ' -fi - -jq -cn --argjson k "$k" '{shard: [range(0; $k)]}' > "$out/matrix.json" +: > "$out/with-files.jsonl" +: > "$out/no-files.jsonl" +for d in "$out"/*/; do + d="${d%/}" + [ -f "$d/state" ] || continue + [ "$(cat "$d/state")" = pending ] || continue + t="$(basename "$d")" + shards="$(find "$d/shards" -name '*.tsv' -type f | sed 's|.*/||; s|\.tsv$||' | sort -n | jq -R 'tonumber' | jq -cs .)" + if [ "$shards" = '[]' ]; then + jq -cn --arg t "$t" '{target: $t, shards: []}' >> "$out/no-files.jsonl" + else + jq -cn --arg t "$t" --argjson s "$shards" '{target: $t, shards: $s}' >> "$out/with-files.jsonl" + fi +done +jq -cs '{include: .}' "$out/with-files.jsonl" > "$out/targets.json" +jq -cs '{include: .}' "$out/no-files.jsonl" > "$out/pending.json" +rm -f "$out/with-files.jsonl" "$out/no-files.jsonl" { printf 'target\tstate\tfiles\tMiB\twhy\n' @@ -396,12 +455,12 @@ jq -cn --argjson k "$k" '{shard: [range(0; $k)]}' > "$out/matrix.json" done } > "$out/summary.tsv" -pending="$(awk -F '\t' 'NR > 1 && $2 == "pending" { n++ } END { print n + 0 }' "$out/summary.tsv")" -log "$files file(s), $((bytes / 1048576)) MiB, in $k shard(s); $pending target(s) pending" +pending="$(jq -s '[.[].include[]] | length' "$out/targets.json" "$out/pending.json")" +log "$pending target(s) pending: $(jq -r '[.include[] | "\(.target) (\(.shards | length) shard(s))"] | join(", ")' "$out/targets.json")$(jq -r '[.include[].target] | if length > 0 then "; with no file to mirror: " + join(", ") else "" end' "$out/pending.json")" if [ -n "${GITHUB_OUTPUT:-}" ]; then { - printf 'matrix=%s\n' "$(cat "$out/matrix.json")" - if [ "$k" -gt 0 ]; then echo "has_files=true"; else echo "has_files=false"; fi - if [ "$pending" -gt 0 ]; then echo "has_work=true"; else echo "has_work=false"; fi + printf 'targets=%s\n' "$(jq -cs '{include: [.[].include[]]}' "$out/targets.json" "$out/pending.json")" + if [ "$pending" -gt 0 ]; then echo "has_targets=true"; else echo "has_targets=false"; fi + jq -r -s '[.[].include[].target] | to_entries[] | "slot\(.key)=\(.value)"' "$out/targets.json" "$out/pending.json" } >> "$GITHUB_OUTPUT" fi diff --git a/scripts/promote b/scripts/promote index d4dc993..787fb53 100755 --- a/scripts/promote +++ b/scripts/promote @@ -19,7 +19,12 @@ # latest last, once everything it points at is in place # # ⛔ Every batch of the target must have passed, for this very staged digest; a -# batch with no verdict, or a verdict about another digest, counts as failed. ⛔ latest never moves backwards: immediately before tagging, the +# batch with no verdict, or a verdict about another digest, counts as failed. +# A batch passes with upstream-refused packages in it (scripts/verify-batch): +# the mirror serves exactly upstream's file, and pacman refuses it from +# upstream too. They are counted in the log. +# +# ⛔ latest never moves backwards: immediately before tagging, the # fetched time of the snapshot latest names now is read back, and a staged # snapshot fetched earlier is refused. A run re-run days later, or a second # writer, cannot roll consumers back. (scripts/plan holds the content floor: @@ -81,6 +86,10 @@ for b in "$dir"/batches/*.txt; do done [ "$batches" -gt 0 ] || die "$target was staged with no verification batch" [ "$passed" -eq "$batches" ] || die "$passed of $batches verification batch(es) of $target passed; nothing is promoted" +refused="$(find "$verify" -name "refused-$target-*.tsv" -type f -exec cat {} + | awk -F '\t' '!seen[$1]++ { n++ } END { print n + 0 }')" +if [ "$refused" -gt 0 ]; then + log "$target: $refused upstream-refused file(s) in the passed batches: the mirror serves upstream's bytes and signature, and pacman refuses them from upstream too" +fi work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT diff --git a/scripts/report b/scripts/report index dc2b437..b46c703 100755 --- a/scripts/report +++ b/scripts/report @@ -3,13 +3,27 @@ # Say what one sync run did, per target, and keep the two issues that do not # heal by themselves in step with it. # -# scripts/report PLAN STAGE PROMOTED +# scripts/report collect TARGET STAGE VERIFY PROMOTED OUT +# scripts/report PLAN SUMMARIES # -# PLAN and STAGE are the plan and stage outputs (STAGE may be missing when the -# run stopped before it), PROMOTED a file of promote results, one line per -# target: target status detail. +# collect, once per target at the end of its chain: what that target's stage, +# verify and promote steps left, into OUT, a small directory the workflow +# uploads as summary-TARGET. STAGE is scripts/stage's output (it may be +# missing when the chain stopped before it), VERIFY holds the verify-*.tsv +# and refused-*.tsv of the target at any depth, PROMOTED a file with promote's +# line for the target (target status detail), empty or missing when promote +# did not run. OUT gets: +# state.tsv target state why: promote's word wins over verify's, over +# stage's +# verify.tsv every verdict line of the target +# refused.tsv file reason: every upstream-refused package of the target +# failing.tsv, replaced.tsv, settings-url as stage left them # -# Writes the table to $GITHUB_STEP_SUMMARY when set, otherwise to stdout. +# The second form writes one table for the run: every target of the plan +# (PLAN/summary.tsv), with the state each chain collected into +# SUMMARIES//state.tsv, and under it the files that hold a target and +# the upstream-refused packages, per target. To $GITHUB_STEP_SUMMARY when set, +# otherwise to stdout. # # Issues (scripts/track-issue), only when TRACK_ISSUES=1: # visibility- raised when the package reads only with the workflow's @@ -22,19 +36,79 @@ # # Exit status: 0 no target is held, failed, replaced, private or in error (a # target behind upstream's own servers, or staged on a branch, is not a -# failure); 1 at least one is, so the run shows red; 2 usage. +# failure; neither is an upstream-refused package); 1 at least one is, so the +# run shows red; 2 usage. set -euo pipefail SCRIPT_NAME=report # shellcheck source=lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -plan="${1:-}" -stage="${2:-}" -promoted="${3:-}" -if [ -z "$plan" ] || [ -z "$stage" ] || [ -z "$promoted" ]; then - echo "usage: scripts/report PLAN STAGE PROMOTED" >&2 +usage() { + echo "usage: scripts/report collect TARGET STAGE VERIFY PROMOTED OUT | scripts/report PLAN SUMMARIES" >&2 exit 2 +} + +#---------------------------------------------------------------------------# +# collect +#---------------------------------------------------------------------------# +if [ "${1:-}" = collect ]; then + [ "$#" -eq 6 ] || usage + target="$2" stage="$3" verify="$4" promoted="$5" sout="$6" + case "$target" in + '' | *[!a-z0-9_.-]*) usage ;; + esac + mkdir -p "$sout" + : > "$sout/verify.tsv" + : > "$sout/refused.tsv" + if [ -d "$verify" ]; then + find "$verify" -name "verify-$target-*.tsv" -type f -exec cat {} + | sort -t $'\t' -k2,2n >> "$sout/verify.tsv" + find "$verify" -name "refused-$target-*.tsv" -type f -exec cat {} + | awk -F '\t' '!seen[$1]++' >> "$sout/refused.tsv" + fi + state="" why="" + if [ -f "$stage/$target/state" ]; then + state="$(cat "$stage/$target/state")" + why="$(cat "$stage/$target/why")" + for f in failing.tsv replaced.tsv settings-url; do + if [ -f "$stage/$target/$f" ]; then + cp "$stage/$target/$f" "$sout/$f" + fi + done + else + state=failed + why="the chain left no stage result for this target; read the run's mirror and stage jobs" + fi + if [ "$state" = staged ]; then + digest="$(sed -n 's/^DIGEST=//p' "$stage/$target/staged.env")" + failed_line="$(awk -F '\t' -v d="$digest" '$3 != "passed" || $5 != d { print "batch " $2 ": " $3 ": " $4; exit }' "$sout/verify.tsv")" + if [ -n "$failed_line" ]; then + state=failed + why="$failed_line" + elif [ "$(awk 'END { print NR + 0 }' "$sout/verify.tsv")" -lt "$(find "$stage/$target/batches" -name '*.txt' | awk 'END { print NR + 0 }')" ]; then + state=failed + why="$(awk 'END { print NR + 0 }' "$sout/verify.tsv") verdict(s) for $(find "$stage/$target/batches" -name '*.txt' | awk 'END { print NR + 0 }') batch(es)" + fi + fi + if [ -f "$promoted" ] && [ -s "$promoted" ]; then + line="$(awk -F '\t' -v t="$target" '$1 == t { l = $2 "\t" $3 } END { print l }' "$promoted")" + if [ -n "$line" ]; then + IFS=$'\t' read -r state why <<< "$line" + fi + fi + refused_n="$(awk 'END { print NR + 0 }' "$sout/refused.tsv")" + if [ "$refused_n" -gt 0 ]; then + why="$why; $refused_n upstream-refused" + fi + printf '%s\t%s\t%s\n' "$target" "$state" "$(printf '%s' "$why" | tr '\t\n' ' ')" > "$sout/state.tsv" + log "$target: $state: $why" + exit 0 fi + +#---------------------------------------------------------------------------# +# The run's table +#---------------------------------------------------------------------------# +[ "$#" -eq 2 ] || usage +plan="$1" +summaries="$2" summary="${GITHUB_STEP_SUMMARY:-/dev/stdout}" run_url="${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:-pkgforge-dev/mirror-archlinux}/actions/runs/${GITHUB_RUN_ID:-0}" @@ -46,35 +120,52 @@ fi work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT -# target -> final state and why: promote's word wins over stage's over plan's +# target -> final state and why: the chain's word wins over the plan's tail -n +2 "$plan/summary.tsv" | cut -f 1,2,5 > "$work/states.tsv" -if [ -f "$stage/summary.tsv" ]; then - tail -n +2 "$stage/summary.tsv" | cut -f 1,2,3 > "$work/stage.tsv" - awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { s[$1] = $2; w[$1] = $3; next } ($1 in s) { $2 = s[$1]; $3 = w[$1] } { print }' \ - "$work/stage.tsv" "$work/states.tsv" > "$work/merged.tsv" - mv -f "$work/merged.tsv" "$work/states.tsv" -fi -if [ -s "$promoted" ]; then - awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { s[$1] = $2; w[$1] = $3; next } ($1 in s) && $2 == "staged" { $2 = s[$1]; $3 = w[$1] } { print }' \ - "$promoted" "$work/states.tsv" > "$work/merged.tsv" - mv -f "$work/merged.tsv" "$work/states.tsv" +: > "$work/chains.tsv" +if [ -d "$summaries" ]; then + find "$summaries" -name state.tsv -type f -exec cat {} + > "$work/chains.tsv" fi +# Every pending target has a chain; one that is still pending here left no +# summary, and nothing says what became of it. +awk -F '\t' -v OFS='\t' ' + FILENAME == ARGV[1] { s[$1] = $2; w[$1] = $3; next } + ($1 in s) { $2 = s[$1]; $3 = w[$1] } + $2 == "pending" { $2 = "failed"; $3 = "its chain left no summary; read the run: " $3 } + { print } +' "$work/chains.tsv" "$work/states.tsv" > "$work/merged.tsv" +mv -f "$work/merged.tsv" "$work/states.tsv" + +# dir_of TARGET -> the directory its chain collected, or nothing +dir_of() { + find "$summaries" -name state.tsv -type f -exec awk -F '\t' -v t="$1" '$1 == t { print FILENAME }' {} + | sed -n 's|/state.tsv$||p' | sed -n 1p +} { printf '### Sync\n\n| target | state | what was measured |\n| --- | --- | --- |\n' awk -F '\t' '{ gsub(/\|/, "\\|", $3); printf "| %s | %s | %s |\n", $1, $2, $3 }' "$work/states.tsv" printf '\n' - for d in "$stage"/*/; do - d="${d%/}" - [ -d "$d" ] || continue + while IFS=$'\t' read -r target _ _; do + [ -n "$target" ] || continue + d="" + if [ -d "$summaries" ]; then + d="$(dir_of "$target")" + fi + [ -n "$d" ] || continue for f in failing replaced; do if [ -s "$d/$f.tsv" ]; then - printf '
%s: %s file(s)\n\n```\n' "$(basename "$d")" "$f" + printf '
%s: %s file(s)\n\n```\n' "$target" "$f" sed -n 1,200p "$d/$f.tsv" printf '```\n
\n\n' fi done - done + if [ -s "$d/refused.tsv" ]; then + printf '
%s: %s upstream-refused file(s), served as upstream serves them\n\n```\n' \ + "$target" "$(awk 'END { print NR + 0 }' "$d/refused.tsv")" + sed -n 1,200p "$d/refused.tsv" + printf '```\n
\n\n' + fi + done < "$work/states.tsv" } >> "$summary" # track KEY raise|clear TITLE BODY|NOTE @@ -97,14 +188,18 @@ while IFS=$'\t' read -r target state why; do case "$state" in held | error | replaced | private | failed) bad=1 ;; esac + d="" + if [ -d "$summaries" ]; then + d="$(dir_of "$target")" + fi case "$state" in private) # shellcheck disable=SC2016 # Markdown backticks, not command substitution { printf 'The package behind `%s` was created private, and nothing can be promoted until it is public: consumers read it with no credential.\n\n' "$target" printf 'GitHub offers no API for package visibility, so this takes one click: open the package settings, Danger Zone, Change visibility, Public.\n\n' - if [ -f "$stage/$target/settings-url" ]; then - printf '%s\n\n' "$(cat "$stage/$target/settings-url")" + if [ -n "$d" ] && [ -f "$d/settings-url" ]; then + printf '%s\n\n' "$(cat "$d/settings-url")" fi printf 'Everything is pushed and verified meanwhile; the next run promotes. Measured: %s\n\nRun: %s\n' "$why" "$run_url" } > "$work/body.md" @@ -120,9 +215,9 @@ while IFS=$'\t' read -r target state why; do { printf 'A file of `%s` changed its bytes upstream under the same name, or its tag already holds other bytes. A package tag is written once, so the target is held: the promoted database and older snapshots may name the bytes the tag holds now.\n\n' "$target" printf 'Measured: %s\n\n' "$why" - if [ -f "$stage/$target/replaced.tsv" ]; then + if [ -n "$d" ] && [ -f "$d/replaced.tsv" ]; then printf '```\n' - sed -n 1,50p "$stage/$target/replaced.tsv" + sed -n 1,50p "$d/replaced.tsv" printf '```\n\n' fi printf 'A person decides: wait for upstream to publish a new filename, or accept the new bytes under a new tag (HISTORY/design.md, write-once tags).\n\nRun: %s\n' "$run_url" diff --git a/scripts/stage b/scripts/stage index ab45f09..1615b08 100755 --- a/scripts/stage +++ b/scripts/stage @@ -1,20 +1,27 @@ #!/usr/bin/env bash # -# Stage each pending target's database: check that every file it names is on -# the registry, push the database under a staging tag, read it back, and split -# the files new since the promoted database into verification batches. +# Stage one target's database: check that every file it names is on the +# registry, push the database under a staging tag, read it back, and split the +# files new since the promoted database into verification batches. # -# scripts/stage PLAN RESULTS OUT +# scripts/stage PLAN RESULTS OUT TARGET # -# PLAN is scripts/plan's output. RESULTS holds the mirror shards' outputs -# (any depth; every results.tsv under it is read). +# PLAN is scripts/plan's output (at least PLAN/TARGET). RESULTS holds the +# target's mirror shard outputs, at any depth: every results.tsv under it is +# read, and every lines/*.tsv, which a shard killed before its results.tsv +# leaves behind. # -# OUT//state staged held replaced private error, or the plan's own -# state for a target that was not pending -# OUT//why what was measured -# OUT//staged.env DIGEST, DB_SHA256, FILES_SHA256, TAG of the staged manifest -# OUT//batches/.txt package names, one per line +# OUT/TARGET/state staged held replaced private error, or the plan's own +# state for a target that was not pending +# OUT/TARGET/why what was measured +# OUT/TARGET/staged.env DIGEST, DB_SHA256, FILES_SHA256, TAG of the staged manifest +# OUT/TARGET/batches/.txt package names, one per line +# OUT/TARGET/canaries.txt the names verify reads first, through the +# wrapper's single-file path; all in batch 0 +# OUT/TARGET/records.tsv the database's records, for verify's account of +# a package pacman refuses # OUT/verify-matrix.json {"include":[{"target":..,"batch":..},..]} +# OUT/summary.tsv target, state, why # # ⛔ A database is staged only when every file new since the promoted one has a # result of pushed or present. A file with no result or a failure holds the @@ -23,11 +30,19 @@ # itself. A file whose tag holds other bytes (replaced) holds the target until # a person decides, and stage says so at once. # +# Verification batches: every file new since the promoted database, plus the +# canaries, which are always present: the smallest package and every keyring +# package of the target, so a run with nothing new still proves the path a +# consumer takes. The number of batches is the larger of ceil(bytes / +# VERIFY_BATCH_BYTES) and ceil(files / VERIFY_BATCH_FILES); when that is more +# than VERIFY_MAX_BATCHES, both limits grow until it is not (lib.sh +# batch_count), and the log says so. +# # Environment: STAGING_TAG (default staging), VERIFY_BATCH_FILES (400), -# VERIFY_BATCH_BYTES (2147483648), MIRROR_NAMESPACE, REGISTRY_USER, -# REGISTRY_TOKEN. +# VERIFY_BATCH_BYTES (2147483648), VERIFY_MAX_BATCHES (40), MIRROR_NAMESPACE, +# REGISTRY_USER, REGISTRY_TOKEN. # -# Exit status: 0 every target has a state, 1 the step itself could not run. +# Exit status: 0 the target has a state, 1 the step itself could not run. set -euo pipefail SCRIPT_NAME=stage # shellcheck source=lib.sh @@ -38,9 +53,14 @@ need curl jq awk sha256sum timeout plan="${1:-}" results="${2:-}" out="${3:-}" -if [ ! -d "$plan" ] || [ -z "$results" ] || [ -z "$out" ]; then - die "usage: scripts/stage PLAN RESULTS OUT" +target="${4:-}" +if [ ! -d "$plan" ] || [ -z "$results" ] || [ -z "$out" ] || [ -z "$target" ]; then + die "usage: scripts/stage PLAN RESULTS OUT TARGET" fi +case "$target" in + *[!a-z0-9_.-]*) die "TARGET is a target name, not '$target'" ;; +esac +[ -f "$plan/$target/state" ] || die "the plan has no state for $target" "reproduce: ls $plan/$target" [ -n "$MIRROR_NAMESPACE" ] || die "MIRROR_NAMESPACE is not set" if [ "$MIRROR_REGISTRY_AUTH" != none ] && [ -z "${REGISTRY_TOKEN:-}" ]; then die "REGISTRY_TOKEN is not set; staging pushes" @@ -50,21 +70,30 @@ STAGING_TAG="${STAGING_TAG:-staging}" die "STAGING_TAG must be staging or staging-, not '$STAGING_TAG'" BATCH_FILES="${VERIFY_BATCH_FILES:-400}" BATCH_BYTES="${VERIFY_BATCH_BYTES:-2147483648}" +MAX_BATCHES="${VERIFY_MAX_BATCHES:-40}" +for v in BATCH_FILES BATCH_BYTES MAX_BATCHES; do + case "${!v}" in + '' | *[!0-9]* | 0) die "$v must be a positive integer, not '${!v}'" ;; + esac +done mkdir -p "$out" work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT mkdir -p "$work/bin" +REGISTRY_TOKEN_CACHE="$work/tokens" +mkdir -m 0700 "$REGISTRY_TOKEN_CACHE" ORAS="$("$REPO_ROOT/scripts/install-oras" "$work/bin")" ORAS_CONFIG="$work/oras-auth.json" oras_login "$ORAS" "$ORAS_CONFIG" # Every result line from every shard: target filename status seconds detail. No # results at all is ordinary: a database that changed by dropping packages -# names no new file. +# names no new file. A results.tsv and the lines/ beside it say the same. : > "$work/results.tsv" if [ -d "$results" ]; then - find "$results" -name results.tsv -type f -exec cat {} + > "$work/results.tsv" + find "$results" -name results.tsv -type f -exec cat {} + >> "$work/results.tsv" + find "$results" -path '*/lines/*' -name '[0-9]*.tsv' -type f -exec cat {} + >> "$work/results.tsv" fi set_state() { @@ -102,12 +131,12 @@ stage_target() { n_bad="$(awk -F '\t' '$2 != "pushed" && $2 != "present" { n++ } END { print n + 0 }' "$dir/accounted.tsv")" if [ "$n_replaced" -gt 0 ]; then awk -F '\t' '$2 == "replaced"' "$dir/accounted.tsv" > "$dir/replaced.tsv" - set_state "$dir" replaced "$n_replaced file(s) have a tag holding other bytes under the same name, first: $(awk -F '\t' '$2 == "replaced" { print $1 ": " $3; exit }' "$dir/accounted.tsv")" + set_state "$dir" replaced "$n_replaced file(s) have a tag holding other bytes under the same name, first: $(awk -F '\t' '$2 == "replaced" && !f { print $1 ": " $3; f = 1 }' "$dir/accounted.tsv")" return 0 fi if [ "$n_bad" -gt 0 ]; then awk -F '\t' '$2 != "pushed" && $2 != "present"' "$dir/accounted.tsv" > "$dir/failing.tsv" - set_state "$dir" held "$n_bad file(s) are not on the registry, first: $(awk -F '\t' '$2 != "pushed" && $2 != "present" { print $1 ": " $2 ", " $3; exit }' "$dir/accounted.tsv")" + set_state "$dir" held "$n_bad file(s) are not on the registry, first: $(awk -F '\t' '$2 != "pushed" && $2 != "present" && !f { print $1 ": " $2 ", " $3; f = 1 }' "$dir/accounted.tsv")" return 0 fi @@ -211,69 +240,64 @@ stage_target() { fi #-------------------------------------------------------------------------# - # Verification batches: every file new since the promoted database, plus a - # canary that is always present (the keyring, and the smallest package), so - # a run with nothing new still proves the path a consumer takes. + # Verification batches. The canaries go to batch 0; the rest, largest first + # onto the lightest batch. #-------------------------------------------------------------------------# - local keyring - keyring="$(target_get "$conf" keyring)" + cp "$pdir/records.tsv" "$dir/records.tsv" + target_keyrings "$conf" > "$work/keyrings.txt" + { + awk -F '\t' 'FILENAME == ARGV[1] { want[$1] = 1; next } ($2 in want) { print $2 }' "$work/keyrings.txt" "$pdir/records.tsv" + sort -t $'\t' -k6,6n -k2,2 "$pdir/records.tsv" | awk -F '\t' 'NR == 1 { print $2 }' + } | awk '!seen[$0]++' > "$dir/canaries.txt" { + awk -F '\t' 'FILENAME == ARGV[1] { c[$1] = 1; next } ($2 in c) { print $2 "\t" $6 }' "$dir/canaries.txt" "$pdir/records.tsv" awk -F '\t' '{ print $6 "\t" $5 }' "$pdir/todo.tsv" - if [ -n "$keyring" ]; then - awk -F '\t' -v k="$keyring" '$2 == k { print $2 "\t" $6 }' "$pdir/records.tsv" - fi - sort -t $'\t' -k6,6n "$pdir/records.tsv" | awk -F '\t' 'NR == 1 { print $2 "\t" $6 }' } | awk -F '\t' '!seen[$1]++' > "$dir/verify.tsv" - local nfiles nbytes k + local nfiles nbytes k flimit blimit wanted nfiles="$(awk 'END { print NR + 0 }' "$dir/verify.tsv")" nbytes="$(awk -F '\t' '{ s += $2 } END { printf "%.0f\n", s }' "$dir/verify.tsv")" - k="$(awk -v b="$nbytes" -v f="$nfiles" -v bb="$BATCH_BYTES" -v bf="$BATCH_FILES" 'BEGIN { - kb = int((b + bb - 1) / bb); kf = int((f + bf - 1) / bf) - k = (kb > kf) ? kb : kf - if (k < 1) k = 1 - print k - }')" - sort -t $'\t' -k2,2nr -k1,1 "$dir/verify.tsv" | awk -F '\t' -v k="$k" -v dir="$dir/batches" ' - { - best = 0 - for (i = 1; i < k; i++) if (load[i] < load[best]) best = i - load[best] += $2 - print $1 > (dir "/" best ".txt") - } - ' + read -r k flimit blimit wanted <<< "$(batch_count "$nfiles" "$nbytes" "$BATCH_FILES" "$BATCH_BYTES" "$MAX_BATCHES")" + if [ "$wanted" -gt "$k" ]; then + log "$name: $nfiles file(s) and $((nbytes / 1048576)) MiB want $wanted batch(es) at $BATCH_FILES files and $((BATCH_BYTES / 1048576)) MiB; VERIFY_MAX_BATCHES=$MAX_BATCHES grows the limits to $flimit files and $((blimit / 1048576)) MiB: $k batch(es)" + fi + local canary_bytes + canary_bytes="$(awk -F '\t' 'FILENAME == ARGV[1] { c[$1] = 1; next } ($1 in c) { s += $2 } END { printf "%.0f\n", s }' "$dir/canaries.txt" "$dir/verify.tsv")" + cp "$dir/canaries.txt" "$dir/batches/0.txt" + awk -F '\t' 'FILENAME == ARGV[1] { c[$1] = 1; next } !($1 in c)' "$dir/canaries.txt" "$dir/verify.tsv" | + sort -t $'\t' -k2,2nr -k1,1 | awk -F '\t' -v k="$k" -v d="$dir/batches" -v c="$canary_bytes" ' + BEGIN { load[0] = c + 0 } + { + best = 0 + for (i = 1; i < k; i++) if (load[i] < load[best]) best = i + load[best] += $2 + print $1 >> (d "/" best ".txt") + } + ' - set_state "$dir" staged "$repo_path@$digest, $(awk 'END { print NR + 0 }' "$dir/verify.tsv") package(s) to verify in $(find "$dir/batches" -name '*.txt' | awk 'END { print NR + 0 }') batch(es)" + set_state "$dir" staged "$repo_path@$digest, $nfiles package(s) to verify in $(find "$dir/batches" -name '*.txt' | awk 'END { print NR + 0 }') batch(es), canaries: $(tr '\n' ' ' < "$dir/canaries.txt" | sed 's/ $//')" } -for pdir in "$plan"/*/; do - pdir="${pdir%/}" - [ -f "$pdir/state" ] || continue - stage_target "$pdir" -done +stage_target "$plan/$target" #---------------------------------------------------------------------------# # The verify matrix #---------------------------------------------------------------------------# { - for dir in "$out"/*/; do - dir="${dir%/}" - [ -f "$dir/state" ] || continue - [ "$(cat "$dir/state")" = staged ] || continue + dir="$out/$target" + if [ "$(cat "$dir/state")" = staged ]; then for b in "$dir"/batches/*.txt; do [ -e "$b" ] || continue - jq -cn --arg t "$(basename "$dir")" --arg b "$(basename "$b" .txt)" '{target: $t, batch: $b}' + basename "$b" .txt + done | sort -n | while IFS= read -r n; do + jq -cn --arg t "$target" --arg b "$n" '{target: $t, batch: $b}' done - done + fi } | jq -cs '{include: .}' > "$out/verify-matrix.json" { printf 'target\tstate\twhy\n' - for dir in "$out"/*/; do - dir="${dir%/}" - [ -f "$dir/state" ] || continue - printf '%s\t%s\t%s\n' "$(basename "$dir")" "$(cat "$dir/state")" "$(cat "$dir/why")" - done + printf '%s\t%s\t%s\n' "$target" "$(cat "$out/$target/state")" "$(cat "$out/$target/why")" } > "$out/summary.tsv" if [ -n "${GITHUB_OUTPUT:-}" ]; then diff --git a/scripts/verify-batch b/scripts/verify-batch index e127719..360ba83 100755 --- a/scripts/verify-batch +++ b/scripts/verify-batch @@ -1,43 +1,80 @@ #!/usr/bin/env bash # # Verify one batch of a staged database the way a consumer reads it: pacman, -# in the target's own image, with the wrapper as its XferCommand and the staged -# manifest, pinned by digest, as its only Server. +# with the wrapper as its XferCommand and the staged manifest, pinned by +# digest, as its Server. # # scripts/verify-batch STAGE TARGET BATCH OUT # -# STAGE is scripts/stage's output. Writes OUT/verify-TARGET-BATCH.tsv: -# target batch result detail digest -# where digest is the staged manifest this verdict is about, so a verdict left -# over from another attempt is never taken for this one. -# with result passed or failed, and the full pacman log beside it. +# STAGE is scripts/stage's output. Writes: +# OUT/verify-TARGET-BATCH.tsv target batch result detail digest +# OUT/verify-TARGET-BATCH.log everything, with a UTC time on each phase +# OUT/refused-TARGET-BATCH.tsv file reason, one line per upstream-refused +# package (below); written only when there is one +# result is passed or failed. digest is the staged manifest the verdict is +# about, so a verdict left over from another attempt is never taken for this +# one. The verdict is written even when the step is killed (EXIT trap), with +# the packages named at fault so far. # # ⭐ pacman checks what it always checks: every file's sha256 against the # database and every signature against the distribution's keyring, with its # own trust model, before it reports a download complete. A failure here is a # failure a consumer would have had. # -# Then one check pacman does not make: with a %PGPSIG% in the database, pacman -# verifies against it and stores the .sig the wrapper served without reading -# it (measured with pacman 7.1, 2026-10-14). Each stored .sig must be that -# %PGPSIG%, decoded, byte for byte, or a later pacman -U from the cache fails. +# The container is the target's verify_image: x86_64 Arch for every target. +# pacman.conf says Architecture = the target's arch, and pacman on x86_64 then +# downloads and checks another arch's packages (measured 2026-10-14), so no +# target needs emulation. The configuration is written here, not the image's: +# the target's repository with its staged manifest as the only Server, +# DownloadUser = alpm (the wrapper runs as that user from a download-XXXXXX +# directory, the stock Arch path), and DatabaseRequired when the upstream +# signs its database. # -# pacman reads a configuration written here, not the image's: one repository, -# its staged manifest as the only Server, and DownloadUser = alpm, so the -# wrapper runs as that user from a download-XXXXXX directory, the stock Arch -# path. A target whose upstream signs its database is checked with -# DatabaseRequired. The first batch of a target also reads the files database. +# Phases, each logged with its time: +# pull the verify image, and keyring_image when the target names one: +# two attempts of 300 s each +# keyring pacman-key --init; the files of keyring_image's +# /usr/share/pacman/keyrings added, never over a file the verify +# image has; --populate. Then every keyring package installed +# with -Sdd, from the staged repository or from keyring_from's +# latest, pinned by the digest latest names now, as a second +# section; --populate again, so a key added upstream is signed +# locally before any package is checked. keyring_from with no +# latest yet: a warning, and no refresh; the trust root still +# applies and pacman still checks every signature. +# canaries batch 0 only: stage's canaries through the wrapper's single-file +# path (PACMAN_OCI_PREFETCH=0), into a cache of their own +# batch every package of the batch through the wrapper's default path +# with PACMAN_OCI_SIG=registry, so the signatures pacman stores are +# the registry's layers +# refusals after a download that failed, each package alone +# sig check with a %PGPSIG% in the database, pacman verifies against it and +# stores the .sig the wrapper served without reading it (measured +# with pacman 7.1, 2026-10-14); a later pacman -U from the cache +# reads it. Each stored .sig must be that %PGPSIG%, decoded. +# files db pacman -Fy: the synced .files must be the staged one +# (FILES_SHA256), and pacman -Fl must read every name of the batch # -# The keyring is refreshed first, from the staged snapshot itself: pacman-key -# --init gives the container a local key, the keyring package is installed with -# the image's trust, and --populate is run again and must succeed, so a master -# key added upstream is signed locally before any package is checked. +# Upstream-refused: a package pacman refuses alone for a reason in its +# signature (lib.sh refusal_reason) is not the mirror's fault when the mirror +# serves exactly upstream's file: its tag holds the database's sha256 and +# size, and its signature layer is the decoded %PGPSIG%, or, with no %PGPSIG%, +# the .sig upstream serves now; or upstream has no signature at all, the tag +# has none, and every server answers 404 for the .sig. pacman refuses it from +# upstream too (loong64 extra, 2026-10-14: three packages signed by keys that +# expired, one signed nowhere). It is listed in OUT/refused-TARGET-BATCH.tsv, +# and the batch passes when every failure is one of these. Any other failure, +# and any stored .sig that is not the %PGPSIG%, fails it. +# +# Budget: VERIFY_BUDGET seconds (default 4800) from the start, for the pulls +# and the container together; the container gets what the pulls leave. The +# workflow step's timeout is above it, so this script, not the runner, ends +# a run that is too long, and says so. # # Environment: MIRROR_NAMESPACE, CONTAINER_RUNTIME (docker or podman; docker -# by default). -# For the integration suite only: VERIFY_NETWORK (a container network, host -# to reach a registry on the loopback) and VERIFY_TRUST_KEY (an armored -# public key the container trusts, for packages signed by a test key). +# by default), VERIFY_BUDGET. +# For the integration suite only: VERIFY_NETWORK, a container network (host, +# to reach a registry on the loopback). # # Exit status: 0 passed, 1 failed or could not run. set -euo pipefail @@ -45,6 +82,7 @@ SCRIPT_NAME=verify-batch # shellcheck source=lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +started="$(date -u +%s)" stage="${1:-}" target="${2:-}" batch="${3:-}" @@ -57,170 +95,477 @@ case "$target$batch" in esac [ -n "$MIRROR_NAMESPACE" ] || die "MIRROR_NAMESPACE is not set" RUNTIME="${CONTAINER_RUNTIME:-docker}" -need "$RUNTIME" timeout +need "$RUNTIME" timeout jq sha256sum base64 +BUDGET="${VERIFY_BUDGET:-4800}" +case "$BUDGET" in + '' | *[!0-9]* | 0) die "VERIFY_BUDGET must be a positive number of seconds, not '$BUDGET'" ;; +esac mkdir -p "$out" report="$out/verify-$target-$batch.tsv" log_file="$out/verify-$target-$batch.log" +refused_file="$out/refused-$target-$batch.tsv" +rm -f "$report" "$refused_file" +: > "$log_file" verdict() { printf '%s\t%s\t%s\t%s\t%s\n' "$target" "$batch" "$1" "$2" "${DIGEST:-unknown}" > "$report" log "$target batch $batch: $1: $2" } -trap '[ -f "$report" ] || verdict failed "ended before reaching a verdict"' EXIT + +# culprits_so_far -> the packages the log names at fault, one line +culprits_so_far() { + awk -F '\t' ' + $1 == "@@refused" && !seen[$2]++ { s = s " " $2 } + $1 == "@@sigbad" && !seen[$2]++ { s = s " " $2 ": " $3 ";" } + END { print substr(s, 2) } + ' "$log_file" +} + +# phase NAME: one timed line in the log and on stderr +phase() { + printf '== %s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >> "$log_file" + log "$*" +} + +work="" +# shellcheck disable=SC2329 # run by the EXIT trap +on_exit() { + local rc=$? c + if [ -n "$work" ]; then + rm -rf "$work" + fi + if [ ! -f "$report" ]; then + c="$(culprits_so_far)" + verdict failed "ended with status $rc before reaching a verdict${c:+; at fault so far: $c}" + fi +} +trap on_exit EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM conf="$TARGETS_DIR/$target.conf" target_check "$conf" +arch="$(target_get "$conf" arch)" repo="$(target_get "$conf" repo)" +repo_path="$(target_repository "$conf")" image="$(target_get "$conf" verify_image)" platform="$(target_get "$conf" verify_platform)" -keyring="$(target_get "$conf" keyring)" +keyring_image="$(target_get "$conf" keyring_image)" +keyring_platform="$(target_get "$conf" keyring_platform)" +keyring_from="$(target_get "$conf" keyring_from)" +keyrings="$(target_keyrings "$conf" | tr '\n' ' ' | sed 's/ $//')" db_siglevel=DatabaseOptional if [ "$(target_get "$conf" db_signed)" = yes ]; then db_siglevel=DatabaseRequired fi -files_db=0 -if [ "$batch" = 0 ]; then - files_db=1 -fi [ -n "$image" ] || die "$conf names no verify_image" -names="$stage/$target/batches/$batch.txt" -[ -s "$names" ] || die "no batch $names" +batch_file="$stage/$target/batches/$batch.txt" +[ -s "$batch_file" ] || die "no batch $batch_file" # shellcheck source=/dev/null # written by scripts/stage . "$stage/$target/staged.env" -: "${DIGEST:?}" +: "${DIGEST:?}" "${FILES_SHA256:?}" work="$(mktemp -d)" -trap 'rm -rf "$work"; [ -f "$report" ] || verdict failed "ended before reaching a verdict"' EXIT +mkdir -p "$work/keyrings" cp "$WRAPPER" "$work/pacman-oci" -cp "$names" "$work/names.txt" -: > "$work/trust.asc" -if [ -n "${VERIFY_TRUST_KEY:-}" ]; then - cp "$VERIFY_TRUST_KEY" "$work/trust.asc" +cp "$batch_file" "$work/names.txt" +: > "$work/canaries.txt" +if [ "$batch" = 0 ] && [ -f "$stage/$target/canaries.txt" ]; then + cp "$stage/$target/canaries.txt" "$work/canaries.txt" fi net_args=() if [ -n "${VERIFY_NETWORK:-}" ]; then net_args=(--network "$VERIFY_NETWORK") fi -server="$OCI_SCHEME://$MIRROR_REGISTRY/$(target_repository "$conf")@$DIGEST" +server="$OCI_SCHEME://$MIRROR_REGISTRY/$repo_path@$DIGEST" + +#---------------------------------------------------------------------------# +# keyring_from: the second section, pinned by the digest its latest names now +#---------------------------------------------------------------------------# +second_repo="" +second_server="" +keyring_repo="" +if [ -n "$keyrings" ]; then + keyring_repo="$repo" +fi +if [ -n "$keyring_from" ]; then + from_conf="$TARGETS_DIR/$keyring_from.conf" + target_check "$from_conf" + from_path="$(target_repository "$from_conf")" + rc=0 + REGISTRY_TOKEN='' registry_manifest "$from_path" latest "$work/from-latest.json" || rc=$? + case "$rc" in + "$REG_STATUS_OK") + second_repo="$(target_get "$from_conf" repo)" + second_server="$OCI_SCHEME://$MIRROR_REGISTRY/$from_path@sha256:$(sha256sum "$work/from-latest.json" | cut -d ' ' -f 1)" + keyring_repo="$second_repo" + ;; + "$REG_STATUS_ABSENT") + keyring_repo="" + warn "$target: keyring_from $keyring_from has no latest yet, so the keyring packages ($keyrings) are not refreshed this run" \ + "the trust root still applies, and pacman still checks every signature against it" \ + "reproduce: curl -s -H 'Authorization: Bearer QQ==' -H 'Accept: application/vnd.oci.image.manifest.v1+json' $REG_URL/v2/$from_path/manifests/latest" + printf 'keyring: keyring_from %s has no latest yet; the refresh is skipped\n' "$keyring_from" >> "$log_file" + ;; + *) + verdict failed "could not read $from_path:latest for keyring_from; no answer is not an absence" + exit 1 + ;; + esac +fi +#---------------------------------------------------------------------------# +# Pulls: two attempts of 300 s each +#---------------------------------------------------------------------------# +# pull IMAGE PLATFORM -> 0 pulled +pull() { + local attempt rc=0 + for attempt in 1 2; do + rc=0 + timeout --kill-after=10 300 "$RUNTIME" pull --platform "$2" "$1" >> "$work/pull.log" 2>&1 || rc=$? + [ "$rc" -ne 0 ] || return 0 + log "pull of $1 for $2, attempt $attempt, ended $rc" + if [ "$attempt" -eq 1 ]; then + sleep 10 + fi + done + return 1 +} + +phase "pull $image ($platform)" +if ! pull "$image" "$platform"; then + cat "$work/pull.log" >> "$log_file" + verdict failed "could not pull $image in two attempts of 300 s: $(tail -n 1 "$work/pull.log")" + exit 1 +fi + +if [ -n "$keyring_image" ]; then + phase "pull $keyring_image ($keyring_platform), for its keyring files" + if ! pull "$keyring_image" "$keyring_platform"; then + cat "$work/pull.log" >> "$log_file" + verdict failed "could not pull $keyring_image in two attempts of 300 s: $(tail -n 1 "$work/pull.log")" + exit 1 + fi + # create and cp run nothing from the image, so no emulation is needed. + cid="$(timeout 120 "$RUNTIME" create --platform "$keyring_platform" "$keyring_image" true)" || + die "could not create a container from $keyring_image" "reproduce: $RUNTIME create --platform $keyring_platform $keyring_image true" + rc=0 + timeout 120 "$RUNTIME" cp "$cid:/usr/share/pacman/keyrings/." "$work/keyrings/" >> "$log_file" 2>&1 || rc=$? + timeout 120 "$RUNTIME" rm "$cid" >> "$log_file" 2>&1 + [ "$rc" -eq 0 ] || die "could not copy /usr/share/pacman/keyrings out of $keyring_image (status $rc)" \ + "reproduce: $RUNTIME create --platform $keyring_platform $keyring_image true, then $RUNTIME cp ID:/usr/share/pacman/keyrings/. DIR" + for f in "$work/keyrings"/* "$work/keyrings"/.[!.]*; do + [ -e "$f" ] || [ -L "$f" ] || continue + b="${f##*/}" + if [ -L "$f" ] || [ ! -f "$f" ] || [[ ! "$b" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + die "$keyring_image holds /usr/share/pacman/keyrings/$b, which is not a plain keyring file" + fi + done + printf 'keyring: from %s: %s\n' "$keyring_image" "$(find "$work/keyrings" -type f -printf '%f ' | sed 's/ $//')" >> "$log_file" +fi + +#---------------------------------------------------------------------------# +# The container +#---------------------------------------------------------------------------# cat > "$work/verify.sh" << 'EOS' set -euo pipefail -repo="$1" server="$2" keyring="$3" files_db="$4" db_siglevel="$5" +repo="$1" arch="$2" server="$3" second_repo="$4" second_server="$5" keyrings="$6" keyring_repo="$7" db_siglevel="$8" files_sha="$9" -install -m 0755 /in/pacman-oci /usr/local/bin/pacman-oci +phase() { printf '== %s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; } -cat > /tmp/pacman.conf << EOF -[options] -Architecture = auto -CheckSpace -SigLevel = Required $db_siglevel -LocalFileSigLevel = Optional -DownloadUser = alpm -XferCommand = /usr/local/bin/pacman-oci %o %u - -[$repo] -Server = $server -EOF +install -m 0755 /in/pacman-oci /usr/local/bin/pacman-oci +{ + printf '[options]\nArchitecture = %s\nCheckSpace\nSigLevel = Required %s\n' "$arch" "$db_siglevel" + printf 'LocalFileSigLevel = Optional\nDownloadUser = alpm\nXferCommand = /usr/local/bin/pacman-oci %%o %%u\n' + printf '\n[%s]\nServer = %s\n' "$repo" "$server" + if [ -n "$second_repo" ]; then + printf '\n[%s]\nServer = %s\n' "$second_repo" "$second_server" + fi +} > /tmp/pacman.conf echo "== pacman.conf" grep -vE '^[[:space:]]*(#|$)' /tmp/pacman.conf pac() { pacman --config /tmp/pacman.conf --noconfirm "$@"; } -echo "== keyring" +phase "keyring" pacman-key --init +copied=() +for f in /in/keyrings/*; do + [ -f "$f" ] || continue + b="${f##*/}" + if [ -e "/usr/share/pacman/keyrings/$b" ]; then + echo "keyring: keeps the verify image's own $b" + else + install -m 0644 "$f" "/usr/share/pacman/keyrings/$b" + copied+=("$b") + echo "keyring: $b from keyring_image" + fi +done pacman-key --populate -if [ -s /in/trust.asc ]; then - pacman-key --add /in/trust.asc - pacman-key --lsign-key "$(gpg --with-colons --show-keys /in/trust.asc | awk -F : '$1 == "fpr" && f == "" { f = $10 } END { print f }')" -fi pac -Sy -if [ -n "$keyring" ]; then - pac -Sdd "$keyring" +if [ -n "$keyrings" ] && [ -n "$keyring_repo" ]; then + # The keyring packages own the files keyring_image supplied; their keys are + # in the keyring now, so the copies give way to the packages. + for b in "${copied[@]}"; do + rm -f "/usr/share/pacman/keyrings/$b" + done + read -r -a kr <<< "$keyrings" + phase "keyring: install ${kr[*]} from [$keyring_repo]" + pac -Sdd "${kr[@]/#/$keyring_repo/}" pacman-key --populate + # pacman checked the keyring packages' own signatures: the trust root works. + printf '@@trust\tkeyring\n' fi -echo "== download and check $(wc -l < /in/names.txt) package(s)" -rc=0 -xargs -a /in/names.txt pacman --config /tmp/pacman.conf --noconfirm -Swdd || rc=$? -if [ "$rc" -ne 0 ]; then - echo "== the batch failed ($rc); each package alone, to name the ones at fault" +failed=0 +# fetch LIST CACHE [NAME=VALUE ...]: pacman -Swdd of every name in LIST, and +# after a failure each one alone, to name the ones at fault. For each one at +# fault, the wrapper is also asked for the package and for its .sig alone, as +# root into /tmp: its exit statuses say whether the registry serves them. +fetch() { + local list="$1" cache="$2" rc=0 p one f pkg_rc sig_rc + shift 2 + mkdir -p "$cache" + sed "s|^|$repo/|" "$list" > /tmp/qualified.txt + env "$@" xargs -a /tmp/qualified.txt pacman --config /tmp/pacman.conf --noconfirm --cachedir "$cache" -Swdd || rc=$? + [ "$rc" -ne 0 ] || return 0 + phase "refusals: the download failed ($rc); each package alone, to name the ones at fault" while IFS= read -r p; do one=0 - pac -Swdd "$p" > "/tmp/one.log" 2>&1 || one=$? + env "$@" pacman --config /tmp/pacman.conf --noconfirm --cachedir "$cache" -Swdd "$repo/$p" > /tmp/one.log 2>&1 || one=$? if [ "$one" -ne 0 ]; then + failed=1 echo "CULPRIT $p" - sed 's/^/ /' /tmp/one.log | tail -n 6 + printf '@@refused\t%s\n' "$p" + awk -v p="$p" 'NR <= 80 { printf "@@log\t%s\t%s\n", p, $0 }' /tmp/one.log + pkg_rc=x + sig_rc=x + f="" + pacman --config /tmp/pacman.conf -Sddp --print-format '%f' "$repo/$p" > /tmp/probe.name || echo " probe: pacman -Sp did not name the file of $p" + f="$(sed -n '$p' /tmp/probe.name)" + if [ -n "$f" ]; then + rm -rf /tmp/probe + mkdir -p /tmp/probe + pkg_rc=0 + /usr/local/bin/pacman-oci "/tmp/probe/$f" "$server/$f" > /tmp/probe.log 2>&1 || pkg_rc=$? + sig_rc=0 + /usr/local/bin/pacman-oci "/tmp/probe/$f.sig" "$server/$f.sig" >> /tmp/probe.log 2>&1 || sig_rc=$? + sed "s|^| probe: |" /tmp/probe.log + fi + printf '@@probe\t%s\t%s\t%s\n' "$p" "$pkg_rc" "$sig_rc" fi - done < /in/names.txt - exit 1 + done < "$list" +} + +if [ -s /in/canaries.txt ]; then + phase "canaries: $(wc -l < /in/canaries.txt) package(s) through the wrapper's single-file path" + fetch /in/canaries.txt /tmp/canary-cache PACMAN_OCI_PREFETCH=0 fi +phase "batch: $(wc -l < /in/names.txt) package(s) through the wrapper's default path, signatures from the registry" +fetch /in/names.txt /var/cache/pacman/pkg PACMAN_OCI_SIG=registry -# pacman checks a package against the database's %PGPSIG% when it carries one, -# and keeps the .sig the wrapper served beside it in the cache unread; a later -# pacman -U from that cache reads it. So each must be that %PGPSIG%, decoded. -echo "== signature layers against the database's %PGPSIG%" -bsdtar -xOf "/var/lib/pacman/sync/$repo.db" | awk ' +# NUL bytes are dropped and a record boundary restored before %FILENAME%: a +# desc that is NUL bytes only (aarch64 extra findnewest-0.3-4) would otherwise +# join the next record's first line. +phase "sig check: each stored .sig against the database's %PGPSIG%, decoded" +bsdtar -xOf "/var/lib/pacman/sync/$repo.db" | tr -d '\000' | sed 's/%FILENAME%/\n&/g' | awk ' k == "F" { f = $0 } k == "P" { print f "\t" $0 } { k = ($0 == "%FILENAME%") ? "F" : ($0 == "%PGPSIG%") ? "P" : "" } ' > /tmp/pgpsig.tsv -cache=/var/cache/pacman/pkg checked=0 -bad=0 -while IFS=$'\t' read -r f sig; do - [ -e "$cache/$f" ] || continue - checked=$((checked + 1)) - if [ ! -f "$cache/$f.sig" ]; then - echo "CULPRIT $f: the database carries a %PGPSIG%, and the wrapper served no .sig" - bad=1 - continue - fi - want="$(printf '%s' "$sig" | base64 -d | sha256sum | cut -d ' ' -f 1)" - got="$(sha256sum < "$cache/$f.sig" | cut -d ' ' -f 1)" - if [ "$want" != "$got" ]; then - echo "CULPRIT $f: its .sig is sha256 $got, its %PGPSIG% decodes to sha256 $want" - bad=1 - fi -done < /tmp/pgpsig.tsv +for cache in /var/cache/pacman/pkg /tmp/canary-cache; do + [ -d "$cache" ] || continue + while IFS=$'\t' read -r f sig; do + [ -e "$cache/$f" ] || continue + checked=$((checked + 1)) + if [ ! -f "$cache/$f.sig" ]; then + echo "CULPRIT $f: the database carries a %PGPSIG%, and the wrapper served no .sig" + printf '@@sigbad\t%s\t%s\n' "$f" "the database carries a %PGPSIG%, and the wrapper served no .sig" + failed=1 + continue + fi + want="$(printf '%s' "$sig" | base64 -d | sha256sum | cut -d ' ' -f 1)" + got="$(sha256sum < "$cache/$f.sig" | cut -d ' ' -f 1)" + if [ "$want" != "$got" ]; then + echo "CULPRIT $f: its .sig is sha256 $got, its %PGPSIG% decodes to sha256 $want" + printf '@@sigbad\t%s\t%s\n' "$f" "its .sig is sha256 $got, its %PGPSIG% decodes to sha256 $want" + failed=1 + fi + done < /tmp/pgpsig.tsv +done echo "$checked signature(s) compared" -if [ "$bad" -ne 0 ]; then + +phase "files db: pacman -Fy, and -Fl over every name of the batch" +pac -Fy +got="$(sha256sum "/var/lib/pacman/sync/$repo.files" | cut -d ' ' -f 1)" +if [ "$got" != "$files_sha" ]; then + echo "the synced $repo.files is sha256 $got, the staged one $files_sha" exit 1 fi +echo "the synced $repo.files is the staged one, sha256 $got" +xargs -a /in/names.txt pacman --config /tmp/pacman.conf -Fl > /tmp/files-list.txt +echo "pacman -Fl listed $(wc -l < /tmp/files-list.txt) path(s) for $(wc -l < /in/names.txt) package(s)" -if [ "$files_db" = 1 ]; then - echo "== files database" - pac -Fy - pacman --config /tmp/pacman.conf -Fl "$(sed -n 1p /in/names.txt)" | sed -n 1,3p -fi -echo "== passed" +phase "done" +echo "@@done" +exit "$failed" EOS -pull_rc=0 -for attempt in 1 2 3; do - pull_rc=0 - timeout --kill-after=30 600 "$RUNTIME" pull --platform "$platform" "$image" > "$work/pull.log" 2>&1 || pull_rc=$? - [ "$pull_rc" -ne 0 ] || break - sleep $((attempt * 10)) -done -if [ "$pull_rc" -ne 0 ]; then - cat "$work/pull.log" > "$log_file" - verdict failed "could not pull $image: $(tail -n 1 "$work/pull.log")" +elapsed=$(($(date -u +%s) - started)) +remaining=$((BUDGET - elapsed)) +if [ "$remaining" -lt 60 ]; then + verdict failed "the own budget ran out before the container started: VERIFY_BUDGET=${BUDGET}s, ${elapsed}s spent on the pulls" exit 1 fi - +phase "container: $remaining s of the ${BUDGET}s budget left" rc=0 -timeout --kill-after=30 5400 "$RUNTIME" run --rm --platform "$platform" "${net_args[@]}" \ - -v "$work/trust.asc:/in/trust.asc:ro" \ +timeout --kill-after=30 "$remaining" "$RUNTIME" run --rm --platform "$platform" "${net_args[@]}" \ -v "$work/pacman-oci:/in/pacman-oci:ro" \ -v "$work/names.txt:/in/names.txt:ro" \ + -v "$work/canaries.txt:/in/canaries.txt:ro" \ + -v "$work/keyrings:/in/keyrings:ro" \ -v "$work/verify.sh:/in/verify.sh:ro" \ - "$image" bash /in/verify.sh "$repo" "$server" "$keyring" "$files_db" "$db_siglevel" > "$log_file" 2>&1 || rc=$? + "$image" bash /in/verify.sh "$repo" "$arch" "$server" "$second_repo" "$second_server" "$keyrings" "$keyring_repo" \ + "$db_siglevel" "$FILES_SHA256" >> "$log_file" 2>&1 || rc=$? +phase "container ended with status $rc" +count="$(awk 'END { print NR + 0 }' "$batch_file")" if [ "$rc" -eq 0 ]; then - verdict passed "$(awk 'END { print NR + 0 }' "$names") package(s) read through the wrapper and checked by pacman" + verdict passed "$count package(s) read through the wrapper and checked by pacman" exit 0 fi -culprits="$(sed -n 's/^CULPRIT //p' "$log_file" | tr '\n' ' ')" tail -n 40 "$log_file" >&2 -verdict failed "the batch failed its checks (status $rc)${culprits:+; at fault: $culprits}" +if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + c="$(culprits_so_far)" + verdict failed "the own budget ran out: VERIFY_BUDGET=${BUDGET}s, the container was stopped after ${remaining}s (status $rc)${c:+; at fault so far: $c}" + exit 1 +fi +if ! grep -q '^@@done$' "$log_file"; then + c="$(culprits_so_far)" + verdict failed "the container ended with status $rc before its last phase${c:+; at fault so far: $c}" + exit 1 +fi + +#---------------------------------------------------------------------------# +# Every package at fault: upstream-refused, or the batch fails +#---------------------------------------------------------------------------# +sigbad="$(awk -F '\t' '$1 == "@@sigbad" { n++ } END { print n + 0 }' "$log_file")" +awk -F '\t' '$1 == "@@refused" && !seen[$2]++ { print $2 }' "$log_file" > "$work/refused-names.txt" +sources=() +mapfile -t sources < <(target_sources "$conf") + +# upstream_refused NAME CLASS -> 0 when the mirror serves exactly upstream's +# file and signature for NAME; sets refused_why either way +refused_why="" +upstream_refused() { + local name="$1" class="$2" file sha csize pgpsig sig_sha="" tag rc decision layer s got + IFS=$'\t' read -r file _ _ _ sha csize _ pgpsig <<<"$(awk -F '\t' -v n="$name" '$2 == n' "$stage/$target/records.tsv")" + if [ -z "${file:-}" ]; then + refused_why="the staged records name no package $name" + return 1 + fi + if [ -n "$pgpsig" ]; then + printf '%s' "$pgpsig" | base64 -d > "$work/refused.sig" + sig_sha="$(sha256sum "$work/refused.sig" | cut -d ' ' -f 1)" + fi + tag="$("$WRAPPER" --tag "$file")" + rc=0 + REGISTRY_TOKEN='' registry_manifest "$repo_path" "$tag" "$work/refused.json" || rc=$? + if [ "$rc" -ne "$REG_STATUS_OK" ]; then + refused_why="its tag $repo_path:$tag does not read (status $rc)" + return 1 + fi + decision="$(decide_push "$work/refused.json" "$file" "$sha" "$csize" "$sig_sha")" + case "$class" in + sig-refused) + if [ "$decision" != present ]; then + refused_why="its tag does not hold the database's bytes and signature ($decision)" + return 1 + fi + [ -z "$sig_sha" ] || return 0 + # No %PGPSIG%: the layer must be the .sig upstream serves now. + read -r layer _ _ <<< "$(manifest_layer "$work/refused.json" "$file.sig")" + for s in "${sources[@]}"; do + rc=0 + fetch_upstream "$s/$file.sig" "$work/upstream.sig" 60 || rc=$? + case "$rc" in + 0) + got="sha256:$(sha256sum "$work/upstream.sig" | cut -d ' ' -f 1)" + if [ "$got" = "$layer" ]; then + return 0 + fi + refused_why="its signature layer is $layer, and $s/$file.sig is $got" + return 1 + ;; + 4) ;; + *) + refused_why="$s/$file.sig did not answer, so upstream's signature is not known" + return 1 + ;; + esac + done + refused_why="no server serves $file.sig, and the tag holds one" + return 1 + ;; + sig-missing) + if [ -n "$sig_sha" ] || [ "$decision" != needsig ]; then + refused_why="the database or the tag has a signature for it, and pacman read none ($decision)" + return 1 + fi + for s in "${sources[@]}"; do + rc=0 + fetch_upstream "$s/$file.sig" "$work/upstream.sig" 60 || rc=$? + if [ "$rc" -ne 4 ]; then + refused_why="$s/$file.sig answered (status $rc), so upstream may sign it" + return 1 + fi + done + return 0 + ;; + esac + refused_why="pacman refused it for a reason that is not in its signature" + return 1 +} + +refused=0 +others="" +: > "$work/refused.tsv" +while IFS= read -r p; do + [ -n "$p" ] || continue + awk -F '\t' -v p="$p" '$1 == "@@log" && $2 == p { sub(/^[^\t]*\t[^\t]*\t/, ""); print }' "$log_file" > "$work/one.log" + read -r pkg_rc sig_rc <<< "$(awk -F '\t' -v p="$p" '$1 == "@@probe" && $2 == p { r = $3 " " $4 } END { print (r == "" ? "x x" : r) }' "$log_file")" + IFS=$'\t' read -r class line <<< "$(refusal_reason "$work/one.log" "$pkg_rc" "$sig_rc")" + if upstream_refused "$p" "$class"; then + refused=$((refused + 1)) + printf '%s\t%s: %s\n' "$(awk -F '\t' -v n="$p" '$2 == n { print $1 }' "$stage/$target/records.tsv")" "$class" "$line" >> "$work/refused.tsv" + log "$p: upstream-refused, $class: $line" + else + others="$others $p ($class: $line; $refused_why);" + fi +done < "$work/refused-names.txt" +if [ "$refused" -gt 0 ]; then + cp "$work/refused.tsv" "$refused_file" +fi + +# Upstream-refused is believed only once the trust root is seen to work: a +# keyring package installed with its signature checked, or one package of the +# batch that pacman accepted. A trust root that accepts nothing would refuse +# every package for a signature reason, and excuse them all. +if [ "$refused" -gt 0 ] && [ -z "$others" ] && ! grep -q '^@@trust' "$log_file" && + [ "$(awk 'END { print NR + 0 }' "$work/refused-names.txt")" -ge "$count" ]; then + others=" every package of the batch was refused, and no keyring package was installed: the trust root is not seen to accept anything;" +fi + +if [ "$sigbad" -eq 0 ] && [ -z "$others" ] && [ "$refused" -gt 0 ]; then + verdict passed "$count package(s) read through the wrapper and checked by pacman; $refused of them upstream-refused, pacman refuses them from upstream too: $(cut -f 1 "$refused_file" | tr '\n' ' ' | sed 's/ $//')" + exit 0 +fi +c="$(awk -F '\t' '$1 == "@@sigbad" && !seen[$2]++ { s = s " " $2 ": " $3 ";" } END { print substr(s, 2) }' "$log_file")" +also="" +if [ "$refused" -gt 0 ]; then + also=" $refused upstream-refused besides: $(cut -f 1 "$refused_file" | tr '\n' ' ' | sed 's/ $//')" +fi +verdict failed "the batch failed its checks (status $rc); at fault:${others}${c:+ $c}${also}" exit 1 diff --git a/targets/aarch64-alarm.conf b/targets/aarch64-alarm.conf new file mode 100644 index 0000000..a2f75ae --- /dev/null +++ b/targets/aarch64-alarm.conf @@ -0,0 +1,39 @@ +# Arch Linux ARM, aarch64, alarm. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/aarch64/alarm. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = aarch64 +repo = alarm + +# ⚠ Arch Linux ARM serves no database over https from its own hosts: +# mirror.archlinuxarm.org is a plain http GeoIP redirector to the official +# mirrors. These are official mirrors that answer https under the name +# archlinuxarm.org gives them, push synchronised from the project, run by three +# operators. +db_server = https://fl.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://ca.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://de3.mirror.archlinuxarm.org/$arch/$repo + +# Official mirrors, over https under their sponsors' own names. +server = https://mirrors.dotsrc.org/archlinuxarm/$arch/$repo +server = https://mirror.united-gameserver.de/archlinux-arm/$arch/$repo +server = https://free.nchc.org.tw/archlinuxarm/$arch/$repo + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root: the keyring files of the aarch64 image of +# pkgforge-dev/docker-archlinux (archlinuxarm.gpg and its trust), copied out +# of it and never run. The x86_64 image keeps its own archlinux keyring. +# tag: aarch64 +keyring_image = ghcr.io/pkgforge-dev/archlinux:aarch64@sha256:b3610e2db83532a105802cf05a40ee7273f58231d5d0e507138f0cd9adef2e27 +keyring_platform = linux/arm64 + +# The keyring package is in core: verify installs it from the promoted +# aarch64 core first. +keyring = archlinuxarm-keyring +keyring_from = aarch64-core diff --git a/targets/aarch64-aur.conf b/targets/aarch64-aur.conf new file mode 100644 index 0000000..7af1944 --- /dev/null +++ b/targets/aarch64-aur.conf @@ -0,0 +1,39 @@ +# Arch Linux ARM, aarch64, aur. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/aarch64/aur. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = aarch64 +repo = aur + +# ⚠ Arch Linux ARM serves no database over https from its own hosts: +# mirror.archlinuxarm.org is a plain http GeoIP redirector to the official +# mirrors. These are official mirrors that answer https under the name +# archlinuxarm.org gives them, push synchronised from the project, run by three +# operators. +db_server = https://fl.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://ca.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://de3.mirror.archlinuxarm.org/$arch/$repo + +# Official mirrors, over https under their sponsors' own names. +server = https://mirrors.dotsrc.org/archlinuxarm/$arch/$repo +server = https://mirror.united-gameserver.de/archlinux-arm/$arch/$repo +server = https://free.nchc.org.tw/archlinuxarm/$arch/$repo + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root: the keyring files of the aarch64 image of +# pkgforge-dev/docker-archlinux (archlinuxarm.gpg and its trust), copied out +# of it and never run. The x86_64 image keeps its own archlinux keyring. +# tag: aarch64 +keyring_image = ghcr.io/pkgforge-dev/archlinux:aarch64@sha256:b3610e2db83532a105802cf05a40ee7273f58231d5d0e507138f0cd9adef2e27 +keyring_platform = linux/arm64 + +# The keyring package is in core: verify installs it from the promoted +# aarch64 core first. +keyring = archlinuxarm-keyring +keyring_from = aarch64-core diff --git a/targets/aarch64-core.conf b/targets/aarch64-core.conf new file mode 100644 index 0000000..fd79ac4 --- /dev/null +++ b/targets/aarch64-core.conf @@ -0,0 +1,38 @@ +# Arch Linux ARM, aarch64, core. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/aarch64/core. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = aarch64 +repo = core + +# ⚠ Arch Linux ARM serves no database over https from its own hosts: +# mirror.archlinuxarm.org is a plain http GeoIP redirector to the official +# mirrors. These are official mirrors that answer https under the name +# archlinuxarm.org gives them, push synchronised from the project, run by three +# operators. +db_server = https://fl.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://ca.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://de3.mirror.archlinuxarm.org/$arch/$repo + +# Official mirrors, over https under their sponsors' own names. +server = https://mirrors.dotsrc.org/archlinuxarm/$arch/$repo +server = https://mirror.united-gameserver.de/archlinux-arm/$arch/$repo +server = https://free.nchc.org.tw/archlinuxarm/$arch/$repo + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root: the keyring files of the aarch64 image of +# pkgforge-dev/docker-archlinux (archlinuxarm.gpg and its trust), copied out +# of it and never run. The x86_64 image keeps its own archlinux keyring. +# tag: aarch64 +keyring_image = ghcr.io/pkgforge-dev/archlinux:aarch64@sha256:b3610e2db83532a105802cf05a40ee7273f58231d5d0e507138f0cd9adef2e27 +keyring_platform = linux/arm64 + +# The keyring package, installed from the snapshot before anything else is +# verified. +keyring = archlinuxarm-keyring diff --git a/targets/aarch64-extra.conf b/targets/aarch64-extra.conf new file mode 100644 index 0000000..cd3742f --- /dev/null +++ b/targets/aarch64-extra.conf @@ -0,0 +1,43 @@ +# Arch Linux ARM, aarch64, extra. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/aarch64/extra. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = aarch64 +repo = extra + +# ⚠ Upstream's extra.db holds findnewest-0.3-4/desc as a GNU sparse member of +# 1271 NUL bytes (2024-10-15). pacman lists it with every field None and cannot +# download it. plan passes over it and names it (lib.sh db_check). + +# ⚠ Arch Linux ARM serves no database over https from its own hosts: +# mirror.archlinuxarm.org is a plain http GeoIP redirector to the official +# mirrors. These are official mirrors that answer https under the name +# archlinuxarm.org gives them, push synchronised from the project, run by three +# operators. +db_server = https://fl.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://ca.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://de3.mirror.archlinuxarm.org/$arch/$repo + +# Official mirrors, over https under their sponsors' own names. +server = https://mirrors.dotsrc.org/archlinuxarm/$arch/$repo +server = https://mirror.united-gameserver.de/archlinux-arm/$arch/$repo +server = https://free.nchc.org.tw/archlinuxarm/$arch/$repo + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root: the keyring files of the aarch64 image of +# pkgforge-dev/docker-archlinux (archlinuxarm.gpg and its trust), copied out +# of it and never run. The x86_64 image keeps its own archlinux keyring. +# tag: aarch64 +keyring_image = ghcr.io/pkgforge-dev/archlinux:aarch64@sha256:b3610e2db83532a105802cf05a40ee7273f58231d5d0e507138f0cd9adef2e27 +keyring_platform = linux/arm64 + +# The keyring package is in core: verify installs it from the promoted +# aarch64 core first. +keyring = archlinuxarm-keyring +keyring_from = aarch64-core diff --git a/targets/armv7h-alarm.conf b/targets/armv7h-alarm.conf new file mode 100644 index 0000000..0e13d70 --- /dev/null +++ b/targets/armv7h-alarm.conf @@ -0,0 +1,39 @@ +# Arch Linux ARM, armv7h, alarm. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/armv7h/alarm. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = armv7h +repo = alarm + +# ⚠ Arch Linux ARM serves no database over https from its own hosts: +# mirror.archlinuxarm.org is a plain http GeoIP redirector to the official +# mirrors. These are official mirrors that answer https under the name +# archlinuxarm.org gives them, push synchronised from the project, run by three +# operators. +db_server = https://fl.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://ca.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://de3.mirror.archlinuxarm.org/$arch/$repo + +# Official mirrors, over https under their sponsors' own names. +server = https://mirrors.dotsrc.org/archlinuxarm/$arch/$repo +server = https://mirror.united-gameserver.de/archlinux-arm/$arch/$repo +server = https://free.nchc.org.tw/archlinuxarm/$arch/$repo + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root: the keyring files of the armv7h image of +# pkgforge-dev/docker-archlinux (archlinuxarm.gpg and its trust), copied out +# of it and never run. The x86_64 image keeps its own archlinux keyring. +# tag: armv7h +keyring_image = ghcr.io/pkgforge-dev/archlinux:armv7h@sha256:d9036f1bb4e272f7b72f1ed81cb6f3ffa6b2a80239ede387fac0ff1dbe5f6298 +keyring_platform = linux/arm/v7 + +# The keyring package is in core: verify installs it from the promoted +# armv7h core first. +keyring = archlinuxarm-keyring +keyring_from = armv7h-core diff --git a/targets/armv7h-aur.conf b/targets/armv7h-aur.conf new file mode 100644 index 0000000..d7a86df --- /dev/null +++ b/targets/armv7h-aur.conf @@ -0,0 +1,39 @@ +# Arch Linux ARM, armv7h, aur. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/armv7h/aur. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = armv7h +repo = aur + +# ⚠ Arch Linux ARM serves no database over https from its own hosts: +# mirror.archlinuxarm.org is a plain http GeoIP redirector to the official +# mirrors. These are official mirrors that answer https under the name +# archlinuxarm.org gives them, push synchronised from the project, run by three +# operators. +db_server = https://fl.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://ca.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://de3.mirror.archlinuxarm.org/$arch/$repo + +# Official mirrors, over https under their sponsors' own names. +server = https://mirrors.dotsrc.org/archlinuxarm/$arch/$repo +server = https://mirror.united-gameserver.de/archlinux-arm/$arch/$repo +server = https://free.nchc.org.tw/archlinuxarm/$arch/$repo + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root: the keyring files of the armv7h image of +# pkgforge-dev/docker-archlinux (archlinuxarm.gpg and its trust), copied out +# of it and never run. The x86_64 image keeps its own archlinux keyring. +# tag: armv7h +keyring_image = ghcr.io/pkgforge-dev/archlinux:armv7h@sha256:d9036f1bb4e272f7b72f1ed81cb6f3ffa6b2a80239ede387fac0ff1dbe5f6298 +keyring_platform = linux/arm/v7 + +# The keyring package is in core: verify installs it from the promoted +# armv7h core first. +keyring = archlinuxarm-keyring +keyring_from = armv7h-core diff --git a/targets/armv7h-core.conf b/targets/armv7h-core.conf new file mode 100644 index 0000000..5c2bd55 --- /dev/null +++ b/targets/armv7h-core.conf @@ -0,0 +1,38 @@ +# Arch Linux ARM, armv7h, core. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/armv7h/core. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = armv7h +repo = core + +# ⚠ Arch Linux ARM serves no database over https from its own hosts: +# mirror.archlinuxarm.org is a plain http GeoIP redirector to the official +# mirrors. These are official mirrors that answer https under the name +# archlinuxarm.org gives them, push synchronised from the project, run by three +# operators. +db_server = https://fl.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://ca.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://de3.mirror.archlinuxarm.org/$arch/$repo + +# Official mirrors, over https under their sponsors' own names. +server = https://mirrors.dotsrc.org/archlinuxarm/$arch/$repo +server = https://mirror.united-gameserver.de/archlinux-arm/$arch/$repo +server = https://free.nchc.org.tw/archlinuxarm/$arch/$repo + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root: the keyring files of the armv7h image of +# pkgforge-dev/docker-archlinux (archlinuxarm.gpg and its trust), copied out +# of it and never run. The x86_64 image keeps its own archlinux keyring. +# tag: armv7h +keyring_image = ghcr.io/pkgforge-dev/archlinux:armv7h@sha256:d9036f1bb4e272f7b72f1ed81cb6f3ffa6b2a80239ede387fac0ff1dbe5f6298 +keyring_platform = linux/arm/v7 + +# The keyring package, installed from the snapshot before anything else is +# verified. +keyring = archlinuxarm-keyring diff --git a/targets/armv7h-extra.conf b/targets/armv7h-extra.conf new file mode 100644 index 0000000..d22b4fb --- /dev/null +++ b/targets/armv7h-extra.conf @@ -0,0 +1,39 @@ +# Arch Linux ARM, armv7h, extra. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/armv7h/extra. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = armv7h +repo = extra + +# ⚠ Arch Linux ARM serves no database over https from its own hosts: +# mirror.archlinuxarm.org is a plain http GeoIP redirector to the official +# mirrors. These are official mirrors that answer https under the name +# archlinuxarm.org gives them, push synchronised from the project, run by three +# operators. +db_server = https://fl.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://ca.us.mirror.archlinuxarm.org/$arch/$repo +db_server = https://de3.mirror.archlinuxarm.org/$arch/$repo + +# Official mirrors, over https under their sponsors' own names. +server = https://mirrors.dotsrc.org/archlinuxarm/$arch/$repo +server = https://mirror.united-gameserver.de/archlinux-arm/$arch/$repo +server = https://free.nchc.org.tw/archlinuxarm/$arch/$repo + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root: the keyring files of the armv7h image of +# pkgforge-dev/docker-archlinux (archlinuxarm.gpg and its trust), copied out +# of it and never run. The x86_64 image keeps its own archlinux keyring. +# tag: armv7h +keyring_image = ghcr.io/pkgforge-dev/archlinux:armv7h@sha256:d9036f1bb4e272f7b72f1ed81cb6f3ffa6b2a80239ede387fac0ff1dbe5f6298 +keyring_platform = linux/arm/v7 + +# The keyring package is in core: verify installs it from the promoted +# armv7h core first. +keyring = archlinuxarm-keyring +keyring_from = armv7h-core diff --git a/targets/loong64-core.conf b/targets/loong64-core.conf new file mode 100644 index 0000000..8aca507 --- /dev/null +++ b/targets/loong64-core.conf @@ -0,0 +1,38 @@ +# Arch Linux for Loong64 (LCPU), loong64, core. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/loong64/core. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = loong64 +repo = core + +# The port's own server: its homepage and status page, and the first entry of +# its first mirrorlist (lcpu-club/loongarch-packages 3cb951f633). Cloudflare is +# in front of it. +db_server = https://loongarchlinux.lcpu.dev/loongarch/archlinux/$repo/os/$arch + +# The rest of pacman-mirrorlist-loong64-20250718-1. All five served the same +# core.db and extra.db as the db_server on 2026-10-14. +server = https://mirrors.wsyu.edu.cn/loongarch/archlinux/$repo/os/$arch +server = https://mirrors.pku.edu.cn/loongarch-lcpu/archlinux/$repo/os/$arch +server = https://mirrors.nju.edu.cn/loongarch/archlinux/$repo/os/$arch +server = https://mirrors.pku.edu.cn/loongarch/archlinux/$repo/os/$arch +server = https://mirror.iscas.ac.cn/loongarch/archlinux/$repo/os/$arch + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root: the keyring files of the loong64 image of +# pkgforge-dev/docker-archlinux (archlinux-lcpu.gpg and its trust), copied out +# of it and never run. The x86_64 image keeps its own archlinux keyring, which +# signs the port's any packages. +# tag: loong64 +keyring_image = ghcr.io/pkgforge-dev/archlinux:loong64@sha256:493c2d153bb1b6b4ea3ff07a22bc63516807d1038473994f3f9e04ba9c01e50c +keyring_platform = linux/loong64 + +# Both keyring packages, installed from the snapshot before anything else is +# verified. +keyring = archlinux-keyring archlinux-lcpu-keyring diff --git a/targets/loong64-extra.conf b/targets/loong64-extra.conf new file mode 100644 index 0000000..23d3106 --- /dev/null +++ b/targets/loong64-extra.conf @@ -0,0 +1,45 @@ +# Arch Linux for Loong64 (LCPU), loong64, extra. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/loong64/extra. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = loong64 +repo = extra + +# ⚠ pacman refuses four of its files from upstream too (2026-10-14): +# compiler-rt18-18.1.8-1, x11-ssh-askpass-1.2.4.1-8 and yt6801-dkms-1.0.28-3 +# are signed by keys that have expired, and python-xarray-2026.07.0-1 has no +# %PGPSIG% and no .sig on any server. verify lists them as upstream-refused +# and passes the batch: the mirror serves exactly upstream's files. + +# The port's own server: its homepage and status page, and the first entry of +# its first mirrorlist (lcpu-club/loongarch-packages 3cb951f633). Cloudflare is +# in front of it. +db_server = https://loongarchlinux.lcpu.dev/loongarch/archlinux/$repo/os/$arch + +# The rest of pacman-mirrorlist-loong64-20250718-1. All five served the same +# core.db and extra.db as the db_server on 2026-10-14. +server = https://mirrors.wsyu.edu.cn/loongarch/archlinux/$repo/os/$arch +server = https://mirrors.pku.edu.cn/loongarch-lcpu/archlinux/$repo/os/$arch +server = https://mirrors.nju.edu.cn/loongarch/archlinux/$repo/os/$arch +server = https://mirrors.pku.edu.cn/loongarch/archlinux/$repo/os/$arch +server = https://mirror.iscas.ac.cn/loongarch/archlinux/$repo/os/$arch + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root: the keyring files of the loong64 image of +# pkgforge-dev/docker-archlinux (archlinux-lcpu.gpg and its trust), copied out +# of it and never run. The x86_64 image keeps its own archlinux keyring, which +# signs the port's any packages. +# tag: loong64 +keyring_image = ghcr.io/pkgforge-dev/archlinux:loong64@sha256:493c2d153bb1b6b4ea3ff07a22bc63516807d1038473994f3f9e04ba9c01e50c +keyring_platform = linux/loong64 + +# Both keyring packages are in core: verify installs them from the promoted +# loong64 core first. +keyring = archlinux-keyring archlinux-lcpu-keyring +keyring_from = loong64-core diff --git a/targets/riscv64-core.conf b/targets/riscv64-core.conf new file mode 100644 index 0000000..5e44394 --- /dev/null +++ b/targets/riscv64-core.conf @@ -0,0 +1,43 @@ +# Arch Linux RISC-V, riscv64, core. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/riscv64/core. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = riscv64 +repo = core + +# The port's own servers. riscv.mirror.pkgbuild.com is on Arch Linux's mirror +# network and the port lists it as recommended; archriscv.felixc.at is the +# port's homepage. Both served the same databases on 2026-10-14. +db_server = https://riscv.mirror.pkgbuild.com/repo/$repo +db_server = https://archriscv.felixc.at/repo/$repo + +# Mirrors from the port's homepage: current first, then those a few hours +# behind on 2026-10-14. Left out: mirror.iscas.ac.cn/archriscv (databases of +# 2026-09-27) and mirrors.felixc.at/archriscv (databases of 2026-02-26). +server = https://mirrors.wsyu.edu.cn/archriscv/repo/$repo +server = https://mirror.nyist.edu.cn/archriscv/repo/$repo +server = https://mirrors.ic.unicamp.br/archriscv/repo/$repo +server = https://mirror.ufscar.br/archriscv/repo/$repo +server = https://archriscv.paulsajna.com/repo/$repo +server = https://mirror.nju.edu.cn/archriscv/repo/$repo +server = https://mirrors.pku.edu.cn/archriscv/repo/$repo +server = https://mirrors.sustech.edu.cn/archriscv/repo/$repo + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root: the keyring files of the riscv64 image of +# pkgforge-dev/docker-archlinux, copied out of it and never run. Every +# signature in the port is by a key in archlinux.gpg (2026-10-14), which the +# x86_64 image keeps as its own; the image follows the port if that changes. +# tag: riscv64 +keyring_image = ghcr.io/pkgforge-dev/archlinux:riscv64@sha256:d2b71c9ac98971be75e0e4b29b6bb24dea2e3b302dfc27288567193bf16091d4 +keyring_platform = linux/riscv64 + +# The keyring package, installed from the snapshot before anything else is +# verified. +keyring = archlinux-keyring diff --git a/targets/riscv64-extra.conf b/targets/riscv64-extra.conf new file mode 100644 index 0000000..e129eba --- /dev/null +++ b/targets/riscv64-extra.conf @@ -0,0 +1,49 @@ +# Arch Linux RISC-V, riscv64, extra. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/riscv64/extra. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = riscv64 +repo = extra + +# ⚠ Upstream's extra.db holds ntk-1.3.1001-4/desc as a GNU sparse member of +# 576 NUL bytes (2024-09-23), on every current mirror. pacman lists it with +# every field None and cannot download it. plan passes over it and names it +# (lib.sh db_check). + +# The port's own servers. riscv.mirror.pkgbuild.com is on Arch Linux's mirror +# network and the port lists it as recommended; archriscv.felixc.at is the +# port's homepage. Both served the same databases on 2026-10-14. +db_server = https://riscv.mirror.pkgbuild.com/repo/$repo +db_server = https://archriscv.felixc.at/repo/$repo + +# Mirrors from the port's homepage: current first, then those a few hours +# behind on 2026-10-14. Left out: mirror.iscas.ac.cn/archriscv (databases of +# 2026-09-27) and mirrors.felixc.at/archriscv (databases of 2026-02-26). +server = https://mirrors.wsyu.edu.cn/archriscv/repo/$repo +server = https://mirror.nyist.edu.cn/archriscv/repo/$repo +server = https://mirrors.ic.unicamp.br/archriscv/repo/$repo +server = https://mirror.ufscar.br/archriscv/repo/$repo +server = https://archriscv.paulsajna.com/repo/$repo +server = https://mirror.nju.edu.cn/archriscv/repo/$repo +server = https://mirrors.pku.edu.cn/archriscv/repo/$repo +server = https://mirrors.sustech.edu.cn/archriscv/repo/$repo + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root: the keyring files of the riscv64 image of +# pkgforge-dev/docker-archlinux, copied out of it and never run. Every +# signature in the port is by a key in archlinux.gpg (2026-10-14), which the +# x86_64 image keeps as its own; the image follows the port if that changes. +# tag: riscv64 +keyring_image = ghcr.io/pkgforge-dev/archlinux:riscv64@sha256:d2b71c9ac98971be75e0e4b29b6bb24dea2e3b302dfc27288567193bf16091d4 +keyring_platform = linux/riscv64 + +# The keyring package is in core: verify installs it from the promoted +# riscv64 core first. +keyring = archlinux-keyring +keyring_from = riscv64-core diff --git a/targets/x86_64-core.conf b/targets/x86_64-core.conf index e4f3457..56870b1 100644 --- a/targets/x86_64-core.conf +++ b/targets/x86_64-core.conf @@ -16,11 +16,15 @@ db_server = https://fastly.mirror.pkgbuild.com/$repo/os/$arch # any mirror will do here. server = https://mirrors.logal.dev/archlinux/$repo/os/$arch server = https://arch.mirror.constant.com/$repo/os/$arch +server = https://ftp.halifax.rwth-aachen.de/archlinux/$repo/os/$arch +server = https://mirrors.rit.edu/archlinux/$repo/os/$arch -# The image pacman verifies a staged snapshot in, and its platform. +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. # tag: base verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 verify_platform = linux/amd64 -# The keyring package, refreshed from the mirror before anything is verified. +# The trust root is the verify image's own keyring. The keyring package is +# installed from the snapshot before anything else is verified. keyring = archlinux-keyring diff --git a/targets/x86_64-extra.conf b/targets/x86_64-extra.conf new file mode 100644 index 0000000..bf448dd --- /dev/null +++ b/targets/x86_64-extra.conf @@ -0,0 +1,31 @@ +# Arch Linux, x86_64, extra. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/x86_64/extra. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = x86_64 +repo = extra + +# Servers that may supply the database, tried in order. Arch Linux runs both, +# so a database is taken only from the distribution's own hosts. +db_server = https://geo.mirror.pkgbuild.com/$repo/os/$arch +db_server = https://fastly.mirror.pkgbuild.com/$repo/os/$arch + +# More servers package bytes may come from, after the db_servers. Every byte +# is checked against the database's sha256 and size before it is pushed, so +# any mirror will do here. +server = https://mirrors.logal.dev/archlinux/$repo/os/$arch +server = https://arch.mirror.constant.com/$repo/os/$arch +server = https://ftp.halifax.rwth-aachen.de/archlinux/$repo/os/$arch +server = https://mirrors.rit.edu/archlinux/$repo/os/$arch + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root is the verify image's own keyring. The keyring package is in +# core: verify installs it from the promoted x86_64 core first. +keyring = archlinux-keyring +keyring_from = x86_64-core diff --git a/targets/x86_64-multilib.conf b/targets/x86_64-multilib.conf new file mode 100644 index 0000000..df2dde9 --- /dev/null +++ b/targets/x86_64-multilib.conf @@ -0,0 +1,31 @@ +# Arch Linux, x86_64, multilib. +# +# Mirrored to ghcr.io/pkgforge-dev/mirror-archlinux/x86_64/multilib. +# Read by scripts/lib.sh, never sourced. One key per line, `key = value`. + +arch = x86_64 +repo = multilib + +# Servers that may supply the database, tried in order. Arch Linux runs both, +# so a database is taken only from the distribution's own hosts. +db_server = https://geo.mirror.pkgbuild.com/$repo/os/$arch +db_server = https://fastly.mirror.pkgbuild.com/$repo/os/$arch + +# More servers package bytes may come from, after the db_servers. Every byte +# is checked against the database's sha256 and size before it is pushed, so +# any mirror will do here. +server = https://mirrors.logal.dev/archlinux/$repo/os/$arch +server = https://arch.mirror.constant.com/$repo/os/$arch +server = https://ftp.halifax.rwth-aachen.de/archlinux/$repo/os/$arch +server = https://mirrors.rit.edu/archlinux/$repo/os/$arch + +# pacman verifies in the x86_64 Arch image, with Architecture set to the +# target's arch: no emulation. +# tag: base +verify_image = docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +verify_platform = linux/amd64 + +# The trust root is the verify image's own keyring. The keyring package is in +# core: verify installs it from the promoted x86_64 core first. +keyring = archlinux-keyring +keyring_from = x86_64-core diff --git a/tests/fixtures/build-repo b/tests/fixtures/build-repo index e8e46bd..e4c8727 100644 --- a/tests/fixtures/build-repo +++ b/tests/fixtures/build-repo @@ -1,14 +1,18 @@ #!/usr/bin/env bash # -# Build the integration suite's upstream: a small signed pacman repository. -# Runs inside an Arch container, as the calling user, with the upstream root -# mounted at /out. +# Build one of the integration suite's upstream repositories: a small signed +# pacman repository. Runs inside an Arch container, as the calling user, with +# the upstream root mounted at /out. # -# bash build-repo STATE [DIR] +# bash build-repo STATE [DIR] [DB] # -# Writes /out/x86_64/DIR (default test). STATE names the contents: +# Writes /out/DIR (default x86_64/test) with the database DB.db (default test). +# STATE names the contents. For the main target, x86_64/test: # one alpha (an epoch and a +), beta_pc (_ before p and c), gamma 3.0-1 -# two as one, with gamma 3.0-2 built later +# two as one, with gamma 3.0-2 built later, and eta signed by the +# second key, which only the keyring package of x86_64/kr trusts +# resigned as two, with alpha signed again: the same bytes, a new signature +# four as two, with theta 1-1 added, the largest package # replaced as two, with beta_pc rebuilt under the same filename # three as two, with delta 1-1 added # ahead-alpha as two, with alpha rebuilt later @@ -17,36 +21,98 @@ # untrusted as two, with epsilon signed by a key no one trusts # nopgpsig as two, with zeta added, and a database that carries no # %PGPSIG%: the detached .sig files are the only signatures, as on -# most of ArchPOWER and archriscv +# most of archriscv and loong64 core +# nopgpsig-four as nopgpsig, with theta 1-1 added +# For the other targets: +# keyring test-keyring 1-1 (x86_64/kr): the keyring files of the first and +# the second key, under /usr/share/pacman/keyrings +# port an aarch64 repository (aarch64/port): port 1-1 aarch64; unsigned +# 1-1, which upstream signs nowhere; expired 1-1, signed by a key +# that has since expired; and ghost-1-1, a package directory whose +# desc is a GNU sparse member of NUL bytes only, as aarch64 extra +# findnewest-0.3-4 is upstream +# port-junk as port, with junk-1-1/desc holding bytes that are no record # -# Every other state's database carries %PGPSIG% for every package, as Arch's -# own core and extra do. pacman 7's repo-add embeds a signature only when given -# --include-sigs; without it the suite would never take the %PGPSIG% path that -# every real core package takes. +# Every database carries %PGPSIG% for every signed package, as Arch's own core +# and extra do, except nopgpsig. pacman 7's repo-add embeds a signature only +# when given --include-sigs; without it the suite would never take the +# %PGPSIG% path that every real core package takes. # -# Package bytes are fixed by their inputs, so a state built twice gives the -# same package files; that is what lets the suite go back to an earlier state -# and see the mirror refuse it. +# Keys, made once per suite run in /out: gnupg (the test key, which the trust +# root holds), gnupg-second (only the keyring package holds it), +# gnupg-untrusted and gnupg-expired (made and used with a clock 10 days back, +# valid for one day). /out/test.asc is the test key alone; /out/trust-root +# holds the keyring files the suite's keyring image carries: the test key and +# the expired one. +# +# ⭐ Package bytes are fixed by their inputs, and each distinct package file is +# signed once per suite run: its .sig is kept in /out/sigcache by the file's +# sha256 and reused whenever the same bytes are built again, as upstream never +# signs a published file twice. So a state built twice gives the same package +# files and the same signatures, which is what lets the suite go back to an +# earlier state and see the mirror refuse it. resigned is the one exception, +# made on purpose: it replaces alpha's kept signature. set -euo pipefail -state="${1:?usage: build-repo STATE [DIR]}" -repo="/out/x86_64/${2:-test}" +state="${1:?usage: build-repo STATE [DIR] [DB]}" +repo="/out/${2:-x86_64/test}" +db="${3:-test}" export GNUPGHOME=/out/gnupg +# mkkey HOME UID [FAKED-TIME] +mkkey() { + local faked=() + if [ -n "${3:-}" ]; then + faked=(--faked-system-time "$3") + fi + mkdir -p "$1" + chmod 0700 "$1" + GNUPGHOME="$1" gpg --batch "${faked[@]}" --passphrase '' --quick-gen-key "$2" ed25519 sign "${4:-never}" +} + +# fpr_of HOME -> the fingerprint of the one key in HOME +fpr_of() { + GNUPGHOME="$1" gpg --batch --with-colons --list-keys | awk -F : '$1 == "fpr" && f == "" { f = $10 } END { print f }' +} + if [ ! -d "$GNUPGHOME" ]; then - mkdir -p -m 0700 "$GNUPGHOME" - gpg --batch --passphrase '' --quick-gen-key 'mirror-archlinux test ' ed25519 sign never + past="$(date -u -d '10 days ago' +%Y%m%dT%H%M%S)" + mkkey "$GNUPGHOME" 'mirror-archlinux test ' + mkkey /out/gnupg-second 'mirror-archlinux second key ' + mkkey /out/gnupg-untrusted 'untrusted ' + mkkey /out/gnupg-expired 'mirror-archlinux expired key ' "$past!" 1d gpg --armor --export > /out/test.asc -fi -if [ "$state" = untrusted ] && [ ! -d /out/gnupg-untrusted ]; then - mkdir -p -m 0700 /out/gnupg-untrusted - GNUPGHOME=/out/gnupg-untrusted gpg --batch --passphrase '' --quick-gen-key 'untrusted ' ed25519 sign never + mkdir -p /out/trust-root /out/keyring-files + # The suite's trust root: the test key and the expired one. + { gpg --export; GNUPGHOME=/out/gnupg-expired gpg --export; } > /out/trust-root/test.gpg + printf '%s:4:\n' "$(fpr_of "$GNUPGHOME")" "$(fpr_of /out/gnupg-expired)" > /out/trust-root/test-trusted + : > /out/trust-root/test-revoked + # The keyring package: the test key and the second key. + { gpg --export; GNUPGHOME=/out/gnupg-second gpg --export; } > /out/keyring-files/test.gpg + printf '%s:4:\n' "$(fpr_of "$GNUPGHOME")" "$(fpr_of /out/gnupg-second)" > /out/keyring-files/test-trusted + : > /out/keyring-files/test-revoked + mkdir -p /out/sigcache fi rm -rf "$repo" mkdir -p "$repo" -# mkpkg NAME VERSION ARCH BUILDDATE CONTENT [GNUPGHOME] +# sign FILE HOME [fresh]: FILE.sig, made once per distinct file and key +sign() { + local file="$1" home="$2" sum cached faked=() + sum="$(sha256sum "$file" | cut -d ' ' -f 1)" + cached="/out/sigcache/$sum-$(basename "$home").sig" + if [ ! -f "$cached" ] || [ "${3:-}" = fresh ]; then + if [ "$home" = /out/gnupg-expired ]; then + faked=(--faked-system-time "$(date -u -d '10 days ago 1 hour' +%Y%m%dT%H%M%S)!") + fi + GNUPGHOME="$home" gpg --batch --yes "${faked[@]}" --detach-sign --no-armor -o "$cached.tmp" "$file" + mv -f "$cached.tmp" "$cached" + fi + cp "$cached" "$file.sig" +} + +# mkpkg NAME VERSION ARCH BUILDDATE CONTENT [GNUPGHOME|none] [fresh] mkpkg() { local name="$1" version="$2" arch="$3" built="$4" content="$5" home="${6:-$GNUPGHOME}" d file d="$(mktemp -d)" @@ -64,15 +130,67 @@ size = 64 arch = $arch license = MIT EOF + local paths=(.PKGINFO usr) + if [ "$name" = test-keyring ]; then + mkdir -p "$d/usr/share/pacman/keyrings" + cp /out/keyring-files/test.gpg /out/keyring-files/test-trusted /out/keyring-files/test-revoked "$d/usr/share/pacman/keyrings/" + touch -d "@$built" "$d"/usr/share/pacman/keyrings/* "$d/usr/share/pacman/keyrings" "$d/usr/share/pacman" + fi touch -d "@$built" "$d/.PKGINFO" "$d/usr/share/mirror-test/$name" "$d/usr/share/mirror-test" "$d/usr/share" "$d/usr" file="$name-$version-$arch.pkg.tar.zst" - (cd "$d" && bsdtar --zstd --uid 0 --gid 0 --uname root --gname root -cf "$repo/$file" .PKGINFO usr) - GNUPGHOME="$home" gpg --batch --yes --detach-sign --no-armor -o "$repo/$file.sig" "$repo/$file" + (cd "$d" && bsdtar --zstd --uid 0 --gid 0 --uname root --gname root -cf "$repo/$file" "${paths[@]}") + if [ "$home" != none ]; then + sign "$repo/$file" "$home" "${7:-}" + fi rm -rf "$d" } +# add_member DB.tar.gz DIR nul|junk: a package directory with a desc that +# gives no record, appended to the database +add_member() { + local t + t="$(mktemp -d)" + gzip -dc "$1" > "$t/db.tar" + mkdir -p "$t/m/$2" + if [ "$3" = nul ]; then + # A file of holes only: GNU tar stores it as a sparse member with no data, + # which reads back as NUL bytes. + truncate -s 1271 "$t/m/$2/desc" + else + printf 'bytes that are no record\n' > "$t/m/$2/desc" + fi + (cd "$t/m" && tar --sparse --format=posix -rf "$t/db.tar" "$2/desc") + gzip -9 -n -c "$t/db.tar" > "$1" + rm -rf "$t" +} + +case "$state" in + keyring) + mkpkg test-keyring 1-1 any 1700000050 "the keyring of the test keys" + repo-add --quiet --include-sigs "$repo/$db.db.tar.gz" "$repo"/*.pkg.tar.zst + ls -l "$repo" + exit 0 + ;; + port | port-junk) + mkpkg port 1-1 aarch64 1700000000 "an aarch64 package" + mkpkg unsigned 1-1 any 1700000010 "upstream signs this nowhere" none + mkpkg expired 1-1 any 1700000020 "signed by a key that has since expired" /out/gnupg-expired + repo-add --quiet --include-sigs "$repo/$db.db.tar.gz" "$repo"/*.pkg.tar.zst + for f in "$db.db.tar.gz" "$db.files.tar.gz"; do + add_member "$repo/$f" ghost-1-1 nul + if [ "$state" = port-junk ]; then + add_member "$repo/$f" junk-1-1 junk + fi + done + tar -tvf "$repo/$db.db.tar.gz" + ls -l "$repo" + exit 0 + ;; +esac + case "$state" in ahead-alpha) mkpkg alpha 1:1.0+r3-1 any 1700300000 "alpha rebuilt later" ;; + resigned) mkpkg alpha 1:1.0+r2-1 any 1700000000 "alpha one" "$GNUPGHOME" fresh ;; *) mkpkg alpha 1:1.0+r2-1 any 1700000000 "alpha one" ;; esac case "$state" in @@ -84,24 +202,32 @@ case "$state" in ahead-gamma) mkpkg gamma 3.0-3 any 1700500000 "gamma three" ;; *) mkpkg gamma 3.0-2 any 1700100200 "gamma two" ;; esac +if [ "$state" != one ]; then + mkpkg eta 1-1 any 1700150000 "eta, signed by the second key" /out/gnupg-second +fi if [ "$state" = three ]; then mkpkg delta 1-1 any 1700200000 "delta one" fi +if [ "$state" = four ] || [ "$state" = nopgpsig-four ]; then + # Larger than any other package, so the smallest one, verify's canary, is a + # file the plan skips. + mkpkg theta 1-1 any 1700700000 "$(for i in $(seq 1 25); do printf 'theta %s\n' "$i" | sha256sum | cut -d ' ' -f 1; done)" +fi if [ "$state" = untrusted ]; then mkpkg epsilon 1-1 any 1700400000 "epsilon, signed by a stranger" /out/gnupg-untrusted fi -if [ "$state" = nopgpsig ]; then +if [ "$state" = nopgpsig ] || [ "$state" = nopgpsig-four ]; then mkpkg zeta 1-1 any 1700600000 "zeta, signed beside the database only" # Without --include-sigs repo-add embeds none; the signatures are also hidden # while it runs, so the database carries none whatever its default becomes. mkdir -p "$repo/.sigs" mv "$repo"/*.sig "$repo/.sigs/" - repo-add --quiet "$repo/test.db.tar.gz" "$repo"/*.pkg.tar.zst + repo-add --quiet "$repo/$db.db.tar.gz" "$repo"/*.pkg.tar.zst mv "$repo"/.sigs/*.sig "$repo/" rmdir "$repo/.sigs" ls -l "$repo" exit 0 fi -repo-add --quiet --include-sigs "$repo/test.db.tar.gz" "$repo"/*.pkg.tar.zst +repo-add --quiet --include-sigs "$repo/$db.db.tar.gz" "$repo"/*.pkg.tar.zst ls -l "$repo" diff --git a/tests/integration/10-pipeline.sh b/tests/integration/10-pipeline.sh index 34be897..d5f02cc 100644 --- a/tests/integration/10-pipeline.sh +++ b/tests/integration/10-pipeline.sh @@ -3,16 +3,28 @@ # The whole mirror, end to end, against a registry and an upstream of its own. # # A local registry (registry:3) stands in for GHCR and a local web server for -# the upstream mirror. A signed repository is built with filenames that stress -# the tag mapping. Then the real scripts run as the workflow runs them: plan, -# mirror-shard, stage, verify-batch, promote. pacman in an Arch container then -# installs from the mirror through the wrapper, with and without DownloadUser, -# and the cases that must be refused are driven on purpose: a database older -# than the promoted one, a file that changed under its own name, and a tag -# that already holds other bytes. +# the upstream mirrors. Three targets are built, each a signed repository: +# x86_64-test filenames that stress the tag mapping; its keyring package +# comes from x86_64-kr (keyring_from) +# x86_64-kr the keyring package of the test keys +# aarch64-port another arch, verified in the x86_64 image with +# Architecture = aarch64; a package upstream signs nowhere, one +# signed by a key that has expired, and a package directory +# whose desc is NUL bytes only +# The trust root of every target is a keyring image of the suite's own, +# pushed to the local registry (keyring_image). Then the real scripts run as +# the workflows run them, each target in a chain of its own: plan, +# mirror-shard, stage, verify-batch, promote, report. pacman in an Arch +# container then installs from the mirror through the wrapper, with and +# without DownloadUser, and the cases that must be refused are driven on +# purpose. +# +# Every port is chosen by the container runtime at run time and every +# container name carries this shell's pid, so several runs share one host. # # Needs docker or podman (CONTAINER_RUNTIME), curl, jq, tar, sha256sum, and -# network to pull the pinned images and oras. +# network to pull the pinned images and oras. IT_ORAS_CACHE names a directory +# that keeps the oras download between runs (scripts/install-oras). set -euo pipefail # shellcheck source=../lib/harness.sh . "$REPO_ROOT/tests/lib/harness.sh" @@ -23,15 +35,15 @@ ARCH_IMAGE=docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2 REGISTRY_IMAGE=docker.io/library/registry:3@sha256:ddf754342cfc8acc51a56d5d0ab6af06826461864460636d8bd5c546dab2a7b8 # tag: 1.37 HTTPD_IMAGE=docker.io/library/busybox:1.37@sha256:bdf57e528e45e4433820e045b29b4597825a1c9e38353532d90a01445013f82e -REG_PORT="${IT_REGISTRY_PORT:-5055}" -UP_PORT="${IT_UPSTREAM_PORT:-5056}" runtime_detect work="$(work_dir)" -reg_name="mirror-it-registry-$$" -up_name="mirror-it-upstream-$$" +suffix="$$-$RANDOM" +reg_name="mirror-it-registry-$suffix" +up_name="mirror-it-upstream-$suffix" +kimage_local="mirror-it-keyring-$suffix" cleanup() { - "$RUNTIME" rm -f "$reg_name" "$up_name" > /dev/null 2>&1 || : + "$RUNTIME" rm -f "$reg_name" "$up_name" > "$work/cleanup.log" 2>&1 || echo "# cleanup: $(cat "$work/cleanup.log")" >&2 if [ "${IT_KEEP:-0}" = 1 ]; then echo "# kept $work" >&2 else @@ -43,49 +55,61 @@ trap cleanup EXIT # Containers that write into the work directory do so as the calling user, # so the directory can be removed afterwards without root. as_me=(--user "$(id -u):$(id -g)") +podman=0 case "$(basename "$RUNTIME")" in - *podman*) as_me=(--userns keep-id) ;; + *podman*) + as_me=(--userns keep-id) + podman=1 + ;; esac mkdir -p "$work/upstream" "$work/targets" "$work/out" #---------------------------------------------------------------------------# -# The registry and the upstream +# The registry and the upstream, on ports the runtime picks #---------------------------------------------------------------------------# -"$RUNTIME" run -d --rm --name "$reg_name" -p "127.0.0.1:$REG_PORT:5000" "$REGISTRY_IMAGE" > /dev/null -"$RUNTIME" run -d --rm --name "$up_name" -p "127.0.0.1:$UP_PORT:8000" -v "$work/upstream:/srv:ro" \ +# port_of NAME PORT -> the host port the runtime mapped to the container's PORT +port_of() { + "$RUNTIME" port "$1" "$2/tcp" | sed -n 's/^127\.0\.0\.1:\([0-9][0-9]*\)$/\1/p' | sed -n 1p +} +"$RUNTIME" run -d --rm --name "$reg_name" -p 127.0.0.1::5000 "$REGISTRY_IMAGE" > /dev/null +"$RUNTIME" run -d --rm --name "$up_name" -p 127.0.0.1::8000 -v "$work/upstream:/srv:ro" \ "$HTTPD_IMAGE" httpd -f -p 8000 -h /srv > /dev/null +REG_PORT="$(port_of "$reg_name" 5000)" +UP_PORT="$(port_of "$up_name" 8000)" ready=0 -for _ in $(seq 1 30); do - if curl -q -fsS --connect-timeout 2 --max-time 5 "http://127.0.0.1:$REG_PORT/v2/" > /dev/null && - curl -q -sS --connect-timeout 2 --max-time 5 -o /dev/null "http://127.0.0.1:$UP_PORT/"; then - ready=1 - break - fi - sleep 1 -done +if [ -n "$REG_PORT" ] && [ -n "$UP_PORT" ]; then + for _ in $(seq 1 30); do + if curl -q -fsS --connect-timeout 2 --max-time 5 "http://127.0.0.1:$REG_PORT/v2/" > /dev/null && + curl -q -sS --connect-timeout 2 --max-time 5 -o /dev/null "http://127.0.0.1:$UP_PORT/"; then + ready=1 + break + fi + sleep 1 + done +fi if [ "$ready" -ne 1 ]; then - fail "the local registry and upstream answer" "after 30 s" "reproduce: $RUNTIME logs $reg_name; $RUNTIME logs $up_name" + fail "the local registry and upstream answer" "registry port '$REG_PORT', upstream port '$UP_PORT', after 30 s" \ + "reproduce: $RUNTIME logs $reg_name; $RUNTIME logs $up_name" summary exit 1 fi +echo "# registry on 127.0.0.1:$REG_PORT, upstream on 127.0.0.1:$UP_PORT" +# podman reads a registry over plain http only when its configuration says +# so; docker does for any registry on the loopback. +if [ "$podman" -eq 1 ]; then + printf '[[registry]]\nlocation = "127.0.0.1:%s"\ninsecure = true\n' "$REG_PORT" > "$work/registries.conf" + export CONTAINERS_REGISTRIES_CONF="$work/registries.conf" +fi + +# build STATE [DIR] [DB]: tests/fixtures/build-repo in the Arch image build() { "$RUNTIME" run --rm "${as_me[@]}" -e HOME=/tmp -v "$work/upstream:/out" \ -v "$REPO_ROOT/tests/fixtures/build-repo:/build-repo.sh:ro" \ - "$ARCH_IMAGE" bash /build-repo.sh "$1" "${2:-test}" > "$work/build-$1.log" 2>&1 + "$ARCH_IMAGE" bash /build-repo.sh "$@" > "$work/build-$1.log" 2>&1 } -cat > "$work/targets/x86_64-test.conf" << EOF -arch = x86_64 -repo = test -db_server = http://127.0.0.1:$UP_PORT/\$arch/\$repo -server = http://127.0.0.1:$UP_PORT/\$arch/\$repo -# tag: base -verify_image = $ARCH_IMAGE -verify_platform = linux/amd64 -EOF - export TARGETS_DIR="$work/targets" export MIRROR_REGISTRY="127.0.0.1:$REG_PORT" export MIRROR_REGISTRY_SCHEME=http @@ -93,98 +117,248 @@ export MIRROR_REGISTRY_AUTH=none export MIRROR_NAMESPACE=it/mirror export UPSTREAM_PROTO='=http,https' export VERIFY_NETWORK=host -export VERIFY_TRUST_KEY="$work/upstream/test.asc" export CONTAINER_RUNTIME="$RUNTIME" export SHARD_FILES=2 export PARALLEL=2 +export ORAS_CACHE="${IT_ORAS_CACHE:-$work/oras-cache}" REPO="it/mirror/x86_64/test" BASE="http://127.0.0.1:$REG_PORT/v2/$REPO" +UP="http://127.0.0.1:$UP_PORT" + +#---------------------------------------------------------------------------# +# The upstream repositories, and the suite's keyring image +#---------------------------------------------------------------------------# +build one +build keyring x86_64/kr kr +build port aarch64/port port + +mkdir -p "$work/kimage/keyrings" +cp "$work/upstream/trust-root/test.gpg" "$work/upstream/trust-root/test-trusted" "$work/upstream/trust-root/test-revoked" "$work/kimage/keyrings/" +printf 'FROM %s\nCOPY keyrings/ /usr/share/pacman/keyrings/\n' "$HTTPD_IMAGE" > "$work/kimage/Containerfile" +"$RUNTIME" build -t "$kimage_local" -f "$work/kimage/Containerfile" "$work/kimage" > "$work/kimage.log" 2>&1 +"$RUNTIME" tag "$kimage_local" "127.0.0.1:$REG_PORT/it/keyring:test" +"$RUNTIME" push "127.0.0.1:$REG_PORT/it/keyring:test" >> "$work/kimage.log" 2>&1 +"$RUNTIME" rmi "$kimage_local" "127.0.0.1:$REG_PORT/it/keyring:test" >> "$work/kimage.log" 2>&1 +kdigest="$(curl -q -sS --connect-timeout 5 --max-time 20 -o /dev/null -D - \ + -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \ + "http://127.0.0.1:$REG_PORT/v2/it/keyring/manifests/test" | + tr -d '\r' | awk 'tolower($1) == "docker-content-digest:" { print $2 }')" +KEYRING_IMAGE="127.0.0.1:$REG_PORT/it/keyring:test@$kdigest" +check "the suite's keyring image is on the local registry" 1 "$(printf '%s' "$kdigest" | grep -cE '^sha256:[0-9a-f]{64}$')" \ + "read $work/kimage.log" + +# conf ARCH REPO [EXTRA LINES...]: a target file. The package sources are the +# upstream twice, by two URLs for the same bytes, so two shards start from +# different servers. +conf() { + local arch="$1" repo="$2" + shift 2 + { + printf 'arch = %s\nrepo = %s\n' "$arch" "$repo" + # shellcheck disable=SC2016 # $arch and $repo are the target file's, not the shell's + printf 'db_server = %s/$arch/$repo\nserver = %s/$arch/./$repo\n' "$UP" "$UP" + printf '# tag: base\nverify_image = %s\nverify_platform = linux/amd64\n' "$ARCH_IMAGE" + printf '# tag: test\nkeyring_image = %s\nkeyring_platform = linux/amd64\n' "$KEYRING_IMAGE" + if [ "$#" -gt 0 ]; then + printf '%s\n' "$@" + fi + } > "$work/targets/$arch-$repo.conf" +} +conf x86_64 test 'keyring = test-keyring' 'keyring_from = x86_64-kr' +conf x86_64 kr 'keyring = test-keyring' +conf aarch64 port -# sync N: one run, every step, as the workflow does it; state in $work/out/N +# sync N [TARGET ...]: one run, as the workflows run it: plan for every +# target, then each pending target's chain in the order given (default kr, +# test, port), then the report; state in $work/out/N sync() { - local o="$work/out/$1" s b rc=0 - mkdir -p "$o" + local n="$1" o t s b rc + shift + o="$work/out/$n" + mkdir -p "$o/summaries" if ! "$REPO_ROOT/scripts/plan" "$o/plan" > "$o/plan.log" 2>&1; then - diag "sync $1: plan failed" - logs_of "$1" + diag "sync $n: plan failed" + logs_of "$n" return 0 fi - for s in "$o"/plan/shards/*.tsv; do - [ -e "$s" ] || continue - rc=0 - SHARD_INDEX="$(basename "$s" .tsv)" "$REPO_ROOT/scripts/mirror-shard" "$s" "$o/results/$(basename "$s" .tsv)" \ - > "$o/mirror-$(basename "$s" .tsv).log" 2>&1 || rc=$? - done - if ! "$REPO_ROOT/scripts/stage" "$o/plan" "$o/results" "$o/stage" > "$o/stage.log" 2>&1; then - diag "sync $1: stage failed" - logs_of "$1" - return 0 + if [ "$#" -eq 0 ]; then + set -- x86_64-kr x86_64-test aarch64-port fi - for b in "$o"/stage/x86_64-test/batches/*.txt; do - [ -e "$b" ] || continue - "$REPO_ROOT/scripts/verify-batch" "$o/stage" x86_64-test "$(basename "$b" .txt)" "$o/verify" \ - > "$o/verify-$(basename "$b" .txt).log" 2>&1 || rc=$? + for t in "$@"; do + [ -f "$o/plan/$t/state" ] && [ "$(cat "$o/plan/$t/state")" = pending ] || continue + for s in "$o/plan/$t"/shards/*.tsv; do + [ -e "$s" ] || continue + rc=0 + SHARD_INDEX="$(basename "$s" .tsv)" "$REPO_ROOT/scripts/mirror-shard" "$s" "$o/results/$t/$(basename "$s" .tsv)" \ + > "$o/mirror-$t-$(basename "$s" .tsv).log" 2>&1 || rc=$? + done + if ! "$REPO_ROOT/scripts/stage" "$o/plan" "$o/results/$t" "$o/stage" "$t" > "$o/stage-$t.log" 2>&1; then + diag "sync $n: stage of $t failed" + logs_of "$n" + continue + fi + for b in "$o/stage/$t"/batches/*.txt; do + [ -e "$b" ] || continue + rc=0 + "$REPO_ROOT/scripts/verify-batch" "$o/stage" "$t" "$(basename "$b" .txt)" "$o/verify" \ + > "$o/verify-$t-$(basename "$b" .txt).log" 2>&1 || rc=$? + done + rc=0 + "$REPO_ROOT/scripts/promote" "$o/stage" "$o/verify" "$t" > "$o/promote-$t.log" 2>&1 || rc=$? + if [ "$rc" -eq 0 ]; then + printf '%s\tpromoted\t%s\n' "$t" "$(cut -f 3,4 "$o/stage/$t/promoted.tsv" | tr '\t' ' ')" > "$o/promote-$t.tsv" + else + printf '%s\tfailed\t%s\n' "$t" "$(grep -m 1 'promote:' "$o/promote-$t.log" | tr '\t' ' ')" > "$o/promote-$t.tsv" + fi + "$REPO_ROOT/scripts/report" collect "$t" "$o/stage" "$o/verify" "$o/promote-$t.tsv" "$o/summaries/$t" \ + > "$o/collect-$t.log" 2>&1 done - "$REPO_ROOT/scripts/promote" "$o/stage" "$o/verify" x86_64-test > "$o/promote.log" 2>&1 || rc=$? + rc=0 + "$REPO_ROOT/scripts/report" "$o/plan" "$o/summaries" > "$o/report.md" 2> "$o/report.log" || rc=$? + printf '%s\n' "$rc" > "$o/report.rc" return 0 } state_of() { - if [ -f "$work/out/$1/$2/x86_64-test/state" ]; then - cat "$work/out/$1/$2/x86_64-test/state" + local t="${3:-x86_64-test}" + if [ -f "$work/out/$1/$2/$t/state" ]; then + cat "$work/out/$1/$2/$t/state" else echo "no state" fi } +# verdict_of N [TARGET] [BATCH] -> result and detail of a verify verdict +verdict_of() { + local f="$work/out/$1/verify/verify-${2:-x86_64-test}-${3:-0}.tsv" + if [ -f "$f" ]; then + cut -f 3,4 "$f" + else + echo "no verdict" + fi +} + manifest() { - curl -q -fsS --connect-timeout 5 --max-time 20 -H 'Accept: application/vnd.oci.image.manifest.v1+json' "$BASE/manifests/$1" + curl -q -fsS --connect-timeout 5 --max-time 20 -H 'Accept: application/vnd.oci.image.manifest.v1+json' "http://127.0.0.1:$REG_PORT/v2/${2:-$REPO}/manifests/$1" } -# digest_of TAG -> the digest the tag names, or nothing when it names none +# digest_of TAG [REPOSITORY] -> the digest the tag names, or nothing digest_of() { curl -q -sS --connect-timeout 5 --max-time 20 -o /dev/null -D - \ - -H 'Accept: application/vnd.oci.image.manifest.v1+json' "$BASE/manifests/$1" | + -H 'Accept: application/vnd.oci.image.manifest.v1+json' "http://127.0.0.1:$REG_PORT/v2/${2:-$REPO}/manifests/$1" | tr -d '\r' | awk '$1 ~ /^HTTP/ { ok = ($2 == 200) } ok && tolower($1) == "docker-content-digest:" { print $2 }' } -# pgpsig_count: how many %PGPSIG% the upstream database carries +# put_manifest TAG FILE: write a manifest as an attacker with push rights would +put_manifest() { + curl -q -fsS --connect-timeout 5 --max-time 20 -X PUT -H 'Content-Type: application/vnd.oci.image.manifest.v1+json' \ + --data-binary "@$2" -o /dev/null "$BASE/manifests/$1" +} + +# tamper_canary: give the smallest package of the upstream test database, +# verify's canary, the signature layer of another package, as a writer with +# push rights could. The manifest it replaces is kept in canary-good.json, and +# $canary and $canary_tag name it. +tamper_canary() { + local other='beta_pc-2.0-1-x86_64.pkg.tar.zst' rc=0 + canary="$(smallest)" + canary_tag="$("$REPO_ROOT/client/pacman-oci" --tag "$canary")" + [ "$canary" != "$other" ] || other='gamma-3.0-2-any.pkg.tar.zst' + { + manifest "$canary_tag" > "$work/canary-good.json" && + manifest "$("$REPO_ROOT/client/pacman-oci" --tag "$other")" > "$work/other.json" && + jq -c --arg t "$canary.sig" --argjson l "$(jq -c --arg t "$other.sig" '.layers[] | select(.annotations["org.opencontainers.image.title"] == $t)' "$work/other.json")" \ + '.layers |= map(if .annotations["org.opencontainers.image.title"] == $t then (.digest = $l.digest | .size = $l.size) else . end)' \ + "$work/canary-good.json" > "$work/canary-bad.json" && + put_manifest "$canary_tag" "$work/canary-bad.json" + } || rc=$? + if [ "$rc" -eq 0 ]; then + ok "the canary $canary is given another package's signature layer, on the registry" + else + fail "the canary $canary is given another package's signature layer, on the registry" "status $rc" \ + "the canary must be a file the registry holds already: curl $BASE/manifests/$canary_tag" + fi +} + +# layer_of MANIFEST-FILE TITLE -> its digest +layer_of() { + jq -r --arg t "$2" '.layers[] | select(.annotations["org.opencontainers.image.title"] == $t) | .digest' "$1" +} + +# pgpsig_count [DIR]: how many %PGPSIG% the upstream database carries pgpsig_count() { - tar -xOzf "$work/upstream/x86_64/test/test.db" | awk '$0 == "%PGPSIG%" { n++ } END { print n + 0 }' + tar -xOzf "$work/upstream/${1:-x86_64/test}/${2:-test}.db" | awk '$0 == "%PGPSIG%" { n++ } END { print n + 0 }' } -# pgpsig_of FILE: the %PGPSIG% the upstream database carries for FILE +# pgpsig_of FILE: the %PGPSIG% the upstream test database carries for FILE pgpsig_of() { tar -xOzf "$work/upstream/x86_64/test/test.db" | awk -v f="$1" 'k == "F" { cur = $0 } k == "P" && cur == f { print } { k = ($0 == "%FILENAME%") ? "F" : ($0 == "%PGPSIG%") ? "P" : "" }' } +# smallest -> the filename of the smallest package in the upstream test database +smallest() { + tar -xOzf "$work/upstream/x86_64/test/test.db" | + awk 'k == "F" { f = $0 } k == "C" { print $0 "\t" f } { k = ($0 == "%FILENAME%") ? "F" : ($0 == "%CSIZE%") ? "C" : "" }' | + sort -n | awk -F '\t' 'NR == 1 { print $2 }' +} + logs_of() { local f for f in "$work/out/$1"/*.log; do + [ -e "$f" ] || continue diag "--- $(basename "$f")" while IFS= read -r l; do diag "$l"; done < <(tail -n 15 "$f") done } #---------------------------------------------------------------------------# -# 1. First sync: everything is new +# 0. The shard cap binds and says so +#---------------------------------------------------------------------------# +rc=0 +MAX_SHARDS=1 "$REPO_ROOT/scripts/plan" "$work/out/cap" "$work/targets/x86_64-test.conf" > "$work/out/cap.log" 2>&1 || rc=$? +check "MAX_SHARDS caps a target's shards" 1 "$(find "$work/out/cap/x86_64-test/shards" -name '*.tsv' | awk 'END { print NR + 0 }')" "read out/cap.log (plan exit $rc)" +if grep -q 'want 2 shard(s); MAX_SHARDS=1 caps them at 1' "$work/out/cap.log"; then + ok "the plan logs that the shard cap binds" +else + fail "the plan logs that the shard cap binds" "$(cat "$work/out/cap.log")" +fi + +#---------------------------------------------------------------------------# +# 1. First sync: everything is new. The test chain runs first, before its +# keyring_from target has a latest. #---------------------------------------------------------------------------# -build one check "the fixture's database carries a %PGPSIG% for each of its 3 packages, as core's does" 3 "$(pgpsig_count)" \ "tar -xOzf upstream/x86_64/test/test.db | grep -c PGPSIG; tests/fixtures/build-repo must run repo-add --include-sigs" -sync 1 +sync 1 x86_64-test x86_64-kr aarch64-port check "first sync: the plan finds work" pending "$(state_of 1 plan)" "tests/integration/10-pipeline.sh, then read out/1/plan.log" -check "first sync: the database is staged" staged "$(state_of 1 stage)" "read out/1/stage.log" +check "first sync: the database is staged" staged "$(state_of 1 stage)" "read out/1/stage-x86_64-test.log" if [ -s "$work/out/1/stage/x86_64-test/promoted.tsv" ]; then ok "first sync: the database is promoted" else fail "first sync: the database is promoted" "no promoted.tsv" logs_of 1 fi -check "verify compares each .sig pacman stored with its %PGPSIG%" 3 \ +check "each target gets shards of its own" \ + '[{"target":"aarch64-port","n":2},{"target":"x86_64-kr","n":1},{"target":"x86_64-test","n":2}]' \ + "$(jq -c '[.include[] | {target, n: (.shards | length)}] | sort_by(.target)' "$work/out/1/plan/targets.json")" \ + "jq . out/1/plan/targets.json" +check "no pending target is without files" '{"include":[]}' "$(jq -c . "$work/out/1/plan/pending.json")" "jq . out/1/plan/pending.json" +check "a shard holds the work of its own target only" 0 \ + "$(for t in x86_64-test x86_64-kr aarch64-port; do awk -F '\t' -v t="$t" '$1 != t' "$work/out/1/plan/$t"/shards/*.tsv; done | awk 'END { print NR + 0 }')" \ + "cut -f 1 out/1/plan/*/shards/*.tsv" +check "verify compares each .sig pacman stored with its %PGPSIG%, in both caches" 4 \ "$(sed -n 's/^\([0-9][0-9]*\) signature(s) compared$/\1/p' "$work/out/1/verify/verify-x86_64-test-0.log")" \ "read out/1/verify/verify-x86_64-test-0.log" +if grep -q 'keyring_from x86_64-kr has no latest yet; the refresh is skipped' "$work/out/1/verify/verify-x86_64-test-0.log"; then + ok "keyring_from with no latest yet: the refresh is skipped, with a warning, and verify goes on" +else + fail "keyring_from with no latest yet: the refresh is skipped, with a warning, and verify goes on" \ + "read out/1/verify/verify-x86_64-test-0.log" +fi +check "the keyring_from target is promoted" staged "$(state_of 1 stage x86_64-kr)" "read out/1/stage-x86_64-kr.log" +check "the keyring_from target's keyring package is a canary of its own" test-keyring \ + "$(cat "$work/out/1/stage/x86_64-kr/canaries.txt" 2> /dev/null || echo none)" "cat out/1/stage/x86_64-kr/canaries.txt" for f in 'alpha-1:1.0+r2-1-any.pkg.tar.zst' 'beta_pc-2.0-1-x86_64.pkg.tar.zst' 'gamma-3.0-1-any.pkg.tar.zst'; do tag="$("$REPO_ROOT/client/pacman-oci" --tag "$f")" @@ -205,6 +379,101 @@ check "latest and the dated tag name the same snapshot" "$latest1" "$(digest_of db_hex="$(sha256sum "$work/upstream/x86_64/test/test.db" | cut -d ' ' -f 1)" check "the snapshot is also tagged by its database's content" "$latest1" "$(digest_of "db-$db_hex")" "curl -I $BASE/manifests/db-$db_hex" +#---------------------------------------------------------------------------# +# 1b. Package manifests are the same bytes whoever pushes them: a second +# writer, in another repository, starting each shard from the other +# server, pushes manifests of the same digests. +#---------------------------------------------------------------------------# +alpha='alpha-1:1.0+r2-1-any.pkg.tar.zst' +manifest "$("$REPO_ROOT/client/pacman-oci" --tag "$alpha")" > "$work/alpha.json" +check "a package manifest's created time is its build date" 2023-11-14T22:13:20Z \ + "$(jq -r '.annotations["org.opencontainers.image.created"]' "$work/alpha.json")" "curl $BASE/manifests/ | jq .annotations" +check "a package manifest names no upstream server" 0 \ + "$(jq '[.annotations | keys[] | select(test("upstream"))] | length' "$work/alpha.json")" "curl $BASE/manifests/ | jq .annotations" +for s in "$work/out/1/plan/x86_64-test"/shards/*.tsv; do + i="$(basename "$s" .tsv)" + rc=0 + MIRROR_NAMESPACE=it/second SHARD_INDEX="$((i + 1))" "$REPO_ROOT/scripts/mirror-shard" "$s" "$work/out/second/$i" \ + > "$work/out/second-$i.log" 2>&1 || rc=$? +done +same=0 +differ="" +while IFS=$'\t' read -r _ f tag _; do + if [ -n "$(digest_of "$tag")" ] && [ "$(digest_of "$tag")" = "$(digest_of "$tag" it/second/x86_64/test)" ]; then + same=$((same + 1)) + else + differ="$differ $f" + fi +done < <(cat "$work/out/1/plan/x86_64-test"/shards/*.tsv) +check "two writers push byte-identical package manifests" "3" "$same" "differ:$differ; read out/second-*.log" +check "the second writer read the bytes from the other server" 1 \ + "$(cat "$work/out/second/0/results.tsv" "$work/out/1/results/x86_64-test/0/results.tsv" | awk -F '\t' '{ sub(/, pushed in.*/, "", $5); print $5 }' | sort -u | awk 'END { print (NR >= 2) ? 1 : 0 }')" \ + "cut -f 5 out/second/0/results.tsv out/1/results/x86_64-test/0/results.tsv" + +#---------------------------------------------------------------------------# +# 1c. Another arch, verified in the x86_64 image; a NUL-only package directory; +# upstream-refused packages promoted and listed +#---------------------------------------------------------------------------# +check "a database with a NUL-only package directory is planned" pending "$(state_of 1 plan aarch64-port)" "read out/1/plan.log" +check "the NUL-only package directory is named" ghost-1-1 "$(cat "$work/out/1/plan/aarch64-port/nul.txt" 2> /dev/null || echo none)" \ + "cat out/1/plan/aarch64-port/nul.txt" +check "the plan summary names it" 1 "$(awk -F '\t' '$1 == "aarch64-port" && $5 ~ /ghost-1-1/ { n++ } END { print n + 0 }' "$work/out/1/plan/summary.tsv")" \ + "cat out/1/plan/summary.tsv" +check "verify writes Architecture from the target" 1 "$(grep -c '^Architecture = aarch64$' "$work/out/1/verify/verify-aarch64-port-0.log")" \ + "read out/1/verify/verify-aarch64-port-0.log" +case "$(verdict_of 1 aarch64-port)" in + passed*'2 of them upstream-refused'*) ok "an aarch64 batch passes in the x86_64 image, its upstream-refused packages listed in the verdict" ;; + *) fail "an aarch64 batch passes in the x86_64 image, its upstream-refused packages listed in the verdict" "verdict: $(verdict_of 1 aarch64-port)" \ + "read out/1/verify/verify-aarch64-port-0.log" ;; +esac +check "the refusals are listed with their reasons" "expired-1-1-any.pkg.tar.zst sig-refused +unsigned-1-1-any.pkg.tar.zst sig-missing" \ + "$(awk -F '\t' '{ split($2, r, ":"); print $1 " " r[1] }' "$work/out/1/verify/refused-aarch64-port-0.tsv" 2> /dev/null | sort)" \ + "cat out/1/verify/refused-aarch64-port-0.tsv" +if [ -n "$(digest_of latest it/mirror/aarch64/port)" ]; then + ok "a target with upstream-refused packages is promoted" +else + fail "a target with upstream-refused packages is promoted" "read out/1/promote-aarch64-port.log" +fi +check "the report lists the upstream-refused packages" 2 \ + "$(grep -cE '^(expired|unsigned)-1-1-any\.pkg\.tar\.zst'$'\t''sig-' "$work/out/1/report.md")" "cat out/1/report.md" +check "the report has a row for every target" 3 "$(grep -cE '^\| (x86_64-test|x86_64-kr|aarch64-port) \| promoted \|' "$work/out/1/report.md")" \ + "cat out/1/report.md" + +#---------------------------------------------------------------------------# +# 1d. The same batch with no trust root at all: pacman refuses every package +# for its signature, and the mirror serves upstream's bytes for each. With +# no keyring package installed and no package accepted, nothing shows the +# trust root works, so nothing is excused. +#---------------------------------------------------------------------------# +mkdir -p "$work/targets-notrust" +grep -v '^keyring_' "$work/targets/aarch64-port.conf" > "$work/targets-notrust/aarch64-port.conf" +rc=0 +TARGETS_DIR="$work/targets-notrust" "$REPO_ROOT/scripts/verify-batch" "$work/out/1/stage" aarch64-port 0 "$work/out/notrust" \ + > "$work/out/notrust.log" 2>&1 || rc=$? +case "$rc:$(cut -f 3,4 "$work/out/notrust/verify-aarch64-port-0.tsv" 2> /dev/null)" in + 1:failed*'every package of the batch was refused, and no keyring package was installed'*) ok "a trust root that accepts nothing excuses nothing" ;; + *) fail "a trust root that accepts nothing excuses nothing" "exit $rc: $(cat "$work/out/notrust/verify-aarch64-port-0.tsv" 2> /dev/null)" \ + "read out/notrust/verify-aarch64-port-0.log" ;; +esac + +#---------------------------------------------------------------------------# +# 1e. A database with an unreadable member that is not NUL bytes is refused +#---------------------------------------------------------------------------# +build port-junk aarch64/port-junk port +mkdir -p "$work/targets-junk" +# shellcheck disable=SC2016 # $arch and $repo are the target file's, not the shell's +sed 's|/\$arch/\$repo$|/$arch/port-junk|; s|/\$arch/\./\$repo$|/$arch/./port-junk|' "$work/targets/aarch64-port.conf" > "$work/targets-junk/aarch64-port.conf" +rc=0 +TARGETS_DIR="$work/targets-junk" "$REPO_ROOT/scripts/plan" "$work/out/junk" > "$work/out/junk.log" 2>&1 || rc=$? +check "a database whose unreadable member is not NUL bytes is refused" error \ + "$(cat "$work/out/junk/aarch64-port/state" 2> /dev/null || echo "no state, plan exit $rc")" "read out/junk.log" +if grep -q 'junk-1-1/desc gives no record named after its directory' "$work/out/junk.log"; then + ok "the refusal names the member" +else + fail "the refusal names the member" "$(cat "$work/out/junk.log")" +fi + #---------------------------------------------------------------------------# # 2. Consumers: pacman through the wrapper, nothing else #---------------------------------------------------------------------------# @@ -227,9 +496,9 @@ printf '\n[test]\nServer = %s\n' "$server" >> /tmp/pacman.conf if [ -n "$extra" ]; then printf 'Server = %s\n' "$extra" >> /tmp/pacman.conf fi -pacman-key --init > /dev/null 2>&1 -pacman-key --add /in/test.asc > /dev/null 2>&1 -pacman-key --lsign-key "$(gpg --with-colons --show-keys /in/test.asc | awk -F : '$1 == "fpr" { print $10; exit }')" > /dev/null 2>&1 +pacman-key --init > /tmp/key.log 2>&1 +pacman-key --add /in/test.asc >> /tmp/key.log 2>&1 +pacman-key --lsign-key "$(gpg --with-colons --show-keys /in/test.asc | awk -F : '$1 == "fpr" && f == "" { f = $10 } END { print f }')" >> /tmp/key.log 2>&1 pac() { pacman --config /tmp/pacman.conf --noconfirm "$@"; } pac -Sy pac -S alpha beta_pc gamma @@ -257,7 +526,7 @@ consumer "no DownloadUser" "oci+http://127.0.0.1:$REG_PORT/$REPO" no consumer "a snapshot pinned by digest" "oci+http://127.0.0.1:$REG_PORT/$REPO@$latest1" yes consumer "a dated snapshot" "oci+http://127.0.0.1:$REG_PORT/$REPO:$day" yes consumer "the registry missing, upstream after it" "oci+http://127.0.0.1:$REG_PORT/it/nowhere/x86_64/test" yes \ - "http://127.0.0.1:$UP_PORT/x86_64/test" + "$UP/x86_64/test" #---------------------------------------------------------------------------# # 3. Nothing changed upstream: nothing to do @@ -265,15 +534,36 @@ consumer "the registry missing, upstream after it" "oci+http://127.0.0.1:$REG_PO sync 2 check "an unchanged upstream is current" current "$(state_of 2 plan)" "read out/2/plan.log" check "an unchanged upstream leaves latest where it was" "$latest1" "$(digest_of latest)" "curl -I $BASE/manifests/latest" +check "every target is current, and the report says so" "current current current 0" \ + "$(state_of 2 plan x86_64-test) $(state_of 2 plan x86_64-kr) $(state_of 2 plan aarch64-port) $(cat "$work/out/2/report.rc")" \ + "cat out/2/plan/summary.tsv out/2/report.md" #---------------------------------------------------------------------------# -# 4. One package updated: only it is mirrored, latest moves, the day's tag -# keeps the first snapshot +# 4. Two packages new, one of them signed by a key only the keyring package +# of x86_64-kr holds: only they are mirrored, verify installs that keyring +# package from kr's latest, latest moves, the day's tag keeps the first +# snapshot #---------------------------------------------------------------------------# build two sync 3 -check "an update is planned as one new file" 1 "$(awk 'END { print NR + 0 }' "$work/out/3/plan/x86_64-test/todo.tsv")" "read out/3/plan/x86_64-test/todo.tsv" -check "the update is promoted" staged "$(state_of 3 stage)" "read out/3/stage.log" +check "an update is planned as its new files only" 2 "$(awk 'END { print NR + 0 }' "$work/out/3/plan/x86_64-test/todo.tsv")" "read out/3/plan/x86_64-test/todo.tsv" +check "the update is staged" staged "$(state_of 3 stage)" "read out/3/stage-x86_64-test.log" +case "$(verdict_of 3)" in + *upstream-refused*) fail "a package signed by a key only keyring_from's keyring package holds passes, refused by nothing" \ + "verdict: $(verdict_of 3)" "the keyring package was not installed: read out/3/verify/verify-x86_64-test-0.log" ;; + passed*) ok "a package signed by a key only keyring_from's keyring package holds passes, refused by nothing" ;; + *) fail "a package signed by a key only keyring_from's keyring package holds passes, refused by nothing" "verdict: $(verdict_of 3)" \ + "read out/3/verify/verify-x86_64-test-0.log" ;; +esac +if grep -q '^== .* keyring: install test-keyring from \[kr\]$' "$work/out/3/verify/verify-x86_64-test-0.log"; then + ok "verify installs the keyring package from keyring_from's repository" +else + fail "verify installs the keyring package from keyring_from's repository" "read out/3/verify/verify-x86_64-test-0.log" +fi +kr_latest="$(digest_of latest it/mirror/x86_64/kr)" +check "verify reads keyring_from's repository pinned by the digest its latest names" 1 \ + "$(grep -cxF "Server = oci+http://127.0.0.1:$REG_PORT/it/mirror/x86_64/kr@$kr_latest" "$work/out/3/verify/verify-x86_64-test-0.log")" \ + "read out/3/verify/verify-x86_64-test-0.log" latest3="$(digest_of latest)" if [ -n "$latest3" ] && [ "$latest3" != "$latest1" ]; then ok "latest moved to the new snapshot" @@ -294,13 +584,55 @@ else fi check "latest stays on the newer snapshot" "$latest3" "$(digest_of latest)" "curl -I $BASE/manifests/latest" +#---------------------------------------------------------------------------# +# 4b. Upstream signs a file again, its bytes unchanged: the file is planned, +# and its signature layer is repaired under the same tag +#---------------------------------------------------------------------------# +alpha_tag="$("$REPO_ROOT/client/pacman-oci" --tag "$alpha")" +alpha_pkg_before="$(manifest "$alpha_tag" | jq -r --arg t "$alpha" '.layers[] | select(.annotations["org.opencontainers.image.title"] == $t) | .digest')" +build resigned +sync 4 +check "a file signed again with the same bytes is planned" "$alpha" "$(cut -f 2 "$work/out/4/plan/x86_64-test/todo.tsv")" \ + "read out/4/plan/x86_64-test/todo.tsv" +check "its signature layer is pushed again" pushed "$(cat "$work/out/4/results/x86_64-test"/*/results.tsv | awk -F '\t' -v f="$alpha" '$2 == f { print $3 }')" \ + "cat out/4/results/x86_64-test/*/results.tsv" +manifest "$alpha_tag" > "$work/alpha.json" +check "the repaired signature layer is the new %PGPSIG%" "sha256:$(pgpsig_of "$alpha" | base64 -d | sha256sum | cut -d ' ' -f 1)" \ + "$(layer_of "$work/alpha.json" "$alpha.sig")" "curl $BASE/manifests/$alpha_tag" +check "its package layer is the same blob" "$alpha_pkg_before" "$(layer_of "$work/alpha.json" "$alpha")" "curl $BASE/manifests/$alpha_tag" +latest4="$(digest_of latest)" +if [ -n "$latest4" ] && [ "$latest4" != "$latest3" ]; then + ok "the snapshot with the new signature is promoted" +else + fail "the snapshot with the new signature is promoted" "latest is $latest4" "read out/4/verify/verify-x86_64-test-0.log" +fi + +#---------------------------------------------------------------------------# +# 4c. A signature layer that is not the %PGPSIG%, written on the registry +# behind the mirror's back to a file plan skips: verify fails the batch, +# and nothing excuses it as upstream's +#---------------------------------------------------------------------------# +build four +tamper_canary +sync 5 +check "the smallest package, the canary, is not new in this update" "theta-1-1-any.pkg.tar.zst" "$(cut -f 2 "$work/out/5/plan/x86_64-test/todo.tsv")" \ + "read out/5/plan/x86_64-test/todo.tsv; the canary is $canary" +case "$(verdict_of 5)" in + failed*"$canary: its .sig is sha256"*) ok "a signature layer that is not the %PGPSIG% fails verify, and is named" ;; + *) fail "a signature layer that is not the %PGPSIG% fails verify, and is named" "verdict: $(verdict_of 5)" "read out/5/verify/verify-x86_64-test-0.log" ;; +esac +check "a batch with a bad signature layer is never promoted" "$latest4" "$(digest_of latest)" "curl -I $BASE/manifests/latest" +put_manifest "$canary_tag" "$work/canary-good.json" || diag "the canary manifest could not be written back" +check "the canary's manifest is restored" "sha256:$(sha256sum "$work/canary-good.json" | cut -d ' ' -f 1)" "$(digest_of "$canary_tag")" \ + "curl -I $BASE/manifests/$canary_tag" + #---------------------------------------------------------------------------# # 5. Upstream goes back to an older database: refused, latest stays #---------------------------------------------------------------------------# build one -sync 4 -check "an older upstream database is refused as behind" behind "$(state_of 4 plan)" "read out/4/plan.log" -check "latest does not move backwards" "$latest3" "$(digest_of latest)" "curl -I $BASE/manifests/latest" +sync 6 +check "an older upstream database is refused as behind" behind "$(state_of 6 plan)" "read out/6/plan.log" +check "latest does not move backwards" "$latest4" "$(digest_of latest)" "curl -I $BASE/manifests/latest" #---------------------------------------------------------------------------# # 6. A file changes under its own name: held, nothing overwritten @@ -308,10 +640,10 @@ check "latest does not move backwards" "$latest3" "$(digest_of latest)" "curl -I build replaced beta_tag="$("$REPO_ROOT/client/pacman-oci" --tag 'beta_pc-2.0-1-x86_64.pkg.tar.zst')" beta_before="$(digest_of "$beta_tag")" -sync 5 -check "a file whose bytes changed under its own name holds the target" replaced "$(state_of 5 stage)" "read out/5/stage.log" +sync 7 +check "a file whose bytes changed under its own name holds the target" replaced "$(state_of 7 stage)" "read out/7/stage-x86_64-test.log" check "its tag is not overwritten" "$beta_before" "$(digest_of "$beta_tag")" "curl -I $BASE/manifests/$beta_tag" -check "latest stays" "$latest3" "$(digest_of latest)" "curl -I $BASE/manifests/latest" +check "latest stays" "$latest4" "$(digest_of latest)" "curl -I $BASE/manifests/latest" #---------------------------------------------------------------------------# # 7. A new file whose tag already holds other bytes: held, never pushed over @@ -325,56 +657,63 @@ ORAS_BIN="$("$REPO_ROOT/scripts/install-oras" "$work/bin")" (cd "$work/bogus" && timeout 120 "$ORAS_BIN" push --plain-http "$MIRROR_REGISTRY/$REPO:$delta_tag" \ "./$delta:application/vnd.pkgforge.archlinux.pkg.v1" > /dev/null) bogus="$(digest_of "$delta_tag")" -sync 6 -check "a new file whose tag already holds other bytes holds the target" replaced "$(state_of 6 stage)" "read out/6/stage.log" +sync 8 +check "a new file whose tag already holds other bytes holds the target" replaced "$(state_of 8 stage)" "read out/8/stage-x86_64-test.log" check "the conflicting tag is not overwritten" "$bogus" "$(digest_of "$delta_tag")" "curl -I $BASE/manifests/$delta_tag" -check "latest stays through a conflict" "$latest3" "$(digest_of latest)" "curl -I $BASE/manifests/latest" +check "latest stays through a conflict" "$latest4" "$(digest_of latest)" "curl -I $BASE/manifests/latest" #---------------------------------------------------------------------------# # 8. Two db_servers that disagree: each is ahead of the other on one package # and behind on another, and neither is behind what is promoted. Nothing # says which is newer, so the target is held, not guessed. #---------------------------------------------------------------------------# -build ahead-alpha test -build ahead-gamma test-b +build ahead-alpha +build ahead-gamma x86_64/test-b mkdir -p "$work/targets-two" +cp "$work/targets/x86_64-kr.conf" "$work/targets-two/" { grep -v '^db_server' "$work/targets/x86_64-test.conf" # shellcheck disable=SC2016 # $arch and $repo are the target file's, not the shell's - printf 'db_server = http://127.0.0.1:%s/$arch/$repo\ndb_server = http://127.0.0.1:%s/$arch/test-b\n' "$UP_PORT" "$UP_PORT" + printf 'db_server = %s/$arch/$repo\ndb_server = %s/$arch/test-b\n' "$UP" "$UP" } > "$work/targets-two/x86_64-test.conf" rc=0 -TARGETS_DIR="$work/targets-two" "$REPO_ROOT/scripts/plan" "$work/out/held" > "$work/out/held.log" 2>&1 || rc=$? +TARGETS_DIR="$work/targets-two" "$REPO_ROOT/scripts/plan" "$work/out/held" "$work/targets-two/x86_64-test.conf" > "$work/out/held.log" 2>&1 || rc=$? check "db_servers that disagree hold the target" held \ "$(cat "$work/out/held/x86_64-test/state" 2> /dev/null || echo "no state, plan exit $rc")" "read out/held.log" #---------------------------------------------------------------------------# -# 9. A package signed by a key nobody trusts: mirrored, staged, refused by -# pacman in verify, and never promoted. +# 9. A package signed by a key nobody trusts: pacman refuses it, from the +# mirror as from upstream, and the mirror serves upstream's bytes and +# signature. It is upstream-refused: listed, and the snapshot promoted. #---------------------------------------------------------------------------# build untrusted -sync 7 -check "a package with an untrusted signature is staged" staged "$(state_of 7 stage)" "read out/7/stage.log" -verdict="$(cut -f 3,4 "$work/out/7/verify/verify-x86_64-test-0.tsv" 2> /dev/null || echo 'no verdict')" -case "$verdict" in - failed*epsilon*) ok "verify fails it, and names the package" ;; - *) fail "verify fails it, and names the package" "verdict: $verdict" "read out/7/verify/verify-x86_64-test-0.log" ;; +sync 9 +check "a package with an untrusted signature is staged" staged "$(state_of 9 stage)" "read out/9/stage-x86_64-test.log" +case "$(verdict_of 9)" in + passed*'1 of them upstream-refused'*epsilon*) ok "verify lists it as upstream-refused, and passes the batch" ;; + *) fail "verify lists it as upstream-refused, and passes the batch" "verdict: $(verdict_of 9)" "read out/9/verify/verify-x86_64-test-0.log" ;; esac -check "an untrusted package is never promoted" "$latest3" "$(digest_of latest)" "curl -I $BASE/manifests/latest" +latest9="$(digest_of latest)" +if [ -n "$latest9" ] && [ "$latest9" != "$latest4" ]; then + ok "a snapshot whose only fault is upstream's is promoted" +else + fail "a snapshot whose only fault is upstream's is promoted" "latest is $latest9" "read out/9/promote-x86_64-test.log" +fi #---------------------------------------------------------------------------# -# 9b. A database with no %PGPSIG%, as most of ArchPOWER's: the signature of a +# 9b. A database with no %PGPSIG%, as most of archriscv's: the signature of a # new package comes from upstream's detached .sig, byte for byte, and # pacman verifies against it through the wrapper. #---------------------------------------------------------------------------# build nopgpsig check "the fixture's database carries no %PGPSIG%" 0 "$(pgpsig_count)" "tar -xOzf upstream/x86_64/test/test.db | grep -c PGPSIG" -sync 8 -check "a database without %PGPSIG% stages" staged "$(state_of 8 stage)" "read out/8/stage.log" +sync 10 +check "a database without %PGPSIG% stages" staged "$(state_of 10 stage)" "read out/10/stage-x86_64-test.log" zeta='zeta-1-1-any.pkg.tar.zst' zeta_sig="$(manifest "$("$REPO_ROOT/client/pacman-oci" --tag "$zeta")" | jq -r --arg t "$zeta.sig" '.layers[] | select(.annotations["org.opencontainers.image.title"] == $t) | .digest')" -check "its signature layer is upstream's detached .sig" "sha256:$(sha256sum "$work/upstream/x86_64/test/$zeta.sig" | cut -d ' ' -f 1)" "$zeta_sig" "curl $BASE/manifests/" -check "pacman verifies it against that .sig" passed "$(cut -f 3 "$work/out/8/verify/verify-x86_64-test-0.tsv" 2> /dev/null || echo 'no verdict')" "read out/8/verify/verify-x86_64-test-0.log" +check "its signature layer is upstream's detached .sig" "sha256:$(sha256sum "$work/upstream/x86_64/test/$zeta.sig" | cut -d ' ' -f 1)" "$zeta_sig" "curl $BASE/manifests/" +check "pacman verifies it against that .sig" passed "$(verdict_of 10 | cut -f 1)" "read out/10/verify/verify-x86_64-test-0.log" +latest10="$(digest_of latest)" #---------------------------------------------------------------------------# # 9c. A lagging db_server that only drops a package: its database differs from @@ -382,18 +721,37 @@ check "pacman verifies it against that .sig" passed "$(cut -f 3 "$work/out/8/ver # it is newer. Listed first, it would win; the promoted database, still # served by the other db_server, is kept instead. #---------------------------------------------------------------------------# -build two test-b +build two x86_64/test-b mkdir -p "$work/targets-lag" +cp "$work/targets/x86_64-kr.conf" "$work/targets-lag/" { grep -v '^db_server' "$work/targets/x86_64-test.conf" # shellcheck disable=SC2016 # $arch and $repo are the target file's, not the shell's - printf 'db_server = http://127.0.0.1:%s/$arch/test-b\ndb_server = http://127.0.0.1:%s/$arch/$repo\n' "$UP_PORT" "$UP_PORT" + printf 'db_server = %s/$arch/test-b\ndb_server = %s/$arch/$repo\n' "$UP" "$UP" } > "$work/targets-lag/x86_64-test.conf" rc=0 -TARGETS_DIR="$work/targets-lag" "$REPO_ROOT/scripts/plan" "$work/out/lag" > "$work/out/lag.log" 2>&1 || rc=$? +TARGETS_DIR="$work/targets-lag" "$REPO_ROOT/scripts/plan" "$work/out/lag" "$work/targets-lag/x86_64-test.conf" > "$work/out/lag.log" 2>&1 || rc=$? check "a lagging db_server that only drops a package does not move latest" current \ "$(cat "$work/out/lag/x86_64-test/state" 2> /dev/null || echo "no state, plan exit $rc")" "read out/lag.log" +#---------------------------------------------------------------------------# +# 9d. With no %PGPSIG%, a signature layer that is not upstream's .sig: pacman +# refuses the package for its signature, and the account of the refusal +# finds the mirror at fault, not upstream: the batch fails. +#---------------------------------------------------------------------------# +build nopgpsig-four +tamper_canary +sync 11 +check "the canary is not new in this update either" "theta-1-1-any.pkg.tar.zst" "$(cut -f 2 "$work/out/11/plan/x86_64-test/todo.tsv")" \ + "read out/11/plan/x86_64-test/todo.tsv; the canary is $canary" +case "$(verdict_of 11)" in + failed*"${canary%%-[0-9]*} (sig-refused"*) ok "a signature layer that is not upstream's .sig is not excused as upstream-refused" ;; + *) fail "a signature layer that is not upstream's .sig is not excused as upstream-refused" "verdict: $(verdict_of 11)" \ + "read out/11/verify/verify-x86_64-test-0.log" ;; +esac +check "it is never promoted" "$latest10" "$(digest_of latest)" "curl -I $BASE/manifests/latest" +put_manifest "$canary_tag" "$work/canary-good.json" || diag "the canary manifest could not be written back" + #---------------------------------------------------------------------------# # 10. What the wrapper must refuse, served by a plain web server standing in # for a registry that lies: a manifest whose bytes are not the digest the @@ -434,7 +792,7 @@ refuses "a package manifest with no signature layer, loudly" \ awk 'BEGIN { printf "{\"schemaVersion\":2,\"layers\":["; for (i = 0; i < 1000000; i++) printf "%s1", (i ? "," : ""); print "]}" }' > "$fake/huge" started_at="$(date +%s)" refuses "a manifest over 64 KiB, quickly" "oci+http://127.0.0.1:$UP_PORT/fake/repo:huge/test.db" "Maximum file size exceeded" -check "the oversized manifest is given up within 20 seconds" 1 "$(( $(date +%s) - started_at < 20 ? 1 : 0 ))" "time client/pacman-oci out.part oci+http://.../fake/repo:huge/test.db" +check "the oversized manifest is given up within 20 seconds" 1 "$(($(date +%s) - started_at < 20 ? 1 : 0))" "time client/pacman-oci out.part oci+http://.../fake/repo:huge/test.db" # serve_layer TAG TITLE BYTES_FILE DIGEST SIZE: a one-layer manifest at TAG # whose descriptor says DIGEST and SIZE, and BYTES_FILE served as that blob diff --git a/tests/static/60-targets.sh b/tests/static/60-targets.sh index 33df79f..afbda9b 100644 --- a/tests/static/60-targets.sh +++ b/tests/static/60-targets.sh @@ -1,13 +1,34 @@ #!/usr/bin/env bash # # Every target file parses, names a database server and a package server, and -# pins its verify image by digest with a tag Renovate can follow. +# pins its images by digest with a tag Renovate can follow. Every target +# verifies in the one x86_64 image; a trust root brought in from another image +# comes from pkgforge-dev's own Arch images only. The sync workflow can upload +# the plan of every enabled target. set -euo pipefail # shellcheck source=../lib/harness.sh . "$REPO_ROOT/tests/lib/harness.sh" cd "$REPO_ROOT" + +# get CONF KEY -> the value, read the way the scripts read it +get() { + SCRIPT_NAME=target bash -c '. scripts/lib.sh; target_get "$1" "$2"' _ "$1" "$2" +} + +# tag_marker CONF KEY -> the tag on the "# tag:" line right before KEY's line +tag_marker() { + awk -v k="$2" ' + { line = $0; key = line; sub(/[[:space:]]*=.*/, "", key) } + key == k && line ~ /=/ { print prev; found = 1 } + { prev = line } + END { if (!found) print "" } + ' "$1" | sed -n 's/^#[[:space:]]*tag:[[:space:]]*\([^[:space:]]*\)[[:space:]]*$/\1/p' +} + found=0 +enabled=0 +verify_images="" for conf in targets/*.conf; do [ -e "$conf" ] || continue found=$((found + 1)) @@ -17,16 +38,53 @@ for conf in targets/*.conf; do else fail "$conf parses" "$out" "reproduce: bash -c '. scripts/lib.sh; target_check $conf'" fi + if SCRIPT_NAME=target bash -c '. scripts/lib.sh; target_enabled "$1"' _ "$conf"; then + enabled=$((enabled + 1)) + fi - image="$(SCRIPT_NAME=target bash -c '. scripts/lib.sh; target_get "$1" verify_image' _ "$conf")" - if [ -z "$image" ]; then - ok "$conf names no verify image, so it is never verified in a container" - elif printf '%s\n' "$image" | grep -Eq '^[a-z0-9.-]+(/[a-z0-9._-]+)+:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}$'; then - ok "$conf pins its verify image as name:tag@sha256" - else - fail "$conf pins its verify image as name:tag@sha256" "verify_image = $image" \ - "a digest alone gives Renovate no tag to follow, a tag alone is not a pin" \ - "reproduce: grep verify_image $conf" + for key in verify_image keyring_image; do + image="$(get "$conf" "$key")" + if [ -z "$image" ]; then + if [ "$key" = verify_image ]; then + fail "$conf names a verify image" "no verify_image" "reproduce: grep verify_image $conf" + fi + continue + fi + if printf '%s\n' "$image" | grep -Eq '^[a-z0-9.-]+(/[a-z0-9._-]+)+:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}$'; then + ok "$conf pins its $key as name:tag@sha256" + else + fail "$conf pins its $key as name:tag@sha256" "$key = $image" \ + "a digest alone gives Renovate no tag to follow, a tag alone is not a pin" \ + "reproduce: grep $key $conf" + fi + tag="${image%@*}" + tag="${tag##*:}" + check "$conf: the line before $key says '# tag: $tag', for Renovate" "$tag" "$(tag_marker "$conf" "$key")" \ + "grep -B1 '^$key' $conf" + done + verify_images="$verify_images$(get "$conf" verify_image) $(get "$conf" verify_platform)"$'\n' + + image="$(get "$conf" keyring_image)" + if [ -n "$image" ]; then + case "$image" in + ghcr.io/pkgforge-dev/archlinux:*) ok "$conf: keyring_image is one of pkgforge-dev's own Arch images" ;; + *) fail "$conf: keyring_image is one of pkgforge-dev's own Arch images" "keyring_image = $image" \ + "the trust root of verify comes from ghcr.io/pkgforge-dev/archlinux only" "reproduce: grep keyring_image $conf" ;; + esac + if [ -n "$(get "$conf" keyring_platform)" ]; then + ok "$conf: keyring_image comes with keyring_platform" + else + fail "$conf: keyring_image comes with keyring_platform" "no keyring_platform" "reproduce: grep keyring_ $conf" + fi + fi + + from="$(get "$conf" keyring_from)" + if [ -n "$from" ]; then + if [ -f "targets/$from.conf" ] && SCRIPT_NAME=target bash -c '. scripts/lib.sh; target_enabled "$1"' _ "targets/$from.conf"; then + ok "$conf: keyring_from $from is an enabled target" + else + fail "$conf: keyring_from $from is an enabled target" "targets/$from.conf is missing or not enabled" "reproduce: ls targets/$from.conf" + fi fi for key in server db_server; do @@ -42,4 +100,23 @@ if [ "$found" -eq 0 ]; then fail "there is a target to mirror" "no targets/*.conf" "reproduce: ls targets" fi +# One verify image for every target: x86_64 Arch, Architecture set from the +# target, so Renovate moves them all in one change and no target needs QEMU. +distinct="$(printf '%s' "$verify_images" | sort -u)" +if [ "$(printf '%s\n' "$distinct" | awk 'NF { n++ } END { print n + 0 }')" -eq 1 ] && [ "${distinct##* }" = linux/amd64 ]; then + ok "every target verifies in the one image, on linux/amd64" +else + fail "every target verifies in the one image, on linux/amd64" "$distinct" "reproduce: grep -h -A1 '^verify_image' targets/*.conf | sort -u" +fi + +# The plan job uploads each pending target's plan from a slot step of its +# own; there must be a slot for every enabled target. +slots="$(grep -oE 'steps\.plan\.outputs\.slot[0-9]+' .github/workflows/sync.yml | sort -u | awk 'END { print NR + 0 }')" +if [ "$slots" -ge "$enabled" ]; then + ok "sync.yml has a plan upload slot for each of the $enabled enabled target(s) ($slots)" +else + fail "sync.yml has a plan upload slot for each of the $enabled enabled target(s)" "$slots slot(s)" \ + "add slot steps to the plan job" "reproduce: grep -c 'outputs.slot' .github/workflows/sync.yml" +fi + summary diff --git a/tests/unit/40-lib.sh b/tests/unit/40-lib.sh index ba89c71..6e7fa72 100644 --- a/tests/unit/40-lib.sh +++ b/tests/unit/40-lib.sh @@ -66,6 +66,44 @@ refused "db_signed that is not yes or no" "$work/t5/x86_64-core.conf" mkdir -p "$work/t6" && { cat "$good"; echo 'not a key value line'; } > "$work/t6/x86_64-core.conf" refused "a line that is not key = value" "$work/t6/x86_64-core.conf" +#---------------------------------------------------------------------------# +# The keys that set verify's trust root +#---------------------------------------------------------------------------# +IMG='ghcr.io/pkgforge-dev/archlinux:aarch64@sha256:b3610e2db83532a105802cf05a40ee7273f58231d5d0e507138f0cd9adef2e27' +# trust NAME LINES...: a target file of x86_64 extra, with x86_64-core beside it +trust() { + local d="$work/trust-$1" + shift + mkdir -p "$d" + cp "$good" "$d/x86_64-core.conf" + { sed 's/^repo = core$/repo = extra/; /^keyring/d' "$good"; printf '%s\n' "$@"; } > "$d/x86_64-extra.conf" + printf '%s\n' "$d/x86_64-extra.conf" +} +accepted() { + local label="$1" file="$2" rc=0 + in_lib target_check "$file" > "$work/accepted.err" 2>&1 || rc=$? + check "accepted: $label" 0 "$rc" "bash -c '. scripts/lib.sh; target_check $file' ($(cat "$work/accepted.err"))" +} +accepted "keyring as a list, keyring_from a target of the same arch" \ + "$(trust ok 'keyring = archlinux-keyring archlinux-lcpu-keyring' 'keyring_from = x86_64-core')" +accepted "keyring_image pinned by digest, with keyring_platform" \ + "$(trust image 'keyring = archlinuxarm-keyring' "keyring_image = $IMG" 'keyring_platform = linux/arm64')" +check "target_keyrings lists each keyring package" "$(printf '%s\n' archlinux-keyring archlinux-lcpu-keyring)" \ + "$(in_lib target_keyrings "$work/trust-ok/x86_64-extra.conf")" "bash -c '. scripts/lib.sh; target_keyrings FILE'" +refused "a keyring name with a leading -" "$(trust dash 'keyring = -rf archlinux-keyring')" +refused "a keyring name with a /" "$(trust slash 'keyring = ../archlinux-keyring')" +refused "a keyring name that is a glob" "$(trust glob 'keyring = *')" +refused "keyring_from with no keyring" "$(trust nokr 'keyring_from = x86_64-core')" +refused "keyring_from naming the target itself" "$(trust self 'keyring = a' 'keyring_from = x86_64-extra')" +arch_conf="$(trust arch 'keyring = a' 'keyring_from = aarch64-core')" +sed 's/^arch = x86_64$/arch = aarch64/' "$good" > "$(dirname "$arch_conf")/aarch64-core.conf" +refused "keyring_from naming a target of another arch" "$arch_conf" +refused "keyring_from naming a target with no file" "$(trust none 'keyring = a' 'keyring_from = x86_64-nothere')" +refused "keyring_image with a tag and no digest" "$(trust tag 'keyring_image = ghcr.io/pkgforge-dev/archlinux:aarch64' 'keyring_platform = linux/arm64')" +refused "keyring_image with no keyring_platform" "$(trust noplat "keyring_image = $IMG")" +refused "keyring_platform with no keyring_image" "$(trust noimg 'keyring_platform = linux/arm64')" +refused "keyring_platform that is not linux/" "$(trust plat "keyring_image = $IMG" 'keyring_platform = arm64')" + #---------------------------------------------------------------------------# # Database fields #---------------------------------------------------------------------------# @@ -104,6 +142,62 @@ rc=0 in_lib db_validate "$work/rec.tsv" > /dev/null 2>&1 || rc=$? check "db field check refuses a filename listed twice" 1 "$rc" "bash -c '. scripts/lib.sh; db_validate RECORDS'" +#---------------------------------------------------------------------------# +# A whole database: one record per package directory, and a directory whose +# desc holds only NUL bytes passed over and named. db_records reads with GNU +# tar, which every runner and Arch has; another tar skips these. +#---------------------------------------------------------------------------# +# mkdb NAME [ghost|empty|junk|twice ...] -> a gzip database with packages a and +# b, and the extra members asked for +mkdb() { + local name="$1" d="$work/db-$1" p extra + shift + rm -rf "$d" + for p in a b; do + mkdir -p "$d/$p-1-1" + printf '%%FILENAME%%\n%s-1-1-any.pkg.tar.zst\n\n%%NAME%%\n%s\n\n%%VERSION%%\n1-1\n\n%%ARCH%%\nany\n\n%%SHA256SUM%%\n%s\n\n%%CSIZE%%\n10\n\n%%BUILDDATE%%\n1700000000\n\n' \ + "$p" "$p" "$SHA" > "$d/$p-1-1/desc" + done + for extra in "$@"; do + case "$extra" in + ghost) mkdir -p "$d/ghost-1-1" && head -c 1271 /dev/zero > "$d/ghost-1-1/desc" ;; + empty) mkdir -p "$d/empty-1-1" && : > "$d/empty-1-1/desc" ;; + junk) mkdir -p "$d/junk-1-1" && printf 'bytes that are no record\n' > "$d/junk-1-1/desc" ;; + twice) printf '%%FILENAME%%\nc-1-1-any.pkg.tar.zst\n\n%%NAME%%\nc\n\n%%VERSION%%\n1-1\n\n%%ARCH%%\nany\n\n%%SHA256SUM%%\n%s\n\n%%CSIZE%%\n10\n\n%%BUILDDATE%%\n1700000000\n\n' "$SHA" >> "$d/b-1-1/desc" ;; + esac + done + (cd "$d" && tar -czf "$work/$name.db" -- *) + printf '%s\n' "$work/$name.db" +} +tar_version="$(tar --version 2>&1 | sed -n 1p)" +if case "$tar_version" in *'GNU tar'*) true ;; *) false ;; esac then + db="$(mkdb whole)" + rc=0 + why="$(in_lib db_check "$db" "$work/whole.tsv" 2>&1)" || rc=$? + check "db_check accepts a whole database" "0 2" "$rc $(awk 'END { print NR }' "$work/whole.tsv")" "bash -c '. scripts/lib.sh; db_check DB OUT' ($why)" + db="$(mkdb nul ghost empty)" + rc=0 + why="$(in_lib db_check "$db" "$work/nul.tsv" 2>&1)" || rc=$? + check "db_check accepts package directories whose desc is NUL bytes or empty, and names them" "0 2 empty-1-1 ghost-1-1" \ + "$rc $(awk 'END { print NR }' "$work/nul.tsv") $(sort "$work/nul.tsv.nul" | tr '\n' ' ' | sed 's/ $//')" "bash -c '. scripts/lib.sh; db_check DB OUT' ($why)" + db="$(mkdb junk ghost junk)" + rc=0 + why="$(in_lib db_check "$db" "$work/junk.tsv" 2>&1)" || rc=$? + case "$rc:$why" in + 1:junk-1-1/desc*) ok "db_check refuses an unreadable member that holds other bytes than NUL, and names it" ;; + *) fail "db_check refuses an unreadable member that holds other bytes than NUL, and names it" "status $rc: $why" "bash -c '. scripts/lib.sh; db_check DB OUT'" ;; + esac + db="$(mkdb twice twice)" + rc=0 + why="$(in_lib db_check "$db" "$work/twice.tsv" 2>&1)" || rc=$? + case "$rc:$why" in + 1:*"2 package directories, 3 readable records"*) ok "db_check refuses a desc that holds two records" ;; + *) fail "db_check refuses a desc that holds two records" "status $rc: $why" "bash -c '. scripts/lib.sh; db_check DB OUT'" ;; + esac +else + ok "skipped: db_check reads with GNU tar, and this tar is $tar_version" +fi + #---------------------------------------------------------------------------# # Behind: compared per package by build date. Fixtures are Arch's archived # core.db of four days, reduced to name and build date: 10-02 and 10-03 share @@ -219,4 +313,121 @@ check "an item that left no result is counted as failed" 1 "$bad" "lib.sh reconc check "every work line has exactly one result line" 3 "$(awk 'END { print NR }' "$work/all.tsv")" "lib.sh reconcile_results" check "the missing one is named, with its target" "t b.pkg.tar.zst failed" "$(awk -F '\t' '$3 == "failed" { print $1 " " $2 " " $3 }' "$work/all.tsv")" "lib.sh reconcile_results" +#---------------------------------------------------------------------------# +# How work is split: shards per target, verify batches per target +#---------------------------------------------------------------------------# +check "shards: by files" "2 2" "$(in_lib shard_count 3 1000 2 536870912 32)" "lib.sh shard_count" +check "shards: x86_64 extra's seed wants 216 by bytes and gets MAX_SHARDS" "32 216" \ + "$(in_lib shard_count 15041 115541940123 400 536870912 32)" "lib.sh shard_count" +check "shards: never more than files" "1 3" "$(in_lib shard_count 1 1610612736 400 536870912 32)" "lib.sh shard_count" +check "shards: no file, no shard" "0 0" "$(in_lib shard_count 0 0 400 536870912 32)" "lib.sh shard_count" +check "batches: under the cap, the limits stay" "1 400 2147483648 1" "$(in_lib batch_count 3 1000 400 2147483648 40)" "lib.sh batch_count" +read -r k flimit blimit wanted <<< "$(in_lib batch_count 15041 115541940123 400 2147483648 40)" +if [ "$wanted" -eq 54 ] && [ "$k" -le 40 ] && [ "$k" -ge 36 ] && [ "$flimit" -gt 400 ] && [ "$blimit" -gt 2147483648 ]; then + ok "batches: x86_64 extra's seed wants 54, both limits grow, and it gets 36 to 40 ($k, $flimit files, $blimit bytes)" +else + fail "batches: x86_64 extra's seed wants 54, both limits grow, and it gets 36 to 40" "$k $flimit $blimit $wanted" "lib.sh batch_count" +fi +check "a build date as ISO 8601 UTC" 2023-11-14T22:13:20Z "$(in_lib iso_of_epoch 1700000000)" "lib.sh iso_of_epoch" +rc=0 +in_lib iso_of_epoch 17x > /dev/null 2>&1 || rc=$? +check "a build date that is not a number is refused" 1 "$rc" "lib.sh iso_of_epoch" + +#---------------------------------------------------------------------------# +# Why pacman refused one package. The logs are pacman 7.1's own words, +# measured in the integration suite on 2026-10-14. +#---------------------------------------------------------------------------# +reason() { + printf '%s\n' "$2" > "$work/one.log" + in_lib refusal_reason "$work/one.log" "${3:-0}" "${4:-0}" | cut -f 1 > "$work/reason" + check "refusal: $1" "$5" "$(cat "$work/reason")" "lib.sh refusal_reason" +} +reason "a primary key that expired is a signature reason" \ + "$(printf '%s\n' 'checking keyring...' 'downloading required keys...' \ + ':: Import PGP key 2CD2A1B4BDFA4272, "x"? [Y/n] error: key "2CD2A1B4BDFA4272" could not be looked up remotely' \ + 'error: required key missing from keyring' 'error: failed to commit transaction (unexpected error)')" 0 0 sig-refused +reason "a signature that is not valid is a signature reason" \ + "$(printf '%s\n' 'error: alpha: signature from "mirror-archlinux test " is invalid' \ + ':: File /var/cache/pacman/pkg/alpha-1-1-any.pkg.tar.zst is corrupted (invalid or corrupted package (PGP signature)).' \ + 'error: failed to commit transaction (invalid or corrupted package (PGP signature))')" 0 0 sig-refused +reason "no .sig served for a package that is served is a missing signature" \ + "$(printf '%s\n' 'pacman-oci: unsigned-1-1-any.pkg.tar.zst.sig: no signature layer' 'error: failed to commit transaction (failed to retrieve some files)')" 0 1 sig-missing +reason "pacman's own words for a missing signature" "error: unsigned: missing required signature" 0 0 sig-missing +reason "a package the wrapper does not serve is not excused" \ + "$(printf '%s\n' 'error: failed to commit transaction (failed to retrieve some files)')" 1 1 other +reason "a checksum is not a signature reason" \ + "$(printf '%s\n' 'error: alpha: signature from "x" is unknown trust' ':: File x is corrupted (invalid or corrupted package (checksum)).')" 0 0 other +reason "an architecture is not a signature reason" \ + "$(printf '%s\n' ':: package port-1-1-aarch64 does not have a valid architecture' 'error: failed to prepare transaction (package architecture is not valid)')" 0 0 other +reason "a failure pacman names no reason for is not excused" "error: something else" 0 0 other + +#---------------------------------------------------------------------------# +# Registry reads on GHCR, against a stand-in for curl: the dummy token first, +# only its 200 believed; any other answer asked again with the real token, +# which every process of a shard shares through REGISTRY_TOKEN_CACHE and +# asks for again once when the registry refuses it. +#---------------------------------------------------------------------------# +fake="$work/fake" +mkdir -p "$fake/bin" +cat > "$fake/bin/curl" << 'EOF' +#!/usr/bin/env bash +# Logs "URLAUTH" to $FAKE/calls.log; answers a token request with +# $FAKE/next-token, and a manifest request from $FAKE/answers.tsv: +# AUTHCODEBODY-FILE, the first line whose AUTH matches. +out="" auth="" url="" +while [ "$#" -gt 0 ]; do + case "$1" in + --output) out="$2"; shift 2 ;; + --header) case "$2" in Authorization:*) auth="${2#Authorization: }" ;; esac; shift 2 ;; + --write-out | --user | --data-urlencode | --connect-timeout | --max-time | --retry | --retry-delay | \ + --max-filesize | --proto | --proto-redir | --max-redirs | --speed-limit | --speed-time | --user-agent) shift 2 ;; + -*) shift ;; + *) url="$1"; shift ;; + esac +done +printf '%s\t%s\n' "$url" "$auth" >> "$FAKE/calls.log" +case "$url" in + */token) printf '{"token":"%s"}' "$(cat "$FAKE/next-token")" > "$out" ;; + */manifests/*) + line="$(awk -F '\t' -v a="$auth" '$1 == a && !f { print $2 "\t" $3; f = 1 }' "$FAKE/answers.tsv")" + [ -n "$line" ] || { printf '000'; exit 7; } + cp "$FAKE/${line#*$'\t'}" "$out" + printf '%s' "${line%%$'\t'*}" + ;; +esac +EOF +chmod +x "$fake/bin/curl" +printf '{"schemaVersion":2,"layers":[]}' > "$fake/manifest.json" +printf '{"errors":[{"code":"MANIFEST_UNKNOWN"}]}' > "$fake/unknown.json" +printf '{"errors":[{"code":"UNAUTHORIZED"}]}' > "$fake/denied.json" +# reg NAME ANSWERS [CACHE] -> "STATUS CALLS TOKEN-CALLS", one registry_manifest +reg() { + local rc=0 + printf '%b' "$2" > "$fake/answers.tsv" + : > "$fake/calls.log" + FAKE="$fake" PATH="$fake/bin:$PATH" MIRROR_REGISTRY=ghcr.io MIRROR_REGISTRY_AUTH=token MIRROR_REGISTRY_SCHEME=https \ + REGISTRY_TOKEN="${REG_TOKEN-secret}" REGISTRY_TOKEN_CACHE="${3:-}" \ + in_lib registry_manifest it/repo latest "$fake/out.json" > /dev/null 2> "$fake/err" || rc=$? + printf '%s %s %s\n' "$rc" "$(awk 'END { print NR + 0 }' "$fake/calls.log")" "$(awk -F '\t' '$1 ~ /\/token$/ { n++ } END { print n + 0 }' "$fake/calls.log")" +} +echo tok1 > "$fake/next-token" +check "registry: the dummy token's 200 is the answer, and no token is asked for" "0 1 0" \ + "$(reg ok 'Bearer QQ==\t200\tmanifest.json\n')" "lib.sh registry_manifest" +check "registry: the dummy token's 404 is asked again with the real token, which finds it" "0 3 1" \ + "$(reg private 'Bearer QQ==\t404\tunknown.json\nBearer tok1\t200\tmanifest.json\n')" "lib.sh registry_manifest" +check "registry: absent only when the real token says so too" "4 3 1" \ + "$(reg absent 'Bearer QQ==\t404\tunknown.json\nBearer tok1\t404\tunknown.json\n')" "lib.sh registry_manifest" +check "registry: with no token of its own, the dummy token's 404 is the answer" "4 1 0" \ + "$(REG_TOKEN='' reg anonymous 'Bearer QQ==\t404\tunknown.json\n')" "lib.sh registry_manifest" +mkdir -m 0700 "$fake/cache" +answers='Bearer QQ==\t404\tunknown.json\nBearer tok1\t200\tmanifest.json\n' +reg first "$answers" "$fake/cache" > /dev/null +check "registry: a second process reads the token the first one cached" "0 2 0" \ + "$(reg second "$answers" "$fake/cache")" "lib.sh registry_manifest, REGISTRY_TOKEN_CACHE" +echo tok2 > "$fake/next-token" +check "registry: a cached token the registry refuses is asked for again, once" "0 4 1" \ + "$(reg refresh 'Bearer QQ==\t404\tunknown.json\nBearer tok1\t401\tdenied.json\nBearer tok2\t200\tmanifest.json\n' "$fake/cache")" \ + "lib.sh registry_manifest, REGISTRY_TOKEN_CACHE" +check "registry: the cache then holds the new token" tok2 "$(cat "$fake"/cache/* 2> /dev/null)" "ls $fake/cache" + summary From 9363966e4c053c5c77221c4d96b26edf0eecc0c4 Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 16:00:35 +0545 Subject: [PATCH 06/22] tests: tag marker on the keyring image pin in 40-lib --- tests/unit/40-lib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/40-lib.sh b/tests/unit/40-lib.sh index 6e7fa72..80329b6 100644 --- a/tests/unit/40-lib.sh +++ b/tests/unit/40-lib.sh @@ -69,6 +69,7 @@ refused "a line that is not key = value" "$work/t6/x86_64-core.conf" #---------------------------------------------------------------------------# # The keys that set verify's trust root #---------------------------------------------------------------------------# +# tag: aarch64 IMG='ghcr.io/pkgforge-dev/archlinux:aarch64@sha256:b3610e2db83532a105802cf05a40ee7273f58231d5d0e507138f0cd9adef2e27' # trust NAME LINES...: a target file of x86_64 extra, with x86_64-core beside it trust() { From c90e807fed0e3377f1d476600b7cf3a3942c73f0 Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 16:01:33 +0545 Subject: [PATCH 07/22] verify-batch: refuse keyserver lookups inside the verify container --- scripts/verify-batch | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/verify-batch b/scripts/verify-batch index 360ba83..5ba35dc 100755 --- a/scripts/verify-batch +++ b/scripts/verify-batch @@ -290,6 +290,11 @@ pac() { pacman --config /tmp/pacman.conf --noconfirm "$@"; } phase "keyring" pacman-key --init +# A key that is missing or expired must fail at once. pacman asks a keyserver +# for it otherwise, and that makes verify wait on a host outside this run. +# Port 9 on the loopback refuses the connection at once. +sed -i '/^[[:space:]]*keyserver[[:space:]]/d' /etc/pacman.d/gnupg/gpg.conf +echo 'keyserver hkp://127.0.0.1:9' >> /etc/pacman.d/gnupg/gpg.conf copied=() for f in /in/keyrings/*; do [ -f "$f" ] || continue From 57a97daaaad80e4b8f6878f06bf1539221c2c82f Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 16:03:55 +0545 Subject: [PATCH 08/22] workflows: each scheduled workflow starts the others when theirs did not fire --- .github/workflows/status.yml | 22 ++++++++ .github/workflows/sync.yml | 6 +++ .github/workflows/watchdog.yml | 8 +++ scripts/dispatch-overdue | 84 +++++++++++++++++++++++++++++++ tests/unit/45-dispatch-overdue.sh | 69 +++++++++++++++++++++++++ 5 files changed, 189 insertions(+) create mode 100755 scripts/dispatch-overdue create mode 100644 tests/unit/45-dispatch-overdue.sh diff --git a/.github/workflows/status.yml b/.github/workflows/status.yml index b2c0596..5904381 100644 --- a/.github/workflows/status.yml +++ b/.github/workflows/status.yml @@ -43,6 +43,28 @@ env: MIRROR_REGISTRY: ghcr.io jobs: + # GitHub can drop a scheduled run. This hourly run starts Sync when its + # newest run on main is older than its 3 h cron plus an hour, and the + # watchdog when its newest run is older than a day plus two hours. No needs, + # in either direction. + kick: + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: write + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Start an overdue sync or watchdog + env: + GH_TOKEN: ${{ github.token }} + run: | + scripts/dispatch-overdue sync.yml 240 + scripts/dispatch-overdue watchdog.yml 1560 + decide: runs-on: ubuntu-24.04 timeout-minutes: 5 diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 314f02c..ad77ae9 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -339,3 +339,9 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: scripts/check-schedules-fired + # Each scheduled workflow starts the others when theirs did not fire. + - name: Start an overdue status run + if: ${{ github.ref == 'refs/heads/main' }} + env: + GH_TOKEN: ${{ github.token }} + run: scripts/dispatch-overdue status.yml 120 diff --git a/.github/workflows/watchdog.yml b/.github/workflows/watchdog.yml index d5bc456..eb40cf1 100644 --- a/.github/workflows/watchdog.yml +++ b/.github/workflows/watchdog.yml @@ -87,6 +87,14 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: scripts/check-schedules-fired + # Each scheduled workflow starts the others when theirs did not fire. + - name: Start an overdue sync or status run + if: ${{ github.ref == 'refs/heads/main' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + scripts/dispatch-overdue sync.yml 240 + scripts/dispatch-overdue status.yml 120 worker: runs-on: ubuntu-24.04 diff --git a/scripts/dispatch-overdue b/scripts/dispatch-overdue new file mode 100755 index 0000000..b6375b6 --- /dev/null +++ b/scripts/dispatch-overdue @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# Start a workflow on main when its newest run is older than a limit. +# +# scripts/dispatch-overdue WORKFLOW MAX_MINUTES +# +# WORKFLOW is a file name in .github/workflows (sync.yml). MAX_MINUTES is the +# longest gap that the workflow's own schedule allows, plus a margin. +# +# ⛔ GitHub can delay or drop a scheduled run, and a dropped run leaves nothing +# behind that could notice. Each scheduled workflow runs this for the others, +# so one schedule that fires restarts the ones that did not. +# +# A run that is queued or in progress counts as fresh: nothing starts twice. +# A workflow with no run at all is started. The run is started on main only. +# +# Needs GH_TOKEN with actions: write, GITHUB_REPOSITORY, gh and jq. +# +# Exit status: 0 the workflow is fresh or was started, 1 could not tell or +# could not start it, 2 usage. +set -euo pipefail + +usage() { + echo "usage: scripts/dispatch-overdue WORKFLOW MAX_MINUTES" >&2 + exit 2 +} + +die() { + echo "dispatch-overdue: $1" >&2 + shift + while [ "$#" -gt 0 ]; do + echo " $1" >&2 + shift + done + exit 1 +} + +wf="${1:-}" +max="${2:-}" +[ "$#" -eq 2 ] || usage +[[ "$wf" =~ ^[a-z0-9][a-z0-9_.-]*\.ya?ml$ ]] || usage +[[ "$max" =~ ^[1-9][0-9]*$ ]] || usage +command -v gh > /dev/null || die "gh is required" +command -v jq > /dev/null || die "jq is required" + +slug="${GITHUB_REPOSITORY:-}" +case "$slug" in + */*) ;; + *) die "GITHUB_REPOSITORY is not owner/name: ${slug:-}" ;; +esac + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +rc=0 +gh api "repos/$slug/actions/workflows/$wf/runs?branch=main&per_page=10" > "$work/runs.json" || rc=$? +[ "$rc" -eq 0 ] || die "could not list the runs of $wf (gh ended $rc)" \ + "reproduce: gh api 'repos/$slug/actions/workflows/$wf/runs?branch=main&per_page=10'" + +# busy: runs that have not ended; age: minutes since the newest run started +busy="$(jq '[.workflow_runs[] | select(.status != "completed")] | length' "$work/runs.json")" +age="$(jq 'if (.workflow_runs | length) == 0 then -1 + else ((now - (.workflow_runs | map(.created_at | fromdateiso8601) | max)) / 60 | floor) end' "$work/runs.json")" + +if [ "$busy" -gt 0 ]; then + echo "dispatch-overdue: $wf has $busy run(s) that have not ended; nothing to start" + exit 0 +fi +if [ "$age" -ge 0 ] && [ "$age" -le "$max" ]; then + echo "dispatch-overdue: $wf last started $age min ago, within $max min" + exit 0 +fi + +if [ "$age" -lt 0 ]; then + why="it has no run on main" +else + why="its newest run on main started $age min ago, over $max min" +fi +echo "::warning::dispatch-overdue: $wf is overdue: $why; starting it" +rc=0 +gh workflow run "$wf" --ref main --repo "$slug" || rc=$? +[ "$rc" -eq 0 ] || die "could not start $wf (gh ended $rc)" \ + "reproduce: gh workflow run $wf --ref main --repo $slug" +echo "dispatch-overdue: started $wf on main" diff --git a/tests/unit/45-dispatch-overdue.sh b/tests/unit/45-dispatch-overdue.sh new file mode 100644 index 0000000..994a040 --- /dev/null +++ b/tests/unit/45-dispatch-overdue.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# scripts/dispatch-overdue starts a workflow only when its newest run on main +# is older than the limit and no run of it is still going. +# +# A stub gh answers the runs list from a file and records every call, so the +# test needs no network and no token. +set -euo pipefail +# shellcheck source=../lib/harness.sh +. "$REPO_ROOT/tests/lib/harness.sh" + +need_tool jq "dispatch-overdue reads the runs list with jq" || { summary; exit; } + +work="$(work_dir)" +trap 'rm -rf "$work"' EXIT +mkdir -p "$work/bin" +cat > "$work/bin/gh" << 'EOF' +#!/usr/bin/env bash +printf 'gh %s\n' "$*" >> "$STUB_DIR/calls.log" +case "$1 $2" in + "api repos/"*) + [ "${STUB_FAIL:-0}" = 1 ] && exit 4 + cat "$STUB_DIR/runs.json" + ;; + "workflow run") exit 0 ;; + *) + echo "stub gh: no answer for: gh $*" >&2 + exit 9 + ;; +esac +EOF +chmod 0755 "$work/bin/gh" +export STUB_DIR="$work" GITHUB_REPOSITORY=pkgforge-dev/mirror-archlinux +export PATH="$work/bin:$PATH" + +# iso MINUTES_AGO -> an ISO 8601 UTC time that many minutes ago +iso() { + jq -nr --argjson m "$1" 'now - ($m * 60) | floor | todate' +} + +# case_run DESC RUNS_JSON EXPECT_RC EXPECT_DISPATCH(yes|no) [ARGS...] +case_run() { + local desc="$1" runs="$2" want_rc="$3" want_dispatch="$4" rc=0 got + shift 4 + printf '%s\n' "$runs" > "$work/runs.json" + : > "$work/calls.log" + bash "$REPO_ROOT/scripts/dispatch-overdue" "$@" > "$work/out.log" 2>&1 || rc=$? + check "$desc: exit status" "$want_rc" "$rc" "scripts/dispatch-overdue $*" + if grep -q '^gh workflow run ' "$work/calls.log"; then got=yes; else got=no; fi + check "$desc: workflow started" "$want_dispatch" "$got" "cat calls.log after scripts/dispatch-overdue $*" +} + +case_run "a run 30 min old, limit 240" \ + "{\"workflow_runs\":[{\"status\":\"completed\",\"created_at\":\"$(iso 30)\"}]}" 0 no sync.yml 240 +case_run "a newest run 300 min old, limit 240" \ + "{\"workflow_runs\":[{\"status\":\"completed\",\"created_at\":\"$(iso 300)\"},{\"status\":\"completed\",\"created_at\":\"$(iso 500)\"}]}" 0 yes sync.yml 240 +case_run "an old run and a newer one 10 min old" \ + "{\"workflow_runs\":[{\"status\":\"completed\",\"created_at\":\"$(iso 900)\"},{\"status\":\"completed\",\"created_at\":\"$(iso 10)\"}]}" 0 no sync.yml 240 +case_run "an old run that is still in progress" \ + "{\"workflow_runs\":[{\"status\":\"in_progress\",\"created_at\":\"$(iso 600)\"}]}" 0 no sync.yml 240 +case_run "an old run and a queued one" \ + "{\"workflow_runs\":[{\"status\":\"completed\",\"created_at\":\"$(iso 600)\"},{\"status\":\"queued\",\"created_at\":\"$(iso 1)\"}]}" 0 no sync.yml 240 +case_run "no run at all" '{"workflow_runs":[]}' 0 yes status.yml 120 +case_run "a workflow name that is not a file name" '{"workflow_runs":[]}' 2 no 'sync yml' 240 +case_run "a limit that is not a number" '{"workflow_runs":[]}' 2 no sync.yml soon + +STUB_FAIL=1 case_run "the runs list does not read" '{"workflow_runs":[]}' 1 no sync.yml 240 + +summary From 38b27bac75822f234a68b9ad4fbd928919a158a4 Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 17:42:33 +0545 Subject: [PATCH 09/22] client: parallel prefetch, direct blob addressing, resume, host blacklist --- client/pacman-oci | 1615 +++++++++++++++++++++++--- scripts/bench-consumer | 333 ++++++ tests/fixtures/client/build-repos | 69 ++ tests/fixtures/client/consumer | 201 ++++ tests/fixtures/client/fault-proxy.py | 115 ++ tests/integration/20-client.sh | 259 +++++ tests/unit/31-run-store.sh | 198 ++++ tests/unit/32-prefetch-plan.sh | 212 ++++ 8 files changed, 2835 insertions(+), 167 deletions(-) create mode 100755 scripts/bench-consumer create mode 100644 tests/fixtures/client/build-repos create mode 100644 tests/fixtures/client/consumer create mode 100644 tests/fixtures/client/fault-proxy.py create mode 100644 tests/integration/20-client.sh create mode 100644 tests/unit/31-run-store.sh create mode 100644 tests/unit/32-prefetch-plan.sh diff --git a/client/pacman-oci b/client/pacman-oci index 8c346a5..085b97c 100755 --- a/client/pacman-oci +++ b/client/pacman-oci @@ -30,28 +30,59 @@ # # ⭐ Everything this reads is checked before pacman sees it: a manifest asked # for by digest must hash to that digest, a blob's size and sha256 must equal -# its descriptor, and pacman then checks the database's sha256 and the package -# signature as it does for any mirror. +# its descriptor or the database's %SHA256SUM%, and pacman then checks the +# database's sha256 and the package signature as it does for any mirror. +# +# Prefetch. With an XferCommand, pacman asks for one file at a time. So the +# first package request of a sync run asks pacman itself for the whole list, in +# print mode (pacman -S --print takes no lock and needs no root), and fetches it +# in parallel. The first database request does the same for the database of +# every repository whose first Server is oci://. Each file is checked, then kept +# under a name of its own beside OUTPUT, and the requests that follow are one +# rename each, with no network. A package signature comes from the database's +# %PGPSIG% when it has one: pacman checks the package against that signature. +# Anything the prefetch did not make ready is fetched alone, as before. +# +# Run store. The parent's pid and start time name the pacman run. Its files sit +# flat in OUTPUT's directory as .pacman-oci.-.. pacman +# removes them with its download-XXXXXX directory, a background process removes +# them when pacman exits, and each call removes those of runs that are gone. A +# call whose parent is not pacman keeps no store. # # pacman runs this with fork and execvp, never through a shell. With # DownloadUser set (stock Arch) it runs as that user, from a download-XXXXXX -# directory inside the sync or cache directory; without it, as root. Nothing is -# written but OUTPUT and one private temporary directory, removed on exit. +# directory inside the sync or cache directory; without it, as root. # -# Needs curl, awk, sed, sha256sum, mktemp. +# Needs curl, awk, sed, sha256sum, mktemp, base64 (coreutils or busybox). Uses +# pacman, pacman-conf and timeout when they are there. # -# Environment: PACMAN_OCI_DEBUG=1 prints every request to stderr. -# PACMAN_OCI_NO_DUMMY=1 skips the dummy token GHCR accepts for public packages -# and goes straight to the registry's token flow; it exists so that flow is -# exercised on schedule. +# Environment: +# PACMAN_OCI_DEBUG=1 print every request and decision to stderr +# PACMAN_OCI_PREFETCH=0 no prefetch: every file is one request +# PACMAN_OCI_JOBS=N transfers at the same time in a prefetch, default +# 32, at most 64; 1 means no prefetch +# PACMAN_OCI_SIG=registry a prefetch reads each package signature from the +# registry's layer, never from %PGPSIG% (default auto) +# PACMAN_OCI_NO_DUMMY=1 skip the dummy token GHCR accepts for public +# packages and use the registry's token flow, with no +# prefetch; it exists so that flow runs on schedule +# PACMAN_OCI_TEST_PROC=DIR for the test suite only: read the parent process +# from DIR, not /proc. Used only when DIR holds a +# file named pacman-oci-test-proc. +# TMPDIR where the private work directory goes # # Exit status: 0 the file is at OUTPUT, 1 not here (pacman tries the next -# Server), 2 usage. +# Server), 2 usage, 3 the --json input is not JSON. LC_ALL=C export LC_ALL -PACMAN_OCI_VERSION=1 +PACMAN_OCI_VERSION=2 + +# A tab and a newline, written out: a command substitution costs a fork. +TAB=' ' +NL=' +' usage() { printf 'pacman-oci: %s\n' "$1" >&2 @@ -65,6 +96,106 @@ debug() { fi } +#---------------------------------------------------------------------------# +# The run store +# +# The key is the parent's pid and its start time, field 22 of /proc/PID/stat +# counted after the last ")", so a pid the kernel reuses is another run. An +# entry is used only when it is a regular file this user owns, never through a +# symbolic link, and each is made under a mktemp name, then renamed. +# FILE a file ready for pacman, its size and sha256 checked +# ,open the store of this run is open in this directory +# ,prefetch the package prefetch of this run ran here +# ,dbprefetch the database prefetch of this run ran here +# ,down.HOST HOST failed once its retries were spent +# ,absent.FILE the database manifest of this run has no layer FILE +# ,index FILESHA256SIZE of each package the prefetch planned +# ,reaper the background remover's output +# tmp.* files being written +#---------------------------------------------------------------------------# +PROC=/proc +if [ -n "${PACMAN_OCI_TEST_PROC:-}" ] && [ -f "$PACMAN_OCI_TEST_PROC/pacman-oci-test-proc" ]; then + PROC=$PACMAN_OCI_TEST_PROC +fi + +# start_of PID: the start time of PID into $_st, empty when PID is gone +start_of() { + _st='' + [ -r "$PROC/$1/stat" ] || return 0 + IFS= read -r _line < "$PROC/$1/stat" || return 0 + case $_line in + *') '*) ;; + *) return 0 ;; + esac + set -f + # shellcheck disable=SC2086 # the fields after the last ")" are words + set -- ${_line##*) } + set +f + [ "$#" -ge 20 ] || return 0 + shift 19 + case $1 in + '' | *[!0-9]*) ;; + *) _st=$1 ;; + esac +} + +# run_key: the key of the pacman run that called this, into $key; empty when +# the parent is not pacman +run_key() { + key='' + if [ -r "$PROC/$PPID/comm" ] && IFS= read -r _comm < "$PROC/$PPID/comm"; then + case $_comm in + pacman*) + start_of "$PPID" + if [ -n "$_st" ]; then + key=$PPID-$_st + fi + ;; + esac + fi +} + +# owned FILE: a regular file of this user, not a symbolic link +owned() { + # shellcheck disable=SC3067 # test -O is in bash, dash and busybox + [ -f "$1" ] && [ ! -L "$1" ] && [ -O "$1" ] +} + +# serve_ready OUTPUT URL: the fast path, before the rest of this file is even +# read. A package, its .sig, or an oci:// database the prefetch of this run +# made ready is renamed to OUTPUT, and the call ends. +serve_ready() { + _f=${2##*/} + case $_f in + '' | ,* | *[!!-~]*) return 0 ;; + *.pkg.tar | *.pkg.tar.*) ;; + *) + case $2 in + oci://* | oci+http://*) ;; + *) return 0 ;; + esac + ;; + esac + case $1 in + */*) _d=${1%/*} ;; + *) _d=. ;; + esac + run_key + [ -n "$key" ] || return 0 + _e=${_d:-/}/.pacman-oci.$key.$_f + if owned "$_e" && mv -f -- "$_e" "$1"; then + debug "$_f from the prefetch" + exit 0 + fi +} + +if [ "$#" -eq 2 ]; then + case $1 in + -* | '') ;; + *) serve_ready "$1" "$2" ;; + esac +fi + #---------------------------------------------------------------------------# # Tags # @@ -82,8 +213,6 @@ hash_tag() { printf 'fn-%s\n' "$(printf '%s' "$1" | sha256sum | cut -d ' ' -f 1)" } -TAB=$(printf '\t') - # Filenames on stdin, one per line; "FILENAMETAG" on stdout. This is the # only implementation of the mapping: --tag and --tags both come through here. tags_of() { @@ -223,6 +352,30 @@ flat_get() { P="$2" awk -F '\t' '$1 == ENVIRON["P"] { print $2; exit }' "$1" } +# layers_of FLAT NAME: the layers titled NAME and NAME.sig, on one line: +# COUNT DIGEST SIZE SIGCOUNT SIGDIGEST SIGSIZE MEDIATYPE +# A digest that is not sha256 hex, or a size that is not a number, is "-". +# shellcheck disable=SC2016 # an awk program: awk expands it, the shell must not +LAYERS_AWK=' +function d(x) { return (x ~ /^sha256:[0-9a-f]+$/ && length(x) == 71) ? x : "-" } +function z(x) { return (x ~ /^(0|[1-9][0-9]*)$/ && length(x) <= 15) ? x : "-" } +BEGIN { FS = "\t"; n = ENVIRON["N"]; sn = n ".sig" } +$1 == "/mediaType" { media = $2 } +$1 ~ /^\/layers\/[0-9]+\/annotations\/org\.opencontainers\.image\.title$/ { + split($1, p, "/") + if ($2 == n) { fc++; fi = p[3] } + if ($2 == sn) { sc++; si = p[3] } +} +$1 ~ /^\/layers\/[0-9]+\/(digest|size)$/ { split($1, p, "/"); v[p[3] "/" p[4]] = $2 } +END { + printf "%d %s %s %d %s %s %s\n", fc, d(v[fi "/digest"]), z(v[fi "/size"]), sc, d(v[si "/digest"]), z(v[si "/size"]), (media == "" ? "-" : media) +} +' + +layers_of() { + N="$2" awk "$LAYERS_AWK" "$1" +} + #---------------------------------------------------------------------------# # Arguments #---------------------------------------------------------------------------# @@ -257,6 +410,43 @@ esac out=$1 url=$2 [ -n "$out" ] || usage 'OUTPUT is empty' +file=${url##*/} + +# pacman writes OUTPUT in its sync or cache dir, or, with DownloadUser set, in +# a download-XXXXXX dir inside it. Either way the files it finished earlier in +# this run sit beside OUTPUT, and the ones it held before sit one level up. +case $out in + */*) out_dir=${out%/*} ;; + *) out_dir=. ;; +esac +[ -n "$out_dir" ] || out_dir=/ +held_dir=$out_dir +case ${out_dir##*/} in + download-*) + case $out_dir in + */*) held_dir=${out_dir%/*} ;; + *) held_dir=. ;; + esac + [ -n "$held_dir" ] || held_dir=/ + ;; +esac + +# What the file is. +# pkg a package: its own tag, a layer named FILE +# pkgsig its signature: the package's tag, a layer named FILE, never optional +# dbsig a database signature: the manifest of the database pacman just took, +# found by that database's content (tag db-HEX or files-HEX) +# db a database: the tag, digest or latest from the URL +case $file in + *.pkg.tar.sig | *.pkg.tar.*.sig) kind=pkgsig ;; + *.pkg.tar | *.pkg.tar.*) kind=pkg ;; + *.db.sig | *.files.sig) kind=dbsig ;; + *) kind=db ;; +esac +case $url in + oci://* | oci+http://*) ours=1 ;; + *) ours=0 ;; +esac # Every failure path removes OUTPUT as well as the working directory: pacman # never deletes a .part after a failed attempt. @@ -291,85 +481,1211 @@ else PROGRESS=--silent fi -# curl's transient set (timeouts, 408, 429, 500, 502, 503, 504), honouring -# Retry-After, starting no new try after 30 seconds: pacman downloads one file -# at a time, so a longer window would stall a whole upgrade during an outage. -# A manifest or a token is a few KiB, so each try of one gets 15 seconds and a -# 64 KiB cap: the parser below is plain awk, and a body far larger than any -# real manifest must fail fast, not keep pacman waiting. +# A manifest or token request retries curl's transient set (timeouts, 408, +# 429, 500, 502, 503, 504), honouring Retry-After, starting no new try after +# 30 seconds: pacman downloads one file at a time, so a longer window would +# stall a whole upgrade during an outage. A manifest or a token is a few KiB, +# so each try of one gets 15 seconds and a 64 KiB cap: the parser below is +# plain awk, and a body far larger than any real manifest must fail fast, not +# keep pacman waiting. A file download retries in fetch_one. RETRY='--retry 2 --retry-max-time 30' +# With DownloadUser set, this runs as that user with root's environment, so a +# TMPDIR only root can write is possible; the directory pacman writes OUTPUT to +# is always writable by whoever runs this. +need_work() { + [ -z "$work" ] || return 0 + _base=${TMPDIR:-/tmp} + if [ ! -d "$_base" ] || [ ! -w "$_base" ]; then + _base=$out_dir + fi + work=$(mktemp -d "$_base/pacman-oci.XXXXXX") || broken "could not make a temporary directory in $_base" +} + +sha256_of() { + _sum=$(sha256sum < "$1") || return 1 + printf '%s\n' "${_sum%% *}" +} + +size_of() { + _n=$(wc -c < "$1") || return 1 + printf '%s\n' $((_n)) +} + +# A clock in nanoseconds. busybox date has no %N, so there it counts seconds. +now_ns() { + _ns=$(date +%s%N) + case $_ns in + *[!0-9]* | '') _ns=$(date +%s)000000000 ;; + esac + if [ "${#_ns}" -lt 16 ]; then + _ns=${_ns}000000000 + fi + printf '%s\n' "$_ns" +} + +# proto_for BASE: the protocols curl may use for BASE and its redirects, in $_p +proto_for() { + case $1 in + https://*) _p='=https' ;; + *) _p='=http,https' ;; + esac +} + #---------------------------------------------------------------------------# -# Not ours: an ordinary download. pacman's own downloader is off whenever an -# XferCommand is set, so this is what fetches from the fallback mirrors. +# The run store, beyond the fast path #---------------------------------------------------------------------------# -case $url in - oci://* | oci+http://*) ;; - *) - rc=0 - # shellcheck disable=SC2086 # RETRY is several words on purpose - curl -q --fail --location --show-error "$PROGRESS" \ - --connect-timeout 20 --speed-limit 1024 --speed-time 60 \ - $RETRY --max-redirs 10 \ - --output "$out" --url "$url" || rc=$? - if [ "$rc" -ne 0 ]; then - rm -f -- "$out" - exit 1 +run_key +store=$out_dir/.pacman-oci.$key. + +# entry_ok NAME: the entry exists, is a regular file, and is ours +entry_ok() { + [ -n "$key" ] && owned "$store$1" +} + +# put NAME [FROM]: make the entry NAME, empty or holding the file FROM, which +# must be a mktemp file in the same directory +put() { + if [ -n "${2:-}" ]; then + mv -f -- "$2" "$store$1" + return + fi + _t=$(mktemp "${store}tmp.XXXXXX") || return 1 + mv -f -- "$_t" "$store$1" +} + +# Remove the entries of runs that are gone. Once this run's store is open in +# this directory, that was done, and the call goes straight on. +sweep() { + [ -d "$PROC" ] || return 0 + if entry_ok ,open; then + return 0 + fi + set -- + _live='' + for _e in "$out_dir"/.pacman-oci.*; do + owned "$_e" || continue + _k=${_e##*/.pacman-oci.} + _k=${_k%%.*} + [ "$_k" != "$key" ] && [ "$_k" != "$_live" ] || continue + _pid=${_k%%-*} + _start=${_k#*-} + case $_pid:$_start in + *[!0-9:]* | :* | *:) continue ;; + esac + start_of "$_pid" + if [ "$_st" = "$_start" ]; then + _live=$_k + continue fi + set -- "$@" "$_e" + done + if [ "$#" -gt 0 ]; then + debug "removing $# file(s) of pacman runs that are gone" + rm -f -- "$@" + fi +} + +# reap PID START PREFIX: wait for the pacman run to end, then remove its +# entries in one directory. pacman waits for its child alone, never for this. +# shellcheck disable=SC2329 # run in the background by store_open +reap() { + trap '' HUP INT + while start_of "$1" && [ "$_st" = "$2" ]; do + sleep 1 + done + rm -f -- "$3"* +} + +store_open() { + [ -n "$key" ] || return 1 + if entry_ok ,open; then + return 0 + fi + put ,reaper || return 1 + put ,open || return 1 + (cd / && reap "${key%%-*}" "${key#*-}" "$store") < /dev/null >> "${store},reaper" 2>&1 & + return 0 +} + +# serve NAME: a ready entry becomes OUTPUT, and the call ends +serve() { + entry_ok "$1" || return 1 + if mv -f -- "$store$1" "$out"; then + debug "$file from the prefetch" exit 0 - ;; -esac + fi + return 1 +} + +# host_of URL: the host[:port] of URL into $_host, empty when it has none +host_of() { + _host=${1#*://} + if [ "$_host" = "$1" ]; then + _host='' + return + fi + _host=${_host%%/*} + _host=${_host##*@} + case $_host in + *[!A-Za-z0-9.:-]*) _host='' ;; + esac +} + +# mark_down HOST HTTP: later requests of this run to HOST miss at once. Only no +# answer at all, 429 and 5xx count. +mark_down() { + [ -n "$1" ] || return 0 + case $2 in + 000 | 429 | 5??) ;; + *) return 0 ;; + esac + if store_open && put ",down.$1"; then + debug "$1 is down for the rest of this run (HTTP $2)" + fi +} #---------------------------------------------------------------------------# -# Parse oci://HOST/REPOSITORY[:TAG|@DIGEST]/FILE +# pacman's own arguments, for a print run +# +# They come from /proc/PPID/cmdline. Anything this cannot read with certainty +# means no prefetch: a newline in an argument, an option it does not know +# (getopt also takes abbreviations), a relative path, or a target read from +# stdin. The rewrite drops every refresh (--refresh, and each y of a short +# group before its first b or r), --confirm, --noconfirm and --print-format, +# and keeps everything after -- as it is. +# +# Input: one argument per line, a newline inside one turned into \001. +# Output: NAMEVALUE lines (why, sync, print, sysroot, arch, config, +# argv0); the rewritten arguments, one per line, go to the file ENVIRON["ARGS"]. #---------------------------------------------------------------------------# -case $url in - oci://*) - scheme=https - rest=${url#oci://} - ;; - *) - scheme=http - rest=${url#oci+http://} - ;; -esac +# shellcheck disable=SC2016 # an awk program: awk expands it, the shell must not +ARGV_AWK=' +BEGIN { + args = ENVIRON["ARGS"] + printf "" > args + split("database files query remove sync deptest upgrade version help cascade changelog clean nodeps deps explicit groups info check list foreign native nosave owns file print quiet recursive search regex machinereadable unrequired upgrades sysupgrade unneeded verbose downloadonly refresh noconfirm confirm force noprogressbar noscriptlet asdeps needed asexplicit dbonly disable-download-timeout disable-sandbox disable-sandbox-filesystem disable-sandbox-syscalls", a, " ") + for (i in a) noarg[a[i]] = 1 + split("dbpath root sysroot config ignore assume-installed overwrite ask cachedir hookdir logfile ignoregroup arch print-format gpgdir color", a, " ") + for (i in a) reqarg[a[i]] = 1 + split("dbpath root sysroot config cachedir hookdir logfile gpgdir", a, " ") + for (i in a) ispath[a[i]] = 1 + shorts = "DFQRSTUVcdefghiklmnopqstuvwxy" + config = "/etc/pacman.conf" +} +function refuse(why) { if (why_ == "") why_ = why } +function emit(s) { print s > args } +function value(name, v) { + if ((name in ispath) && substr(v, 1, 1) != "/") refuse("--" name " is a relative path") + if (name == "config") config = v + if (name == "sysroot") sysroot = 1 + if (name == "arch") arch = 1 +} +function long(s, eq, name) { + eq = index(s, "=") + name = eq ? substr(s, 1, eq - 1) : s + if (name == "debug") { emit("--" s); return } + if (name in noarg) { + if (eq) { refuse("--" name " takes no value"); return } + if (name == "refresh" || name == "confirm" || name == "noconfirm") return + if (name == "print") print_ = 1 + if (name == "sync") sync = 1 + emit("--" s) + return + } + if (name in reqarg) { + if (eq) { + value(name, substr(s, eq + 1)) + if (name != "print-format") emit("--" s) + return + } + pending = name + drop = (name == "print-format") + if (!drop) emit("--" s) + return + } + refuse("pacman option --" name " is not known here") +} +function short(s, i, c, o, n) { + o = "-" + n = length(s) + for (i = 1; i <= n; i++) { + c = substr(s, i, 1) + if (c == "b" || c == "r") { + emit(o substr(s, i)) + if (i == n) { pending = (c == "b") ? "dbpath" : "root"; drop = 0 } + else value((c == "b") ? "dbpath" : "root", substr(s, i + 1)) + return + } + if (index(shorts, c) == 0) { refuse("pacman option -" c " is not known here"); return } + if (c == "y") continue + if (c == "S") sync = 1 + if (c == "p") print_ = 1 + o = o c + } + if (o != "-") emit(o) +} +NR == 1 { argv0 = $0; next } +index($0, "\001") { refuse("an argument holds a newline"); next } +pending != "" { value(pending, $0); if (!drop) emit($0); pending = ""; next } +dashdash { if ($0 == "-") refuse("a target is read from stdin"); emit($0); next } +$0 == "--" { dashdash = 1; emit($0); next } +/^--/ { long(substr($0, 3)); next } +/^-./ { short(substr($0, 2)); next } +$0 == "-" { refuse("a target is read from stdin"); next } +{ emit($0) } +END { + if (NR == 0) refuse("the argument list is empty") + if (pending != "") refuse("--" pending " has no value") + if (index(argv0, "\001")) refuse("an argument holds a newline") + printf "why\t%s\nsync\t%d\nprint\t%d\nsysroot\t%d\narch\t%d\nconfig\t%s\nargv0\t%s\n", why_, sync, print_, sysroot, arch, config, argv0 +} +' -file=${rest##*/} -ref=${rest%/*} -host=${ref%%/*} -path=${ref#*/} -if [ "$ref" = "$rest" ] || [ "$host" = "$ref" ]; then - broken "not oci://HOST/REPOSITORY/FILE: $url" -fi +# argv_scan: read and rewrite pacman's arguments, once per call. Sets scan_why +# (empty when they are usable), scan_sync, scan_print, scan_sysroot, scan_arch, +# scan_config and scan_argv0, and writes $work/args. +argv_scan() { + if [ -n "${scan_done:-}" ]; then + [ -z "$scan_why" ] + return + fi + scan_done=1 + scan_why="the argument list of pacman ($PROC/$PPID/cmdline) could not be read" + scan_sync=0 scan_print=0 scan_sysroot=0 scan_arch=0 scan_config='' scan_argv0='' + need_work + [ -r "$PROC/$PPID/cmdline" ] || return 1 + tr '\000\n' '\n\001' < "$PROC/$PPID/cmdline" > "$work/cmdline" || return 1 + ARGS="$work/args" awk "$ARGV_AWK" "$work/cmdline" > "$work/scan" || return 1 + while IFS= read -r _l; do + _v=${_l#*"$TAB"} + case $_l in + "why$TAB"*) scan_why=$_v ;; + "sync$TAB"*) scan_sync=$_v ;; + "print$TAB"*) scan_print=$_v ;; + "sysroot$TAB"*) scan_sysroot=$_v ;; + "arch$TAB"*) scan_arch=$_v ;; + "config$TAB"*) scan_config=$_v ;; + "argv0$TAB"*) scan_argv0=$_v ;; + esac + done < "$work/scan" + [ -z "$scan_why" ] +} -ref_tag="" -ref_digest="" -case $path in - *@*) - ref_digest=${path#*@} - repo=${path%%@*} - ;; - *) - case ${path##*/} in - *:*) - ref_tag=${path##*:} - repo=${path%:*} - ;; - *) repo=$path ;; +# find_exe: the pacman binary into $exe. /proc/PPID/exe is unreadable for the +# download user (measured), so argv[0] is the fallback: absolute, or on PATH. +find_exe() { + exe='' + if [ -r "$PROC/$PPID/exe" ]; then + exe=$(readlink "$PROC/$PPID/exe") || exe='' + fi + if [ -z "$exe" ]; then + case $scan_argv0 in + /*) exe=$scan_argv0 ;; + */* | '') ;; + *) exe=$(command -v "$scan_argv0") || exe='' ;; esac - ;; + fi + case $exe in + /*) [ -x "$exe" ] ;; + *) return 1 ;; + esac +} + +#---------------------------------------------------------------------------# +# Parallel transfers +# +# One curl process runs a whole list with --parallel, from a generated config +# with one block per transfer. Each block carries its own deadlines and limits, +# and writes one line when it ends: EXITHTTPBYTESOUTPUT. +#---------------------------------------------------------------------------# +AUTH='Bearer QQ==' +if [ "${PACMAN_OCI_NO_DUMMY:-0}" = 1 ]; then + AUTH='' +fi +WRITE_OUT='%{exitcode}\t%{http_code}\t%{size_download}\t%{filename_effective}\n' + +# 32 transfers at a time: curl multiplexes them over one HTTP/2 connection per +# host. Measured with 40 small packages from Nepal, median of 8 runs through +# pacman: 3.8 s with 8, 7.8 s with 16, 3.0 s with 24, 2.7 s with 32. +jobs=${PACMAN_OCI_JOBS:-32} +case $jobs in + '' | *[!0-9]*) jobs=1 ;; esac +if [ "$jobs" -gt 64 ]; then + jobs=64 +fi +sigmode=auto +if [ "${PACMAN_OCI_SIG:-auto}" = registry ]; then + sigmode=registry +fi + +prefetch_allowed() { + [ -n "$key" ] || return 1 + [ "${PACMAN_OCI_PREFETCH:-1}" != 0 ] || return 1 + [ "$jobs" -gt 1 ] || return 1 + [ "${PACMAN_OCI_NO_DUMMY:-0}" != 1 ] || return 1 + # The directory goes into curl's config as it is, so it may hold nothing + # that needs quoting there. + case $out_dir in + *\"* | *\\* | *"$TAB"* | *"$NL"*) return 1 ;; + esac + return 0 +} + +# A connection to GHCR's blob host can stall: it goes on, at a few KB/s, for +# minutes, while a new connection is fast. Measured from a WSL2 VM in Nepal: +# 150 small blobs on one HTTP/2 connection that stalled took 425 and 440 s, +# and 4 to 6 s on a new connection right after; one 208 MB blob ran at +# 330 KB/s for 625 s; in about 1 of 4 runs of 40 small blobs, the connection +# sent at a few KB/s for 10 s or more, where a normal run takes 1 to 2 s. So +# each transfer gets a short time limit, and a transfer it cuts keeps its +# bytes and goes on in the next attempt, on a new connection. That costs +# about one round trip. + +# deadline SIZE ATTEMPT: the time limit of one attempt of the single-file +# path, into $_dl. Attempt 1 allows 3 s plus 1 MiB/s, attempt 2 30 s plus +# 128 KiB/s, attempt 3 no limit. +deadline() { + case $2 in + 1) _dl=$((3 + $1 / 1048576)) ;; + 2) _dl=$((30 + $1 / 131072)) ;; + *) _dl='' ;; + esac +} + +# slice SIZE PASS: the time limit of one transfer in one pass of a prefetch, +# into $_dl: 3 s for each pass so far plus 1 MiB/s, at most 30 s. A file that +# needs longer goes on in the next pass, so a slow connection costs at most +# 30 s, not the time of the whole file. Measured with 40 small packages: a +# limit of 3 s, not 10 s, took a run with a stall from 11 to 17 s down to 6.5 +# to 9 s. With one limit of 3 s plus 1 MiB/s for the whole file, 3 large +# packages (540 MB) took 238 s in one run, where the official mirror took 40 s. +slice() { + _dl=$((3 * $2 + $1 / 1048576)) + if [ "$_dl" -gt 30 ]; then + _dl=30 + fi +} + +# block URL OUTPUT SIZE AUTH(0|1) RESUME(0|1) PROTO DEADLINE: one transfer, +# added to the config file $cfg +block() { + if [ -s "$cfg" ]; then + printf 'next\n' >> "$cfg" + fi + _max=$3 + if [ "$_max" -eq 0 ]; then + _max=1 + fi + { + printf 'url = "%s"\noutput = "%s"\n' "$1" "$2" + if [ "$4" = 1 ] && [ -n "$AUTH" ]; then + printf 'header = "Authorization: %s"\n' "$AUTH" + fi + if [ "$5" = 1 ]; then + printf 'continue-at = "-"\n' + fi + if [ -n "$7" ]; then + printf 'max-time = %s\n' "$7" + fi + printf 'max-filesize = %s\nconnect-timeout = 10\nspeed-limit = 1024\nspeed-time = 60\n' "$_max" + printf 'location\nmax-redirs = 5\nfail\nproto = "%s"\nproto-redir = "%s"\n' "$6" "$6" + printf 'user-agent = "pacman-oci/%s"\nwrite-out = "%s"\n' "$PACMAN_OCI_VERSION" "$WRITE_OUT" + } >> "$cfg" +} + +#---------------------------------------------------------------------------# +# Blobs +# +# A blob item is a set of variables named by its number N: u_N the URL, h_N the +# sha256, s_N the size, r_N the entry it becomes, a_N 1 to send the token, p_N +# curl's protocols, c_N 1 to resume, z_N 1 once it started again from zero. +# It downloads to $tmpbase.N. Each pass is one curl process, so each pass has +# new connections. A transfer that ends with bytes on disk goes on in the next +# pass with --continue-at -; one that met 408, 429, 5xx or a broken connection +# with no bytes starts again; a resumed file with a wrong sha256 starts again +# from zero, once. A file whose size and sha256 check out becomes its entry. +# The passes go on while they move data: a pass after the first that makes no +# file ready and moves less than 30 KiB, 1 KiB/s for its 30 s, is the last. +#---------------------------------------------------------------------------# +blobs='' +blob_count=0 +got_files=0 +got_bytes=0 + +# add_blob URL SHA256 SIZE ENTRY AUTH PROTO +add_blob() { + blob_count=$((blob_count + 1)) + eval "u_$blob_count=\$1 h_$blob_count=\$2 s_$blob_count=\$3 r_$blob_count=\$4 a_$blob_count=\$5 p_$blob_count=\$6 c_$blob_count=0" + blobs="$blobs $blob_count" +} + +# check_file FILE SIZE SHA256 +check_file() { + [ "$(size_of "$1")" = "$2" ] && [ "$(sha256_of "$1")" = "$3" ] +} + +# again ID RESUME(0|1): item ID goes into the next pass +again() { + eval "c_$1=\$2" + _next="$_next $1" +} + +# ended EXIT HTTP ID: one transfer of a pass has ended. A whole file waits in +# $_whole for check_whole; a cut one goes into the next pass. +ended() { + if [ "$1" = 0 ] && { [ "$2" = 200 ] || [ "$2" = 206 ]; }; then + eval "k_$3=\$2" + _whole="$_whole $3" + return 0 + fi + _o=$tmpbase.$3 + _u='' + eval "_u=\$u_$3" + debug "prefetch GET $_u -> $2 (curl $1)" + # A time limit goes on in the next pass. No answer at all, a refusal, or a + # file over its size: no other pass does better, and the call for that file + # decides alone. + case $1:$2 in + 28:*) ;; + *:408 | *:429 | *:5??) ;; + *:000 | 63:* | *:4??) + rm -f -- "$_o" + return 0 + ;; + 16:* | 18:* | 35:* | 52:* | 55:* | 56:* | 92:*) ;; + *) + rm -f -- "$_o" + return 0 + ;; + esac + if [ -s "$_o" ]; then + again "$3" 1 + else + again "$3" 0 + fi +} -printf '%s\n' "$host" | grep -Eq '^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?(:[0-9]+)?$' || - broken "not a registry host: $host" -printf '%s\n' "$repo" | grep -Eq '^[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*$' || - broken "not a repository name: $repo" -if [ -n "$ref_tag" ]; then - printf '%s\n' "$ref_tag" | grep -Eq '^[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$' || broken "not a tag: $ref_tag" +# CHECK_AWK reads the output of sha256sum and then of wc -c on files named +# ENVIRON["P"]ID, and prints IDSIZESHA256 for each file both read. +# shellcheck disable=SC2016 # an awk program: awk expands it, the shell must not +CHECK_AWK=' +BEGIN { p = ENVIRON["P"]; lp = length(p) } +FILENAME == ARGV[1] { + n = substr($0, 67) + if (length($0) > 66 && substr($0, 1, 64) !~ /[^0-9a-f]/ && substr(n, 1, lp) == p) h[substr(n, lp + 1)] = substr($0, 1, 64) + next +} +{ + l = $0 + sub(/^ +/, "", l) + i = index(l, " ") + if (i < 2) next + s = substr(l, 1, i - 1) + n = substr(l, i + 1) + if (s ~ /^[0-9]+$/ && substr(n, 1, lp) == p) z[substr(n, lp + 1)] = s +} +END { for (k in h) if ((k in z) && k ~ /^[0-9]+$/) print k "\t" z[k] "\t" h[k] } +' + +# check_whole: the size and sha256 of each file in $_whole, from one sha256sum +# and one wc for all of them. A file that matches becomes its entry. A resumed +# file that does not match goes into the next pass, from zero, once. +check_whole() { + [ -n "$_whole" ] || return 0 + set -- + for _id in $_whole; do + set -- "$@" "$tmpbase.$_id" + done + rc=0 + sha256sum -- "$@" > "$work/sums" 2>> "$work/check.err" || rc=$? + [ "$rc" -eq 0 ] || debug "prefetch: sha256sum ended $rc" + rc=0 + wc -c -- "$@" > "$work/sizes" 2>> "$work/check.err" || rc=$? + [ "$rc" -eq 0 ] || debug "prefetch: wc ended $rc" + P="$tmpbase." awk "$CHECK_AWK" "$work/sums" "$work/sizes" > "$work/checked" || debug "prefetch: the check ended $?" + while IFS="$TAB" read -r _id _gs _gh; do + case $_id in + '' | *[!0-9]*) continue ;; + esac + eval "gs_$_id=\$_gs gh_$_id=\$_gh" + done < "$work/checked" + for _id in $_whole; do + _h='' _s='' _r='' _c='' _u='' _k='' _z='' _gs='' _gh='' + eval "_h=\$h_$_id _s=\$s_$_id _r=\$r_$_id _c=\$c_$_id _u=\$u_$_id _k=\$k_$_id _z=\${z_$_id:-} _gs=\${gs_$_id:-} _gh=\${gh_$_id:-} gs_$_id='' gh_$_id=''" + if [ "$_gs" = "$_s" ] && [ "$_gh" = "$_h" ] && put "$_r" "$tmpbase.$_id"; then + debug "prefetch GET $_u -> $_k, $_r ready" + got_files=$((got_files + 1)) + got_bytes=$((got_bytes + _s)) + continue + fi + rm -f -- "$tmpbase.$_id" + if [ "$_c" = 1 ] && [ "$_z" != 1 ]; then + debug "prefetch GET $_u -> $_k: the resumed file is not sha256 $_h, once more from zero" + eval "z_$_id=1" + again "$_id" 0 + else + debug "prefetch GET $_u -> $_k: not $_s bytes with sha256 $_h" + fi + done +} + +# fetch_blobs: every blob item, in passes while they move data, at most 100. +# Adds to got_files and got_bytes. +fetch_blobs() { + [ "$blob_count" -gt 0 ] || return 0 + tmpbase=$(mktemp "${store}tmp.XXXXXX") || return 0 + _todo=$blobs + _pass=1 + while [ -n "$_todo" ] && [ "$_pass" -le 100 ]; do + cfg=$work/blobs.cfg + : > "$cfg" + for _id in $_todo; do + eval "_u=\$u_$_id _s=\$s_$_id _a=\$a_$_id _c=\$c_$_id _p=\$p_$_id" + slice "$_s" "$_pass" + block "$_u" "$tmpbase.$_id" "$_s" "$_a" "$_c" "$_p" "$_dl" + done + debug "prefetch: pass $_pass starts" + rc=0 + curl -q --parallel --parallel-max "$jobs" --silent --show-error \ + --connect-timeout 10 --speed-limit 1024 --speed-time 60 \ + --config "$cfg" > "$work/blobs.out" 2>> "$work/curl.err" || rc=$? + _next='' + _whole='' + _moved=0 + _ready=$got_files + while IFS="$TAB" read -r _x _code _bytes _fn; do + _id=${_fn##*.} + case $_id in + '' | *[!0-9]*) continue ;; + esac + case $_bytes in + '' | *[!0-9]*) ;; + *) _moved=$((_moved + _bytes)) ;; + esac + ended "$_x" "$_code" "$_id" + done < "$work/blobs.out" + check_whole + debug "prefetch: pass $_pass ended, curl $rc, $_moved bytes, $((got_files - _ready)) ready" + _todo=$_next + if [ "$_pass" -gt 1 ] && [ "$got_files" -eq "$_ready" ] && [ "$_moved" -lt 30720 ]; then + break + fi + _pass=$((_pass + 1)) + done + if [ -s "$work/curl.err" ]; then + debug "prefetch: curl said: $(sed -n 1,5p "$work/curl.err")" + fi + rm -f -- "$tmpbase" "$tmpbase".* +} + +# fetch_manifests LIST: LIST lines are IDURL. Each body goes to $work/m.ID, +# and $work/m.res gets IDEXITHTTP lines. +fetch_manifests() { + cfg=$work/m.cfg + : > "$cfg" + : > "$work/m.res" + while IFS="$TAB" read -r _id _u; do + if [ -s "$cfg" ]; then + printf 'next\n' >> "$cfg" + fi + proto_for "$_u" + { + printf 'url = "%s"\noutput = "%s/m.%s"\n' "$_u" "$work" "$_id" + if [ -n "$AUTH" ]; then + printf 'header = "Authorization: %s"\n' "$AUTH" + fi + printf 'header = "Accept: application/vnd.oci.image.manifest.v1+json"\n' + printf 'max-filesize = 65536\nconnect-timeout = 10\nmax-time = 15\nproto = "%s"\n' "$_p" + printf 'user-agent = "pacman-oci/%s"\nwrite-out = "%s"\n' "$PACMAN_OCI_VERSION" "$WRITE_OUT" + } >> "$cfg" + done < "$1" + [ -s "$cfg" ] || return 0 + rc=0 + curl -q --parallel --parallel-max "$jobs" --silent --show-error \ + --connect-timeout 10 --max-time 15 \ + --config "$cfg" > "$work/m.out" 2>> "$work/curl.err" || rc=$? + while IFS="$TAB" read -r _x _code _bytes _fn; do + _id=${_fn##*/m.} + case $_id in + '' | *[!0-9]*) continue ;; + esac + debug "prefetch GET manifest $_id -> $_code (curl $_x)" + printf '%s\t%s\t%s\n' "$_id" "$_x" "$_code" >> "$work/m.res" + done < "$work/m.out" + debug "prefetch: manifests done, curl ended $rc" +} + +# say_summary START FILES BYTES: one line, on a terminal +say_summary() { + [ "$2" -gt 0 ] || return 0 + if [ -t 2 ] || [ "${PACMAN_OCI_DEBUG:-0}" = 1 ]; then + _end=$(now_ns) + awk -v n="$2" -v b="$3" -v t0="$1" -v t1="$_end" 'BEGIN { + printf "pacman-oci: prefetched %d file(s), %.1f MiB in %.1f s\n", n, b / 1048576, (t1 - t0) / 1e9 + }' >&2 + fi +} + +#---------------------------------------------------------------------------# +# Package prefetch +# +# pacman -S --print with the run's own arguments, less the refresh, prints one +# line per file it would download: %l (the first Server, or file:// when the +# file is cached), %h (sha256), %s (bytes to download) and %g (base64 +# %PGPSIG%). Print mode picks the first provider and never prompts. %s is the +# size less any .part pacman holds in the cache, so that .part is added back. +# +# PLAN_AWK prints, per file to fetch: +# TYPE FILE SHA256 SIZE PGPSIG BASE URL (TYPE oci or http; "-" when absent) +#---------------------------------------------------------------------------# +# shellcheck disable=SC2016 # an awk program: awk expands it, the shell must not +PLAN_AWK=' +BEGIN { FS = "\t" } +function hex(x) { return (length(x) == 64 && x !~ /[^0-9a-f]/) ? x : "-" } +NF != 4 { next } +{ + u = $1; h = hex($2); s = $3; g = $4 + if (s !~ /^(0|[1-9][0-9]*)$/ || length(s) > 15) next + if (g !~ /^[A-Za-z0-9+\/]+=*$/ || length(g) % 4 != 0) g = "-" + f = u + sub(/.*\//, "", f) + if (f !~ /^[!-~]+$/ || f ~ /^,/ || f ~ /\.sig$/ || f !~ /\.pkg\.tar(\.[^\/]+)?$/) next + if (u ~ /^oci:\/\//) { sch = "https"; r = substr(u, 7) } + else if (u ~ /^oci\+http:\/\//) { sch = "http"; r = substr(u, 12) } + else if (u ~ /^https?:\/\/[^\/]/) { + if (u ~ /[" \\]/ || h == "-") next + print "http\t" f "\t" h "\t" s "\t" g "\t-\t" u + next + } + else next + i = index(r, "/") + if (i == 0) next + host = substr(r, 1, i - 1) + p = substr(r, i + 1, length(r) - i - length(f) - 1) + at = index(p, "@") + if (at) p = substr(p, 1, at - 1) + else { + last = p + sub(/.*\//, "", last) + c = index(last, ":") + if (c) p = substr(p, 1, length(p) - length(last) + c - 1) + } + if (host !~ /^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?(:[0-9]+)?$/) next + if (p !~ /^[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(\/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*$/) next + print "oci\t" f "\t" h "\t" s "\t" g "\t" sch "://" host "/v2/" p "\t" u +} +' + +# sig_from_db FILE BASE64 TMP: the entry FILE.sig, from the decoded %PGPSIG%, +# through TMP, a new name in the store's directory +sig_from_db() { + if entry_ok "$1.sig"; then + return 0 + fi + if base64 -d > "$3" << EOF && [ -s "$3" ] && put "$1.sig" "$3"; then +$2 +EOF + return 0 + fi + rm -f -- "$3" + return 1 +} + +# decode_sigs: an entry FILE.sig for each line FILEBASE64 of $work/sigs. +# It runs beside the downloads. The name of each entry it made goes to +# $work/sigs.ok. +decode_sigs() { + : > "$work/sigs.ok" + _sb=$(mktemp "${store}tmp.XXXXXX") || return 1 + _i=0 + while IFS="$TAB" read -r _f _g; do + _i=$((_i + 1)) + if sig_from_db "$_f" "$_g" "$_sb.$_i"; then + printf '%s\n' "$_f" >> "$work/sigs.ok" + fi + done < "$work/sigs" + rm -f -- "$_sb" +} + +# print_run: pacman -S --print with the rewritten arguments of $work/args +print_run() { + set -- + while IFS= read -r _a; do + set -- "$@" "$_a" + done < "$work/args" + _fmt="%l$TAB%h$TAB%s$TAB%g" + debug "prefetch: $exe --print $*" + rc=0 + if command -v timeout > /dev/null; then + timeout 120 "$exe" --print --print-format "$_fmt" --noconfirm --noprogressbar "$@" \ + < /dev/null > "$work/print.out" 2> "$work/print.err" || rc=$? + else + "$exe" --print --print-format "$_fmt" --noconfirm --noprogressbar "$@" \ + < /dev/null > "$work/print.out" 2> "$work/print.err" || rc=$? + fi + if [ "$rc" -ne 0 ]; then + debug "no prefetch: the print run ended $rc: $(sed -n 1,5p "$work/print.err")" + return 1 + fi +} + +# plan_pkgs: one blob item per file of the print run, a manifest request for +# each file whose sha256 or signature has to come from the registry, and a +# line in $work/sigs for each signature decode_sigs writes from %PGPSIG%. A +# signature decode_sigs cannot write is fetched alone, in the serial pass. +plan_pkgs() { + : > "$work/index" + : > "$work/mnames" + : > "$work/sigs" + mitems='' + _n=0 + while IFS="$TAB" read -r _type _f _h _s _g _b _u; do + _n=$((_n + 1)) + if [ -f "$held_dir/$_f.part" ] && [ ! -L "$held_dir/$_f.part" ]; then + _s=$((_s + $(size_of "$held_dir/$_f.part"))) + fi + if [ "$_h" != - ]; then + printf '%s\t%s\t%s\n' "$_f" "$_h" "$_s" >> "$work/index" + fi + _sig=0 + if [ "$sigmode" = auto ] && [ "$_g" != - ]; then + _sig=1 + printf '%s\t%s\n' "$_f" "$_g" >> "$work/sigs" + fi + if [ "$_type" = http ]; then + add_blob "$_u" "$_h" "$_s" "$_f" 0 '=http,https' + elif [ "$_h" != - ] && [ "$_sig" = 1 ]; then + proto_for "$_b" + add_blob "$_b/blobs/sha256:$_h" "$_h" "$_s" "$_f" 1 "$_p" + else + eval "mb_$_n=\$_b mf_$_n=\$_f mh_$_n=\$_h mg_$_n=\$_sig" + mitems="$mitems $_n" + printf '%s\n' "$_f" >> "$work/mnames" + fi + done < "$work/plan" + if [ -s "$work/index" ]; then + if ! { _t=$(mktemp "${store}tmp.XXXXXX") && cp -- "$work/index" "$_t" && put ,index "$_t"; }; then + debug "prefetch: could not keep the index of sha256 values" + fi + fi +} + +# pkg_manifests: the manifest of each planned file that needs one, then its +# package layer and, when the signature comes from the registry, its .sig layer +pkg_manifests() { + tags_of < "$work/mnames" > "$work/mtags" || return 0 + : > "$work/mlist" + # shellcheck disable=SC2086 # a list of numbers + set -- $mitems + while IFS="$TAB" read -r _f _tag; do + [ "$#" -gt 0 ] || break + eval "_b=\$mb_$1" + printf '%s\t%s/manifests/%s\n' "$1" "$_b" "$_tag" >> "$work/mlist" + shift + done < "$work/mtags" + fetch_manifests "$work/mlist" + while IFS="$TAB" read -r _n _x _code; do + [ "$_x" = 0 ] && [ "$_code" = 200 ] || continue + eval "_b=\$mb_$_n _f=\$mf_$_n _h=\$mh_$_n _sig=\$mg_$_n" + json_flat "$work/m.$_n" > "$work/m.$_n.flat" 2>> "$work/json.err" || continue + layers_of "$work/m.$_n.flat" "$_f" > "$work/m.$_n.layers" || continue + read -r _fc _fd _fs _sc _sd _ss _media < "$work/m.$_n.layers" || continue + case $_media in + application/vnd.oci.image.manifest.v1+json | -) ;; + *) continue ;; + esac + [ "$_fc" = 1 ] && [ "$_fd" != - ] && [ "$_fs" != - ] || continue + if [ "$_h" != - ] && [ "$_fd" != "sha256:$_h" ]; then + debug "prefetch: the registry's $_f is $_fd, the database says sha256:$_h" + continue + fi + proto_for "$_b" + add_blob "$_b/blobs/$_fd" "${_fd#sha256:}" "$_fs" "$_f" 1 "$_p" + if [ "$_sig" = 0 ] && [ "$_sc" = 1 ] && [ "$_sd" != - ] && [ "$_ss" != - ]; then + add_blob "$_b/blobs/$_sd" "${_sd#sha256:}" "$_ss" "$_f.sig" 1 "$_p" + fi + done < "$work/m.res" +} + +pkg_prefetch() { + if ! store_open || ! put ,prefetch; then + return 0 + fi + if ! argv_scan; then + debug "no prefetch: $scan_why" + return 0 + fi + if [ "$scan_sync" != 1 ] || [ "$scan_print" != 0 ]; then + debug "no prefetch: pacman is not in a sync operation, or prints only" + return 0 + fi + if ! find_exe; then + debug "no prefetch: no pacman binary found for argv[0] '$scan_argv0'" + return 0 + fi + _t0=$(now_ns) + print_run || return 0 + debug "prefetch: the print run ended" + awk "$PLAN_AWK" "$work/print.out" > "$work/plan" || return 0 + plan_pkgs + # The signatures are decoded while the downloads run. + _spid='' + if [ -s "$work/sigs" ]; then + decode_sigs & + _spid=$! + fi + if [ -n "$mitems" ]; then + pkg_manifests + fi + fetch_blobs + sigs_from_db=0 + if [ -n "$_spid" ]; then + wait "$_spid" || debug "prefetch: the signature decoder ended $?" + while IFS= read -r _l; do + sigs_from_db=$((sigs_from_db + 1)) + done < "$work/sigs.ok" + fi + say_summary "$_t0" "$((got_files + sigs_from_db))" "$got_bytes" +} + +#---------------------------------------------------------------------------# +# Database prefetch +# +# The config file comes from pacman's arguments (default /etc/pacman.conf), the +# repositories and their first Server from pacman-conf. For each first Server +# that is oci://, the manifests in parallel, then each database (.db or .files, +# as pacman asked) that differs from the copy pacman holds, and its signature. +# A manifest asked for by digest must hash to that digest. A database whose +# manifest has no signature layer gets an ,absent entry, so the request for its +# .sig misses with no network. +#---------------------------------------------------------------------------# +db_prefetch() { + if ! store_open || ! put ,dbprefetch; then + return 0 + fi + if ! argv_scan; then + debug "no prefetch: $scan_why" + return 0 + fi + if [ "$scan_sysroot" = 1 ] || [ "$scan_arch" = 1 ]; then + debug "no prefetch: pacman runs with --sysroot or --arch" + return 0 + fi + if ! command -v pacman-conf > /dev/null; then + debug "no prefetch: no pacman-conf" + return 0 + fi + _t0=$(now_ns) + _ext=${file##*.} + rc=0 + pacman-conf --config "$scan_config" --repo-list > "$work/repos" 2> "$work/conf.err" || rc=$? + if [ "$rc" -ne 0 ]; then + debug "no prefetch: pacman-conf --repo-list ended $rc: $(sed -n 1,3p "$work/conf.err")" + return 0 + fi + : > "$work/mlist" + _n=0 + while IFS= read -r _r; do + case $_r in + '' | ,* | *[!A-Za-z0-9._+-]*) continue ;; + esac + rc=0 + _srv=$(pacman-conf --config "$scan_config" --repo "$_r" Server < /dev/null 2>> "$work/conf.err") || rc=$? + [ "$rc" -eq 0 ] || continue + _srv=${_srv%%"$NL"*} + case $_srv in + oci://* | oci+http://*) ;; + *) continue ;; + esac + parse_oci "$_srv/$_r.$_ext" || continue + _n=$((_n + 1)) + eval "dr_$_n=\$_r db_$_n=\$p_base dd_$_n=\$p_digest" + printf '%s\t%s/manifests/%s\n' "$_n" "$p_base" "$p_ref" >> "$work/mlist" + done < "$work/repos" + [ "$_n" -gt 0 ] || return 0 + fetch_manifests "$work/mlist" + _repos='' + while IFS="$TAB" read -r _n _x _code; do + [ "$_x" = 0 ] && [ "$_code" = 200 ] || continue + eval "_r=\$dr_$_n _b=\$db_$_n _d=\$dd_$_n" + if [ -n "$_d" ] && [ "sha256:$(sha256_of "$work/m.$_n")" != "$_d" ]; then + printf 'pacman-oci: %s answered %s with a manifest of another digest\n' "$_b" "$_d" >&2 + continue + fi + json_flat "$work/m.$_n" > "$work/m.$_n.flat" 2>> "$work/json.err" || continue + layers_of "$work/m.$_n.flat" "$_r.$_ext" > "$work/m.$_n.layers" || continue + read -r _fc _fd _fs _sc _sd _ss _media < "$work/m.$_n.layers" || continue + case $_media in + application/vnd.oci.image.manifest.v1+json | -) ;; + *) continue ;; + esac + [ "$_fc" = 1 ] && [ "$_fd" != - ] && [ "$_fs" != - ] && [ "$_sc" -le 1 ] || continue + proto_for "$_b" + _held=$held_dir/$_r.$_ext + if [ -f "$_held" ] && check_file "$_held" "$_fs" "${_fd#sha256:}" && + _t=$(mktemp "${store}tmp.XXXXXX") && cp -- "$_held" "$_t" && put "$_r.$_ext" "$_t"; then + debug "$_r.$_ext unchanged, $_fd" + got_files=$((got_files + 1)) + else + add_blob "$_b/blobs/$_fd" "${_fd#sha256:}" "$_fs" "$_r.$_ext" 1 "$_p" + fi + if [ "$_sc" = 1 ] && [ "$_sd" != - ] && [ "$_ss" != - ]; then + add_blob "$_b/blobs/$_sd" "${_sd#sha256:}" "$_ss" "$_r.$_ext.sig" 1 "$_p" + eval "ds_$_n=1" + else + eval "ds_$_n=0" + fi + _repos="$_repos $_n" + done < "$work/m.res" + fetch_blobs + # A signature is kept only beside the database of its own manifest. + for _n in $_repos; do + eval "_r=\$dr_$_n _s=\$ds_$_n" + if entry_ok "$_r.$_ext"; then + if [ "$_s" = 0 ]; then + put ",absent.$_r.$_ext.sig" || debug "could not mark $_r.$_ext.sig absent" + fi + elif entry_ok "$_r.$_ext.sig"; then + rm -f -- "$store$_r.$_ext.sig" + fi + done + say_summary "$_t0" "$got_files" "$got_bytes" +} + +#---------------------------------------------------------------------------# +# Parse oci://HOST/REPOSITORY[:TAG|@DIGEST]/FILE +# +# parse_oci URL sets p_scheme, p_host, p_repo, p_tag, p_digest, p_base, p_ref +# (what a database is read from) and p_proto. It returns 1, with the reason in +# p_why, for a URL that is not one. +#---------------------------------------------------------------------------# +parse_oci() { + case $1 in + oci://*) + p_scheme=https + _rest=${1#oci://} + ;; + oci+http://*) + p_scheme=http + _rest=${1#oci+http://} + ;; + *) + p_why="not an oci:// URL: $1" + return 1 + ;; + esac + _ref=${_rest%/*} + p_host=${_ref%%/*} + _path=${_ref#*/} + if [ "$_ref" = "$_rest" ] || [ "$p_host" = "$_ref" ]; then + p_why="not oci://HOST/REPOSITORY/FILE: $1" + return 1 + fi + p_tag='' + p_digest='' + case $_path in + *@*) + p_digest=${_path#*@} + p_repo=${_path%%@*} + ;; + *) + case ${_path##*/} in + *:*) + p_tag=${_path##*:} + p_repo=${_path%:*} + ;; + *) p_repo=$_path ;; + esac + ;; + esac + if ! printf '%s\n' "$p_host" | grep -Eq '^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?(:[0-9]+)?$'; then + p_why="not a registry host: $p_host" + return 1 + fi + if ! printf '%s\n' "$p_repo" | grep -Eq '^[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*$'; then + p_why="not a repository name: $p_repo" + return 1 + fi + if [ -n "$p_tag" ] && ! printf '%s\n' "$p_tag" | grep -Eq '^[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$'; then + p_why="not a tag: $p_tag" + return 1 + fi + if [ -n "$p_digest" ] && ! printf '%s\n' "$p_digest" | grep -Eq '^sha256:[0-9a-f]{64}$'; then + p_why="not a sha256 digest: $p_digest" + return 1 + fi + p_base=$p_scheme://$p_host/v2/$p_repo + p_ref=${p_digest:-${p_tag:-latest}} + proto_for "$p_base" + p_proto=$_p + return 0 +} + +#---------------------------------------------------------------------------# +# One file +# +# fetch_one URL SIZE AUTH(0|1) PROTO RESUMABLE(0|1): OUTPUT from URL in up to +# three attempts. A resumable file keeps what it got and the next attempt +# resumes with --continue-at -; anything else starts from zero. When SIZE is +# known, each attempt has its time limit (see deadline). No attempt +# starts after 30 s, unless the one before it added bytes. Sets one_rc (curl's +# exit), one_code (the last HTTP status) and one_resumed. +#---------------------------------------------------------------------------# +fetch_one() { + _url=$1 _max=$2 _auth=$3 _proto=$4 _resumable=$5 + _try=1 + _stop=$(($(date +%s) + 30)) + while :; do + one_resumed=0 + _had=0 + set -- + if [ "$_resumable" = 1 ] && [ "${no_resume:-0}" != 1 ] && [ -s "$out" ]; then + _had=$(size_of "$out") + fi + if [ "$_had" -gt 0 ] && { [ -z "$_max" ] || [ "$_had" -lt "$_max" ]; }; then + set -- --continue-at - + one_resumed=1 + else + _had=0 + rm -f -- "$out" + fi + if [ -n "$_max" ]; then + set -- "$@" --max-filesize "$_max" + deadline "$_max" "$_try" + if [ -n "$_dl" ]; then + set -- "$@" --max-time "$_dl" + fi + fi + if [ -n "$_proto" ]; then + set -- "$@" --proto "$_proto" --proto-redir "$_proto" --max-redirs 5 + else + set -- "$@" --max-redirs 10 + fi + if [ "$_auth" = 1 ]; then + set -- "$@" --user-agent "pacman-oci/$PACMAN_OCI_VERSION" + if [ -n "$AUTH" ]; then + set -- "$@" --header "Authorization: $AUTH" + fi + fi + one_rc=0 + one_code=$(curl -q --fail --location --show-error "$PROGRESS" \ + --connect-timeout 20 --speed-limit 1024 --speed-time 60 \ + --write-out '%{http_code}' "$@" --output "$out" --url "$_url") || one_rc=$? + [ -n "$one_code" ] || one_code=000 + debug "GET $_url -> $one_code (curl $one_rc, resumed $one_resumed)" + [ "$one_rc" -ne 0 ] || return 0 + case $one_rc:$one_code in + 33:*) rm -f -- "$out" ;; + *:408 | *:429 | *:5?? | 6:* | 16:* | 18:* | 28:* | 35:* | 52:* | 55:* | 56:* | 92:*) ;; + *) return 1 ;; + esac + [ "$_try" -lt 3 ] || return 1 + if [ "$(date +%s)" -ge "$_stop" ]; then + if [ "$_resumable" != 1 ] || [ ! -s "$out" ] || [ "$(size_of "$out")" -le "$_had" ]; then + return 1 + fi + fi + sleep "$_try" + _try=$((_try + 1)) + done +} + +#---------------------------------------------------------------------------# +# The run store first: a file the prefetch made ready needs no network. +#---------------------------------------------------------------------------# +sweep +store_ok=0 +if [ -n "$key" ]; then + case $file in + '' | ,* | *[!!-~]*) ;; + *) store_ok=1 ;; + esac fi -if [ -n "$ref_digest" ]; then - printf '%s\n' "$ref_digest" | grep -Eq '^sha256:[0-9a-f]{64}$' || broken "not a sha256 digest: $ref_digest" +if [ "$store_ok" = 1 ]; then + case $kind in + pkg | pkgsig) + serve "$file" + if prefetch_allowed && ! entry_ok ,prefetch; then + pkg_prefetch + serve "$file" + fi + ;; + db) + if [ "$ours" = 1 ]; then + serve "$file" + case $file in + *.db | *.files) + if prefetch_allowed && ! entry_ok ,dbprefetch; then + db_prefetch + serve "$file" + fi + ;; + esac + fi + ;; + dbsig) + if [ "$ours" = 1 ]; then + serve "$file" + if entry_ok ",absent.$file"; then + miss "no layer $file in the database manifest of this run" + fi + fi + ;; + esac + # A host that failed earlier in this run is skipped, never for a signature: + # pacman asks for a .sig only of the server that served its file. + case $kind in + pkg | db) + host_of "$url" + if [ -n "$_host" ] && entry_ok ",down.$_host"; then + miss "$_host failed earlier in this run; pacman tries the next Server" + fi + ;; + esac +fi + +#---------------------------------------------------------------------------# +# Not ours: an ordinary download. pacman's own downloader is off whenever an +# XferCommand is set, so this is what fetches from the fallback mirrors. A +# package resumes what it got. When the prefetch of this run planned the +# package, its size bounds the download, and a resumed file must have its +# sha256, or it is fetched once more from zero. +#---------------------------------------------------------------------------# +if [ "$ours" = 0 ]; then + want='' want_size='' resumable=0 + if [ "$kind" = pkg ]; then + resumable=1 + if entry_ok ,index; then + want=$(F="$file" awk -F '\t' '$1 == ENVIRON["F"] { print $2 "\t" $3; exit }' "$store,index") + want_size=${want#*"$TAB"} + want=${want%%"$TAB"*} + fi + fi + if ! fetch_one "$url" "$want_size" 0 '' "$resumable"; then + host_of "$url" + mark_down "$_host" "$one_code" + rm -f -- "$out" + exit 1 + fi + if [ "$one_resumed" = 1 ] && [ -n "$want" ] && [ "$(sha256_of "$out")" != "$want" ]; then + debug "$file: the resumed file is not sha256 $want, once more from zero" + no_resume=1 + if ! fetch_one "$url" "$want_size" 0 '' 0; then + rm -f -- "$out" + exit 1 + fi + fi + exit 0 fi + +parse_oci "$url" || broken "$p_why" +host=$p_host +repo=$p_repo +scheme=$p_scheme +base=$p_base +PROTO=$p_proto # No shell ever reads FILE: it becomes a tag, which the mapping escapes or # hashes, and reaches awk through the environment. A name makepkg could never # produce is refused: empty, a space, a control character, or a byte above 0x7e. @@ -377,43 +1693,14 @@ case $file in '' | . | .. | *[!!-~]*) broken "not a file name pacman asks for: $file" ;; esac -sha256_of() { - sha256sum "$1" | cut -d ' ' -f 1 -} - -size_of() { - wc -c < "$1" | tr -d ' ' -} - -# pacman writes OUTPUT in its sync or cache dir, or, with DownloadUser set, in -# a download-XXXXXX dir inside it. Either way the files it finished earlier in -# this run sit beside OUTPUT, and the ones it held before sit one level up. -out_dir=$(dirname -- "$out") -case $(basename -- "$out_dir") in - download-*) held_dir=$(dirname -- "$out_dir") ;; - *) held_dir=$out_dir ;; -esac - -# What to ask the registry for, and what must be in the answer. -# pkg a package: its own tag, a layer named FILE -# pkgsig its signature: the package's tag, a layer named FILE, never optional -# dbsig a database signature: the manifest of the database pacman just took, -# found by that database's content (tag db-HEX or files-HEX) -# db a database: the tag, digest or latest from the URL +# What to ask the registry for, and what must be in the answer (see kind). pair_hex="" -case $file in - *.pkg.tar.sig | *.pkg.tar.*.sig) - kind=pkgsig - manifest_ref=$(oci_tag "${file%.sig}") - ;; - *.pkg.tar | *.pkg.tar.*) - kind=pkg - manifest_ref=$(oci_tag "$file") - ;; - *.db.sig | *.files.sig) - kind=dbsig +case $kind in + pkgsig) manifest_ref=$(oci_tag "${file%.sig}") ;; + pkg) manifest_ref=$(oci_tag "$file") ;; + dbsig) sibling="$out_dir/${file%.sig}" - manifest_ref=${ref_digest:-${ref_tag:-latest}} + manifest_ref=$p_ref if [ -f "$sibling" ]; then pair_hex=$(sha256_of "$sibling") case $file in @@ -422,27 +1709,10 @@ case $file in esac fi ;; - *) - kind=db - manifest_ref=${ref_digest:-${ref_tag:-latest}} - ;; + *) manifest_ref=$p_ref ;; esac -base="$scheme://$host/v2/$repo" -if [ "$scheme" = https ]; then - PROTO='=https' -else - PROTO='=http,https' -fi - -# With DownloadUser set, this runs as that user with root's environment, so a -# TMPDIR only root can write is possible; the directory pacman writes OUTPUT to -# is always writable by whoever runs this. -tmp_base=${TMPDIR:-/tmp} -if [ ! -d "$tmp_base" ] || [ ! -w "$tmp_base" ]; then - tmp_base=$out_dir -fi -work=$(mktemp -d "$tmp_base/pacman-oci.XXXXXX") || broken "could not make a temporary directory in $tmp_base" +need_work work_hdr="$work/headers" work_json="$work/body.json" work_flat="$work/body.flat" @@ -457,15 +1727,12 @@ work_flat="$work/body.flat" # names is fetched once. Manifest and token requests follow no redirect: GHCR # answers a reference it cannot parse with 303 to an HTML page. #---------------------------------------------------------------------------# -AUTH='Bearer QQ==' -if [ "${PACMAN_OCI_NO_DUMMY:-0}" = 1 ]; then - AUTH='' -fi # get URL FILE: the body into FILE, the HTTP status into $code (000 when -# nothing answered), the headers into $work_hdr +# nothing answered, cut when the body was not read whole), the headers into +# $work_hdr get() { - code=000 + get_rc=0 if [ -n "$AUTH" ]; then # shellcheck disable=SC2086 # RETRY is several words on purpose code=$(curl -q --silent --show-error --proto "$PROTO" --max-filesize 65536 \ @@ -473,16 +1740,21 @@ get() { --user-agent "pacman-oci/$PACMAN_OCI_VERSION" \ --header "Authorization: $AUTH" \ --header 'Accept: application/vnd.oci.image.manifest.v1+json' \ - --dump-header "$work_hdr" --output "$2" --write-out '%{http_code}' --url "$1") || code=000 + --dump-header "$work_hdr" --output "$2" --write-out '%{http_code}' --url "$1") || get_rc=$? else # shellcheck disable=SC2086 # RETRY is several words on purpose code=$(curl -q --silent --show-error --proto "$PROTO" --max-filesize 65536 \ --connect-timeout 10 --max-time 15 $RETRY \ --user-agent "pacman-oci/$PACMAN_OCI_VERSION" \ --header 'Accept: application/vnd.oci.image.manifest.v1+json' \ - --dump-header "$work_hdr" --output "$2" --write-out '%{http_code}' --url "$1") || code=000 + --dump-header "$work_hdr" --output "$2" --write-out '%{http_code}' --url "$1") || get_rc=$? fi - debug "GET $1 -> $code" + case $get_rc:$code in + 0:???) ;; + *:000 | *:) code='000' ;; + *) code='cut' ;; + esac + debug "GET $1 -> $code (curl $get_rc)" } # The anonymous pull token named by the Bearer challenge in $work_hdr. The @@ -497,13 +1769,15 @@ challenge() { http://*) [ "$scheme" = http ] || return 1 ;; *) return 1 ;; esac - debug "token from $_realm service=$_service scope=repository:$repo:pull" + _trc=0 # shellcheck disable=SC2086 # RETRY is several words on purpose curl -q --silent --show-error --fail --proto "$PROTO" --max-filesize 65536 \ --connect-timeout 10 --max-time 15 $RETRY \ --user-agent "pacman-oci/$PACMAN_OCI_VERSION" \ --get --data-urlencode "service=$_service" --data-urlencode "scope=repository:$repo:pull" \ - --output "$work_json" --url "$_realm" || return 1 + --output "$work_json" --url "$_realm" || _trc=$? + debug "GET $_realm service=$_service scope=repository:$repo:pull -> token (curl $_trc)" + [ "$_trc" -eq 0 ] || return 1 json_flat "$work_json" > "$work_flat" || return 1 _token=$(flat_get "$work_flat" /token) [ -n "$_token" ] || _token=$(flat_get "$work_flat" /access_token) @@ -513,7 +1787,8 @@ challenge() { } # fetch_manifest REF: the body into $work_json, the status into $code. Runs in -# this shell, never in $( ), so a token won here is the one the blob uses. +# this shell, never in $( ), so a token won here is the one the blob uses. A +# 303 is asked again once, after 1 s: every REF here passed its check. fetch_manifest() { get "$base/manifests/$1" "$work_json" case $code in @@ -530,6 +1805,10 @@ fetch_manifest() { fi ;; esac + if [ "$code" = 303 ]; then + sleep 1 + get "$base/manifests/$1" "$work_json" + fi } fetch_manifest "$manifest_ref" @@ -537,7 +1816,7 @@ fetch_manifest "$manifest_ref" # A database signature whose content tag is missing: read the URL's own # manifest, and use it only if it names the very database pacman holds. if [ "$kind" = dbsig ] && [ -n "$pair_hex" ] && [ "$code" = 404 ]; then - manifest_ref=${ref_digest:-${ref_tag:-latest}} + manifest_ref=$p_ref fetch_manifest "$manifest_ref" fi @@ -545,8 +1824,15 @@ case $code in 200) ;; 404) miss "no manifest $manifest_ref in $host/$repo" ;; 401 | 403) broken "the registry refused to serve $host/$repo:$manifest_ref (HTTP $code); is the package public?" ;; - 000) miss "no answer from $host" ;; - *) miss "HTTP $code for $host/$repo:$manifest_ref" ;; + 000) + mark_down "$host" 000 + miss "no answer from $host" + ;; + cut) miss "the manifest $host/$repo:$manifest_ref was not read whole" ;; + *) + mark_down "$host" "$code" + miss "HTTP $code for $host/$repo:$manifest_ref" + ;; esac # A manifest asked for by digest must be that digest, or pinning means nothing. @@ -624,32 +1910,27 @@ max_size=$size if [ "$max_size" -eq 0 ]; then max_size=1 fi -rc=0 -if [ -n "$AUTH" ]; then - # shellcheck disable=SC2086 # RETRY is several words on purpose - curl -q --fail --location --show-error "$PROGRESS" --proto "$PROTO" --proto-redir "$PROTO" \ - --max-redirs 5 --max-filesize "$max_size" --connect-timeout 20 --speed-limit 1024 --speed-time 60 \ - $RETRY \ - --user-agent "pacman-oci/$PACMAN_OCI_VERSION" \ - --header "Authorization: $AUTH" \ - --output "$out" --url "$base/blobs/$digest" || rc=$? -else - # shellcheck disable=SC2086 # RETRY is several words on purpose - curl -q --fail --location --show-error "$PROGRESS" --proto "$PROTO" --proto-redir "$PROTO" \ - --max-redirs 5 --max-filesize "$max_size" --connect-timeout 20 --speed-limit 1024 --speed-time 60 \ - $RETRY \ - --user-agent "pacman-oci/$PACMAN_OCI_VERSION" \ - --output "$out" --url "$base/blobs/$digest" || rc=$? -fi -if [ "$rc" -ne 0 ]; then - miss "could not read blob $digest for $file (curl $rc)" -fi -got_size=$(size_of "$out") +# blob: the blob into OUTPUT; 0 when its size and sha256 are the descriptor's +blob() { + if ! fetch_one "$base/blobs/$digest" "$max_size" 1 "$PROTO" 1; then + mark_down "$host" "$one_code" + miss "could not read blob $digest for $file (curl $one_rc, HTTP $one_code)" + fi + got_size=$(size_of "$out") + got=$(sha256_of "$out") + [ "$got_size" = "$size" ] && [ "$got" = "$hex" ] +} + +# A resumed file that fails its check starts again from zero, once. +if ! blob && [ "$one_resumed" = 1 ]; then + debug "$file: the resumed file is not sha256 $hex, once more from zero" + no_resume=1 + blob +fi if [ "$got_size" != "$size" ]; then broken "$file from $host/$repo is $got_size bytes, the manifest says $size" fi -got=$(sha256_of "$out") if [ "$got" != "$hex" ]; then broken "$file from $host/$repo has sha256 $got, the manifest says $hex" fi diff --git a/scripts/bench-consumer b/scripts/bench-consumer new file mode 100755 index 0000000..db85cb7 --- /dev/null +++ b/scripts/bench-consumer @@ -0,0 +1,333 @@ +#!/usr/bin/env bash +# +# Measure how fast a consumer reads the mirror: pacman's own downloader on an +# official mirror and on the Worker, against client/pacman-oci with its +# prefetch and without it. +# +# scripts/bench-consumer OUT.json +# +# Runs on this host when pacman is here and this runs as root. Otherwise it +# runs itself again inside the pinned archlinux:base container, through +# CONTAINER_RUNTIME (docker or podman; docker by default), with the network on. +# +# Cases, each with a throwaway root, database path, cache and config: +# official https://geo.mirror.pkgbuild.com, pacman's downloader +# worker https://ghcr.mirror.archlinux.pkgforge.dev, pacman's downloader +# oci oci://ghcr.io/pkgforge-dev/mirror-archlinux, client/pacman-oci +# of this checkout, prefetch on +# oci-serial the same with PACMAN_OCI_PREFETCH=0 +# Operations, timed one by one, every case in turn in each round: +# sy pacman -Sy into an empty database path +# sw-small pacman -Swdd of the BENCH_SMALL smallest core packages +# sw-large pacman -Swdd of the BENCH_LARGE largest core packages +# The packages are those whose file name is in both the mirror's core database +# and upstream's, so every case downloads the same files. They are chosen by +# size and passed to pacman by name. +# +# Settings every case shares: ParallelDownloads = 5; SigLevel = Required +# DatabaseNever (Arch publishes no database signature; with an XferCommand +# pacman pauses 2 s after each missing one, which is pacman's cost, not the +# download's); package signatures checked against a throwaway keyring, first +# refreshed from the current archlinux-keyring package. The two oci cases set +# DownloadUser = alpm, as stock Arch does. The two native cases set none: +# pacman's own sandbox for its downloader needs Landlock, which a container +# without CAP_SYS_ADMIN refuses (measured: podman on WSL2, pacman 7.1). +# +# Environment: +# BENCH_ARCH pacman's $arch (default x86_64) +# BENCH_SMALL number of small packages (default 40) +# BENCH_LARGE number of large packages (default 3) +# BENCH_ROUNDS rounds; seconds is their median (default 3) +# CONTAINER_RUNTIME docker or podman, when this host is not Arch +# BENCH_HOST the "host" value; set by the outer run for the inner +# +# OUT.json: +# {"schema":1,"date":"","host":"...","pacman":"", +# "cases":[{"path":"official|worker|oci|oci-serial","op":"sy|sw-small|sw-large", +# "seconds":,"rounds":[...],"files":,"bytes":,"ok":true}]} +# files and bytes are what the last round downloaded; ok is false when any +# round failed or downloaded a different number of files. +# +# Exit status: 0 measured (a failed case has "ok":false), 1 could not run, +# 2 usage. +set -euo pipefail +# No exit is silent: a command that ends the script under set -e is named. +trap 'printf "bench-consumer: line %s: \"%s\" ended %s\n" "$LINENO" "$BASH_COMMAND" "$?" >&2' ERR +# Sorting and join must agree on one order. +export LC_ALL=C + +# tag: base +ARCH_IMAGE=docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ARCH="${BENCH_ARCH:-x86_64}" +SMALL="${BENCH_SMALL:-40}" +LARGE="${BENCH_LARGE:-3}" +ROUNDS="${BENCH_ROUNDS:-3}" +OP_TIMEOUT=1800 + +say() { + printf 'bench-consumer: %s\n' "$1" >&2 +} + +die() { + say "$1" + shift + while [ "$#" -gt 0 ]; do + printf ' %s\n' "$1" >&2 + shift + done + exit 1 +} + +out="${1:-}" +if [ -z "$out" ] || [ "$#" -ne 1 ]; then + printf 'usage: scripts/bench-consumer OUT.json\n' >&2 + exit 2 +fi +for n in "$SMALL" "$LARGE" "$ROUNDS"; do + case "$n" in + '' | *[!0-9]* | 0) die "BENCH_SMALL, BENCH_LARGE and BENCH_ROUNDS are counts above 0, not '$n'" ;; + esac +done +# shellcheck disable=SC2016 # $arch and $repo are pacman's, not the shell's +case "$ARCH" in + x86_64) OFFICIAL='https://geo.mirror.pkgbuild.com/$repo/os/$arch' ;; + aarch64 | armv7h) OFFICIAL='http://fl.us.mirror.archlinuxarm.org/$arch/$repo' ;; + *) die "no official mirror is known for BENCH_ARCH=$ARCH" ;; +esac +# shellcheck disable=SC2016 # $arch and $repo are pacman's, not the shell's +WORKER='https://ghcr.mirror.archlinux.pkgforge.dev/$arch/$repo' +# shellcheck disable=SC2016 # $arch and $repo are pacman's, not the shell's +OCI='oci://ghcr.io/pkgforge-dev/mirror-archlinux/$arch/$repo' + +#---------------------------------------------------------------------------# +# Not Arch, or not root: the same script inside the pinned container +#---------------------------------------------------------------------------# +if ! command -v pacman > /dev/null || [ "$(id -u)" -ne 0 ]; then + RUNTIME="${CONTAINER_RUNTIME:-docker}" + command -v "$RUNTIME" > /dev/null || die "no pacman here, and no $RUNTIME to run $ARCH_IMAGE" \ + "reproduce: CONTAINER_RUNTIME=podman scripts/bench-consumer OUT.json" + out_dir="$(cd "$(dirname "$out")" && pwd)" + rc=0 + timeout --kill-after=60 21600 "$RUNTIME" run --rm \ + -e BENCH_ARCH="$ARCH" -e BENCH_SMALL="$SMALL" -e BENCH_LARGE="$LARGE" -e BENCH_ROUNDS="$ROUNDS" \ + -e BENCH_HOST="$(uname -n) ($(uname -sm)), in archlinux:base through $(basename "$RUNTIME")" \ + -v "$REPO_ROOT:/src:ro" -v "$out_dir:/out" \ + "$ARCH_IMAGE" bash /src/scripts/bench-consumer "/out/$(basename "$out")" || rc=$? + [ "$rc" -eq 0 ] || die "the run in $ARCH_IMAGE ended $rc" "reproduce: CONTAINER_RUNTIME=$RUNTIME scripts/bench-consumer $out" + exit 0 +fi + +for tool in curl bsdtar sha256sum awk timeout pacman-key; do + command -v "$tool" > /dev/null || die "$tool is missing" +done +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +# The download user runs the wrapper and pacman's print run from here. +chmod 0755 "$work" +install -m 0755 "$REPO_ROOT/client/pacman-oci" "$work/pacman-oci" + +now_ns() { + date +%s%N +} + +#---------------------------------------------------------------------------# +# Configs, keyring, package list +#---------------------------------------------------------------------------# +# conf CASE: the config file of one case +conf() { + local f="$work/$1.conf" + { + printf '[options]\nArchitecture = %s\nSigLevel = Required DatabaseNever\n' "$ARCH" + printf 'LocalFileSigLevel = Optional\nParallelDownloads = 5\nGPGDir = %s/gnupg\n' "$work" + printf 'HookDir = %s/hooks\nLogFile = %s/%s.log\n' "$work" "$work" "$1" + case "$1" in + oci | oci-serial) printf 'DownloadUser = alpm\nXferCommand = %s/pacman-oci %%o %%u\n' "$work" ;; + esac + printf '\n[core]\n' + case "$1" in + official | keyring) printf 'Server = %s\n' "$OFFICIAL" ;; + worker) printf 'Server = %s\n' "$WORKER" ;; + oci | oci-serial) printf 'Server = %s\n' "$OCI" ;; + esac + } > "$f" + printf '%s\n' "$f" +} +mkdir -p "$work/hooks" + +# pac CASE ROOT ARGS...: pacman for one case, with its own root, database path +# and cache under ROOT +pac() { + local c="$1" r="$2" + shift 2 + if [ "$c" = oci-serial ]; then + PACMAN_OCI_PREFETCH=0 timeout --kill-after=30 "$OP_TIMEOUT" pacman --config "$work/$c.conf" \ + --root "$r/root" --dbpath "$r/db" --noconfirm "$@" + else + timeout --kill-after=30 "$OP_TIMEOUT" pacman --config "$work/$c.conf" \ + --root "$r/root" --dbpath "$r/db" --noconfirm "$@" + fi +} + +for c in keyring official worker oci oci-serial; do + conf "$c" > /dev/null +done + +say "keyring: the image's, then the current archlinux-keyring from $OFFICIAL" +pacman-key --gpgdir "$work/gnupg" --init > "$work/keyring.log" 2>&1 +pacman-key --gpgdir "$work/gnupg" --populate >> "$work/keyring.log" 2>&1 +mkdir -p "$work/k/root" "$work/k/db" "$work/k/cache" +rc=0 +{ + pac keyring "$work/k" -Sy && + pac keyring "$work/k" --cachedir "$work/k/cache" -Swdd archlinux-keyring +} >> "$work/keyring.log" 2>&1 || rc=$? +if [ "$rc" -eq 0 ]; then + mkdir -p "$work/keyrings" + bsdtar -xf "$work"/k/cache/archlinux-keyring-*.pkg.tar.zst -C "$work/keyrings" usr/share/pacman/keyrings + pacman-key --gpgdir "$work/gnupg" --populate-from "$work/keyrings/usr/share/pacman/keyrings" --populate \ + >> "$work/keyring.log" 2>&1 || rc=$? +fi +if [ "$rc" -ne 0 ]; then + say "could not refresh the keyring (exit $rc); the image's keyring is used: $(tail -n 2 "$work/keyring.log" | tr '\n' ' ')" +fi + +# names_by_size DB: FILENAMENAMECSIZE of every package in DB. Each +# package's desc starts with %FILENAME%; a blank line ends one field. +names_by_size() { + bsdtar -xOf "$1" | awk ' + function emit() { if (f != "" && n != "" && c != "") print f "\t" n "\t" c; f = n = c = "" } + /^%FILENAME%$/ { emit(); k = "f"; next } + /^%NAME%$/ { k = "n"; next } + /^%CSIZE%$/ { k = "c"; next } + /^%/ || /^$/ { k = ""; next } + k == "f" { f = $0 } + k == "n" { n = $0 } + k == "c" { c = $0 } + END { emit() } + ' | sort -u +} + +say "package list: core files present in the mirror and upstream" +upstream_db_url="${OFFICIAL//\$repo/core}" +upstream_db_url="${upstream_db_url//\$arch/$ARCH}/core.db" +rc=0 +curl -q -fsS --connect-timeout 20 --max-time 300 -o "$work/upstream.db" "$upstream_db_url" || rc=$? +[ "$rc" -eq 0 ] || die "could not read $upstream_db_url (curl $rc)" "reproduce: curl -fsSO $upstream_db_url" +mirror_db_url="oci://ghcr.io/pkgforge-dev/mirror-archlinux/$ARCH/core/core.db" +rc=0 +"$work/pacman-oci" "$work/mirror.db" "$mirror_db_url" || rc=$? +[ "$rc" -eq 0 ] || die "could not read $mirror_db_url (pacman-oci $rc)" \ + "reproduce: PACMAN_OCI_DEBUG=1 client/pacman-oci core.db $mirror_db_url" +names_by_size "$work/upstream.db" > "$work/upstream.tsv" +names_by_size "$work/mirror.db" > "$work/mirror.tsv" +# Both lists are sorted whole; join keeps the file names in both. Then by size: +# the SMALL first and the LARGE last, chosen by position, every line read. +join -t "$(printf '\t')" "$work/upstream.tsv" "$work/mirror.tsv" | sort -t "$(printf '\t')" -k3,3n -k1,1 > "$work/both.tsv" +total="$(awk 'END { print NR + 0 }' "$work/both.tsv")" +[ "$total" -ge $((SMALL + LARGE)) ] || die "only $total core files are in both the mirror and upstream; $((SMALL + LARGE)) are needed" +awk -F '\t' -v n="$SMALL" 'NR <= n { print $2 }' "$work/both.tsv" > "$work/small.txt" +awk -F '\t' -v n="$LARGE" -v t="$total" 'NR > t - n { print $2 }' "$work/both.tsv" > "$work/large.txt" +small_bytes="$(awk -F '\t' -v n="$SMALL" 'NR <= n { s += $3 } END { print s + 0 }' "$work/both.tsv")" +large_bytes="$(awk -F '\t' -v n="$LARGE" -v t="$total" 'NR > t - n { s += $3 } END { print s + 0 }' "$work/both.tsv")" +say "$SMALL small packages ($small_bytes bytes), $LARGE large ($large_bytes bytes), of $total in both" + +#---------------------------------------------------------------------------# +# The rounds +#---------------------------------------------------------------------------# +CASES="official worker oci oci-serial" +OPS="sy sw-small sw-large" +: > "$work/results.tsv" + +# measure CASE OP ROUND: one timed operation; a line CASE OP ROUND SECONDS +# FILES BYTES OK into results.tsv +measure() { + local c="$1" op="$2" r="$3" dir="$work/run/$1/$3" t0 t1 rc=0 files bytes ok=true want + mkdir -p "$dir/root" "$dir/db" + case "$op" in + sy) + t0="$(now_ns)" + pac "$c" "$dir" -Sy > "$dir/sy.log" 2>&1 || rc=$? + t1="$(now_ns)" + files=0 + bytes=0 + if [ -f "$dir/db/sync/core.db" ]; then + files=1 + bytes="$(wc -c < "$dir/db/sync/core.db" | tr -d ' ')" + fi + want=1 + ;; + sw-small | sw-large) + mkdir -p "$dir/$op" + local names=() + mapfile -t names < "$work/${op#sw-}.txt" + t0="$(now_ns)" + pac "$c" "$dir" --cachedir "$dir/$op" -Swdd "${names[@]}" > "$dir/$op.log" 2>&1 || rc=$? + t1="$(now_ns)" + files="$(find "$dir/$op" -maxdepth 1 -type f -name '*.pkg.tar*' | wc -l | tr -d ' ')" + bytes="$(find "$dir/$op" -maxdepth 1 -type f -name '*.pkg.tar*' -printf '%s\n' | awk '{ s += $1 } END { print s + 0 }')" + # A package and its signature each + want=$((2 * $(awk 'END { print NR + 0 }' "$work/${op#sw-}.txt"))) + rm -rf "${dir:?}/$op" + ;; + esac + if [ "$rc" -ne 0 ] || [ "$files" -ne "$want" ]; then + ok=false + say "$c $op round $r: exit $rc, $files of $want files; log: $(tail -n 2 "$dir/$op.log" | tr '\n' ' ')" + fi + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$c" "$op" "$r" \ + "$(awk -v a="$t0" -v b="$t1" 'BEGIN { printf "%.3f", (b - a) / 1e9 }')" "$files" "$bytes" "$ok" >> "$work/results.tsv" +} + +for r in $(seq 1 "$ROUNDS"); do + for c in $CASES; do + for op in $OPS; do + measure "$c" "$op" "$r" + say "round $r $c $op: $(awk -F '\t' 'END { print $4 " s, " $5 " files, " $6 " bytes, ok " $7 }' "$work/results.tsv")" + done + done +done + +#---------------------------------------------------------------------------# +# OUT.json +#---------------------------------------------------------------------------# +# No quote and no backslash may reach the JSON string. +host="${BENCH_HOST:-$(uname -n) ($(uname -sm))}" +host="${host//\"/}" +host="${host//\\/}" +# From the local database: pacman -Q would warn about the image's own sync +# databases, which this never reads. +version=unknown +for d in /var/lib/pacman/local/pacman-[0-9]*; do + [ -d "$d" ] && version="${d##*/pacman-}" +done +{ + printf '{"schema":1,"date":"%s","host":"%s","pacman":"%s","cases":[' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$host" "$version" + first=1 + for c in $CASES; do + for op in $OPS; do + [ "$first" = 1 ] || printf ',' + first=0 + # rounds in the order they ran; seconds is their median + awk -F '\t' -v c="$c" -v op="$op" ' + $1 == c && $2 == op { + n++; r[n] = $4; s[n] = $4; files = $5; bytes = $6 + if ($7 != "true" || (n > 1 && $5 != prev)) bad = 1 + prev = $5 + } + END { + for (i = 1; i <= n; i++) for (j = i + 1; j <= n; j++) if (s[j] + 0 < s[i] + 0) { x = s[i]; s[i] = s[j]; s[j] = x } + med = (n % 2) ? s[(n + 1) / 2] : (s[n / 2] + s[n / 2 + 1]) / 2 + printf "{\"path\":\"%s\",\"op\":\"%s\",\"seconds\":%.3f,\"rounds\":[", c, op, med + for (i = 1; i <= n; i++) printf "%s%s", (i > 1 ? "," : ""), r[i] + printf "],\"files\":%d,\"bytes\":%d,\"ok\":%s}", files, bytes, (bad ? "false" : "true") + } + ' "$work/results.tsv" + done + done + printf ']}\n' +} > "$work/out.json" +mv -f "$work/out.json" "$out" +say "wrote $out" diff --git a/tests/fixtures/client/build-repos b/tests/fixtures/client/build-repos new file mode 100644 index 0000000..88ec9eb --- /dev/null +++ b/tests/fixtures/client/build-repos @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# Build the client suite's upstream: two small signed pacman repositories. +# Runs inside an Arch container, as the calling user, with the output root +# mounted at /out. +# +# bash build-repos +# +# Writes: +# /out/one/ alpha 1.0-1, beta 2.0-1, prov-a 1-1 and prov-b 1-1 (both +# provide virt), big 1-1 (4 MiB that do not compress, for the +# resume cases), each with its .sig, and one.db, one.files +# /out/two/ gamma 3.0-1 with its .sig, and two.db, two.files +# /out/gnupg the signing key. It trusts its own key ultimately, so a copy +# of it serves as pacman's GPGDir with no pacman-key, which +# needs root. +# +# Each database carries a %PGPSIG% for every package, as Arch's own do +# (repo-add --include-sigs). The .db and .files are regular files, never the +# symbolic links repo-add makes. +# +# Exit status: 0 built, 1 a step failed (set -e names it in the log). +set -euo pipefail + +export GNUPGHOME=/out/gnupg +mkdir -m 0700 "$GNUPGHOME" +gpg --batch --passphrase '' --quick-gen-key 'mirror-archlinux client test ' ed25519 sign never + +# mkpkg REPO NAME VERSION [PROVIDES] [BYTES]: one signed package in /out/REPO +mkpkg() { + local repo="/out/$1" name="$2" version="$3" provides="${4:-}" bytes="${5:-0}" d file + mkdir -p "$repo" + d="$(mktemp -d)" + mkdir -p "$d/usr/share/client-test" + printf '%s %s\n' "$name" "$version" > "$d/usr/share/client-test/$name" + if [ "$bytes" -gt 0 ]; then + head -c "$bytes" /dev/urandom > "$d/usr/share/client-test/$name.bin" + fi + { + printf 'pkgname = %s\npkgbase = %s\npkgver = %s\n' "$name" "$name" "$version" + printf 'pkgdesc = mirror-archlinux client fixture\nurl = https://example.invalid\n' + printf 'builddate = 1700000000\npackager = client test \n' + printf 'size = %s\narch = any\nlicense = MIT\n' "$((bytes + 64))" + if [ -n "$provides" ]; then + printf 'provides = %s\n' "$provides" + fi + } > "$d/.PKGINFO" + file="$name-$version-any.pkg.tar.zst" + (cd "$d" && bsdtar --zstd --uid 0 --gid 0 --uname root --gname root -cf "$repo/$file" .PKGINFO usr) + gpg --batch --yes --detach-sign --no-armor -o "$repo/$file.sig" "$repo/$file" + rm -rf "$d" +} + +mkpkg one alpha 1.0-1 +mkpkg one beta 2.0-1 +mkpkg one prov-a 1-1 virt +mkpkg one prov-b 1-1 virt +mkpkg one big 1-1 '' 4194304 +mkpkg two gamma 3.0-1 + +for repo in one two; do + repo-add --quiet --include-sigs "/out/$repo/$repo.db.tar.gz" "/out/$repo"/*.pkg.tar.zst + for kind in db files; do + cp --remove-destination -L "/out/$repo/$repo.$kind.tar.gz" "/out/$repo/$repo.$kind" + rm -f "/out/$repo/$repo.$kind.tar.gz" "/out/$repo/$repo.$kind.tar.gz.old" + done +done +gpgconf --kill gpg-agent +ls -l /out/one /out/two /out/gnupg diff --git a/tests/fixtures/client/consumer b/tests/fixtures/client/consumer new file mode 100644 index 0000000..8bf0f52 --- /dev/null +++ b/tests/fixtures/client/consumer @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# +# One case of the client suite, inside an Arch container on the host network. +# Prints "RESULT NAME VALUE" lines for tests/integration/20-client.sh to check, +# and the logs it read them from. +# +# bash consumer CASE SERVER +# +# SERVER is the repository path without its last part, e.g. +# oci+http://127.0.0.1:5000/it/client/x86_64; [one] and [two] hang below it. +# Mounted: /in/pacman-oci (the wrapper), /in/gnupg (the signing key's home). +# +# Cases: +# main-du -Sy, -S of three packages, -Fy, with DownloadUser +# main-root the same without DownloadUser +# noprefetch -Sy and -S with PACMAN_OCI_PREFETCH=0 +# registry -S with PACMAN_OCI_SIG=registry +# provider -S virt, answering 2 at the provider prompt +# dead the wrapper alone, under a parent named pacman, against a +# registry that answers 503 +# resume -Sy and -S big through the proxy that cuts each blob +# resume-serial the same with PACMAN_OCI_PREFETCH=0 +# stall -Sy and -S big through the proxy that stops sending big for +# 40 s after its first 64 KiB +# flip -Sy and -S alpha gamma through the proxy that changes the +# last byte of the first answer for each blob +# +# Exit status: 0 the case ran (its results say how it went), 1 it could not. +set -euo pipefail + +case_name="$1" +server="$2" + +install -m 0755 /in/pacman-oci /usr/local/bin/pacman-oci +# The public keyring and the trust database are all pacman reads; the agent's +# sockets may sit beside them and are not copied. +mkdir -m 0700 /tmp/gnupg +cp /in/gnupg/pubring.kbx /in/gnupg/trustdb.gpg /tmp/gnupg/ + +# conf DOWNLOADUSER(yes|no) SIGLEVEL: /tmp/pacman.conf with both repositories +conf() { + { + printf '[options]\nArchitecture = auto\nSigLevel = %s\n' "$2" + printf 'LocalFileSigLevel = Optional\nGPGDir = /tmp/gnupg\n' + printf 'XferCommand = /usr/local/bin/pacman-oci %%o %%u\n' + if [ "$1" = yes ]; then + printf 'DownloadUser = alpm\n' + fi + printf '\n[one]\nServer = %s/one\n\n[two]\nServer = %s/two\n' "$server" "$server" + } > /tmp/pacman.conf +} + +pac() { + pacman --config /tmp/pacman.conf --noconfirm "$@" +} + +# count TEXT FILE: lines of FILE that hold TEXT +count() { + awk -v t="$1" 'index($0, t) { n++ } END { print n + 0 }' "$2" +} + +result() { + printf 'RESULT %s %s\n' "$1" "$2" +} + +# show FILE: a log, indented, so a failing case can be read from the output +show() { + printf '%s\n' "--- $1" + sed 's/^/ /' "$1" +} + +# leftovers: store files and download directories once pacman has ended. The +# background remover polls once a second, so it gets 3 s. +leftovers() { + sleep 3 + result "$1_store_files" "$(find /var/lib/pacman/sync /var/cache/pacman/pkg -name '.pacman-oci.*' | wc -l)" + result "$1_download_dirs" "$(find /var/lib/pacman/sync /var/cache/pacman/pkg -name 'download-*' | wc -l)" +} + +# sync LABEL ARGS...: one pacman call with the wrapper's debug output +sync() { + local label="$1" rc=0 + shift + PACMAN_OCI_DEBUG=1 pac "$@" > "/tmp/$label.out" 2> "/tmp/$label.err" || rc=$? + result "${label}_rc" "$rc" + result "${label}_get" "$(count 'pacman-oci: GET ' "/tmp/$label.err")" + result "${label}_prefetch_get" "$(count 'pacman-oci: prefetch GET ' "/tmp/$label.err")" + result "${label}_served" "$(count ' from the prefetch' "/tmp/$label.err")" + show "/tmp/$label.err" +} + +installed() { + local rc=0 + cat /usr/share/client-test/"$1" > /dev/null || rc=$? + result "installed_$1" "$rc" +} + +case "$case_name" in + main-du | main-root) + du=no + if [ "$case_name" = main-du ]; then + du=yes + fi + conf "$du" 'Required DatabaseOptional' + sync sy -Sy + sync s -S alpha beta gamma + installed alpha + installed beta + installed gamma + sync fy -Fy + rc=0 + pacman --config /tmp/pacman.conf -Fl alpha > /tmp/fl.out || rc=$? + result fl "$rc:$(count usr/share/client-test/alpha /tmp/fl.out)" + leftovers main + ;; + noprefetch) + conf yes 'Required DatabaseNever' + PACMAN_OCI_PREFETCH=0 sync sy -Sy + PACMAN_OCI_PREFETCH=0 sync s -S alpha gamma + installed alpha + installed gamma + leftovers noprefetch + ;; + registry) + conf yes 'Required DatabaseNever' + sync sy -Sy + PACMAN_OCI_SIG=registry sync s -S alpha beta gamma + result s_sig_layers "$(count '.sig ready' /tmp/s.err)" + installed alpha + installed gamma + ;; + provider) + conf yes 'Required DatabaseNever' + sync sy -Sy + rc=0 + printf '2\ny\n' | PACMAN_OCI_DEBUG=1 pacman --config /tmp/pacman.conf -S virt > /tmp/s.out 2> /tmp/s.err || rc=$? + result s_rc "$rc" + result s_prefetched_default "$(count 'prov-a-1-1-any.pkg.tar.zst ready' /tmp/s.err)" + result s_get_chosen "$(count 'pacman-oci: GET http' /tmp/s.err)" + rc=0 + pacman --config /tmp/pacman.conf -Q prov-b > /dev/null || rc=$? + result chosen_installed "$rc" + show /tmp/s.err + ;; + dead) + # A process named pacman, so the wrapper keeps a run store for it. + mkdir -p /tmp/fake /tmp/fakedl + cat > /tmp/fake/pacman << 'EOS' +#!/bin/sh +for f in alpha-1.0-1-any.pkg.tar.zst beta-2.0-1-any.pkg.tar.zst beta-2.0-1-any.pkg.tar.zst.sig; do + t0=$(date +%s%N) + rc=0 + PACMAN_OCI_DEBUG=1 /usr/local/bin/pacman-oci "/tmp/fakedl/$f.part" "$1/one/$f" 2> "/tmp/dead.$f.err" || rc=$? + echo "$f $rc $((($(date +%s%N) - t0) / 1000000))" +done +EOS + chmod 0755 /tmp/fake/pacman + /tmp/fake/pacman "$server" > /tmp/dead.out + while read -r f rc ms; do + case $f in + alpha-*) n=first ;; + *.sig) n=sig ;; + *) n=second ;; + esac + result "dead_${n}_rc" "$rc" + result "dead_${n}_ms" "$ms" + result "dead_${n}_get" "$(count 'pacman-oci: GET ' "/tmp/dead.$f.err")" + show "/tmp/dead.$f.err" + done < /tmp/dead.out + ;; + resume | resume-serial) + conf yes 'Required DatabaseNever' + if [ "$case_name" = resume-serial ]; then + export PACMAN_OCI_PREFETCH=0 + fi + sync sy -Sy + sync s -S big + installed big + ;; + stall) + conf yes 'Required DatabaseNever' + sync sy -Sy + t0=$(date +%s%N) + sync s -S big + result s_ms "$((($(date +%s%N) - t0) / 1000000))" + installed big + ;; + flip) + conf yes 'Required DatabaseNever' + sync sy -Sy + result sy_refused "$(count 'bytes with sha256' /tmp/sy.err)" + sync s -S alpha gamma + result s_refused "$(count 'bytes with sha256' /tmp/s.err)" + installed alpha + installed gamma + ;; + *) + echo "consumer: no case $case_name" >&2 + exit 1 + ;; +esac diff --git a/tests/fixtures/client/fault-proxy.py b/tests/fixtures/client/fault-proxy.py new file mode 100644 index 0000000..22e4722 --- /dev/null +++ b/tests/fixtures/client/fault-proxy.py @@ -0,0 +1,115 @@ +"""A registry proxy that breaks requests on purpose, for the client suite. + + python3 -I fault-proxy.py MODE UPSTREAM + +MODE is one of: + cut a blob GET with no Range header gets its headers, with the full + Content-Length, and half of its body; then the connection closes. + A GET with a Range header is passed through. So every blob arrives + only when the client resumes it. + 503 every request is answered 503, with no Retry-After. + flip the first GET of each blob gets its whole body with the last byte + changed, so its size is right and its sha256 is wrong. Every later + GET of that blob is passed through. + stall the first GET with no Range header of each blob over 1 MiB gets its + headers, with the full Content-Length, and 64 KiB of its body; then + nothing for 40 s, and the connection closes. Every other GET is + passed through. + +UPSTREAM is the registry, host:port. The proxy listens on 127.0.0.1 on a port +the system picks, and prints "PORT " first. Then each request is one line: +" range=". + +Standard library only. +""" + +import http.client +import http.server +import sys +import threading +import time + +MODE = sys.argv[1] +UPSTREAM = sys.argv[2] +PASS_HEADERS = ("accept", "authorization", "range", "user-agent") +SEEN = set() +SEEN_LOCK = threading.Lock() + + +def first_time(path): + """True the first time the proxy sees PATH.""" + with SEEN_LOCK: + if path in SEEN: + return False + SEEN.add(path) + return True + + +class Proxy(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def answer(self, code, headers, body): + self.send_response(code) + for name, value in headers: + self.send_header(name, value) + self.send_header("Connection", "close") + self.end_headers() + if body: + self.wfile.write(body) + + def handle_one(self): + rng = self.headers.get("Range") + print("%s %s range=%s" % (self.command, self.path, rng or "-"), flush=True) + if MODE == "503": + self.answer(503, [("Content-Length", "0")], b"") + return + conn = http.client.HTTPConnection(UPSTREAM, timeout=30) + sent = {k: v for k, v in self.headers.items() if k.lower() in PASS_HEADERS} + conn.request(self.command, self.path, headers=sent) + resp = conn.getresponse() + body = resp.read() + conn.close() + keep = [(k, v) for k, v in resp.getheaders() + if k.lower() not in ("connection", "transfer-encoding", "content-length")] + keep.append(("Content-Length", str(len(body)))) + blob = "/blobs/" in self.path and rng is None and resp.status == 200 and len(body) > 1 + if MODE == "flip" and blob and first_time(self.path): + body = body[:-1] + bytes([body[-1] ^ 0xFF]) + print(" last byte changed", flush=True) + self.answer(resp.status, keep, body) + return + if MODE == "cut" and blob: + self.send_response(resp.status) + for name, value in keep: + self.send_header(name, value) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(body[: len(body) // 2]) + self.wfile.flush() + self.close_connection = True + print(" cut after %d of %d bytes" % (len(body) // 2, len(body)), flush=True) + return + if MODE == "stall" and blob and len(body) > 1048576 and first_time(self.path): + self.send_response(resp.status) + for name, value in keep: + self.send_header(name, value) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(body[:65536]) + self.wfile.flush() + print(" stalled after 65536 of %d bytes" % len(body), flush=True) + time.sleep(40) + self.close_connection = True + return + self.answer(resp.status, keep, body) + + def do_GET(self): + self.handle_one() + + +server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Proxy) +print("PORT %d" % server.server_address[1], flush=True) +server.serve_forever() diff --git a/tests/integration/20-client.sh b/tests/integration/20-client.sh new file mode 100644 index 0000000..6f2525d --- /dev/null +++ b/tests/integration/20-client.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash +# +# The client against a registry of its own. +# +# A local registry (registry:3) holds two signed repositories in the mirror's +# layout: each package under its own tag, with layers titled FILE and FILE.sig, +# and each database under latest. pacman in an Arch container then reads them +# through client/pacman-oci: with the prefetch and without it, with and +# without DownloadUser, with the signatures from the registry, with a provider +# other than the default, through a proxy that answers 503, through a proxy +# that cuts every blob in half, through a proxy that stops sending a large +# blob, and through a proxy that changes one byte of the first answer for each +# blob. +# +# Needs docker or podman (CONTAINER_RUNTIME), curl, jq, sha256sum, and network +# to pull the pinned images and oras. Ports are picked at run time and every +# container name carries this shell's pid and a random number, so several runs +# can share one host. +set -euo pipefail +# shellcheck source=../lib/harness.sh +. "$REPO_ROOT/tests/lib/harness.sh" + +# tag: base +ARCH_IMAGE=docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 +# tag: 3 +REGISTRY_IMAGE=docker.io/library/registry:3@sha256:ddf754342cfc8acc51a56d5d0ab6af06826461864460636d8bd5c546dab2a7b8 +# tag: 3.13-alpine +PYTHON_IMAGE=docker.io/library/python:3.13-alpine@sha256:2d9aefe2fef018a7eb2c13064c89c71929800fd2e5dccdbf52ea5da5bb8d929a + +FIX="$REPO_ROOT/tests/fixtures/client" +runtime_detect +work="$(work_dir)" +id="$$-$RANDOM" +reg_name="client-it-registry-$id" +cut_name="client-it-cut-$id" +dead_name="client-it-dead-$id" +flip_name="client-it-flip-$id" +stall_name="client-it-stall-$id" +cleanup() { + local out + out="$("$RUNTIME" rm -f "$reg_name" "$cut_name" "$dead_name" "$flip_name" "$stall_name" 2>&1)" || diag "cleanup: $out" + if [ "${IT_KEEP:-0}" = 1 ]; then + echo "# kept $work" >&2 + else + rm -rf "$work" + fi +} +trap cleanup EXIT + +# The build container writes into the work directory as the calling user, so +# the directory can be removed afterwards without root. +as_me=(--user "$(id -u):$(id -g)") +case "$(basename "$RUNTIME")" in + *podman*) as_me=(--userns keep-id) ;; +esac + +stop() { + fail "$1" "${@:2}" + summary + exit 1 +} + +#---------------------------------------------------------------------------# +# The registry, on a port the runtime picks +#---------------------------------------------------------------------------# +"$RUNTIME" run -d --rm --name "$reg_name" -p 127.0.0.1::5000 "$REGISTRY_IMAGE" > /dev/null +reg_port="$("$RUNTIME" port "$reg_name" 5000/tcp)" +reg_port="${reg_port##*:}" +ready=0 +for _ in $(seq 1 30); do + if curl -q -fsS --connect-timeout 2 --max-time 5 -o /dev/null "http://127.0.0.1:$reg_port/v2/"; then + ready=1 + break + fi + sleep 1 +done +[ "$ready" = 1 ] || stop "the local registry answers" "after 30 s on port '$reg_port'" "reproduce: $RUNTIME logs $reg_name" +REG="127.0.0.1:$reg_port" +SERVER="oci+http://$REG/it/client/x86_64" + +#---------------------------------------------------------------------------# +# Two signed repositories, pushed in the mirror's layout +#---------------------------------------------------------------------------# +mkdir -p "$work/up" +if ! "$RUNTIME" run --rm "${as_me[@]}" -e HOME=/tmp -v "$work/up:/out" -v "$FIX/build-repos:/build-repos:ro" \ + "$ARCH_IMAGE" bash /build-repos > "$work/build.log" 2>&1; then + stop "the fixture repositories build" "$(tail -n 5 "$work/build.log")" "reproduce: bash tests/fixtures/client/build-repos in $ARCH_IMAGE" +fi +ORAS="$("$REPO_ROOT/scripts/install-oras" "$work/bin")" + +# push REPO TAG FILE:TYPE...: one manifest, from the repository's directory +push() { + local repo="$1" tag="$2" artifact="$3" rc=0 + shift 3 + (cd "$work/up/$repo" && timeout 120 "$ORAS" push --plain-http --artifact-type "$artifact" \ + "$REG/it/client/x86_64/$repo:$tag" "$@") >> "$work/push.log" 2>&1 || rc=$? + [ "$rc" -eq 0 ] || stop "oras pushes $repo:$tag" "exit $rc" "$(tail -n 3 "$work/push.log")" +} +for repo in one two; do + for f in "$work/up/$repo"/*.pkg.tar.zst; do + f="${f##*/}" + push "$repo" "$("$REPO_ROOT/client/pacman-oci" --tag "$f")" application/vnd.pkgforge.archlinux.package.v1 \ + "./$f:application/vnd.pkgforge.archlinux.pkg.v1" "./$f.sig:application/pgp-signature" + done + push "$repo" latest application/vnd.pkgforge.archlinux.db.v1 \ + "./$repo.db:application/vnd.pkgforge.archlinux.db.v1" "./$repo.files:application/vnd.pkgforge.archlinux.files.v1" +done +titles="$(curl -q -fsS --connect-timeout 5 --max-time 20 -H 'Accept: application/vnd.oci.image.manifest.v1+json' \ + "http://$REG/v2/it/client/x86_64/one/manifests/$("$REPO_ROOT/client/pacman-oci" --tag alpha-1.0-1-any.pkg.tar.zst)" | + jq -r '[.layers[].annotations["org.opencontainers.image.title"]] | join(" ")')" +check "a package is pushed as the mirror lays it out" "alpha-1.0-1-any.pkg.tar.zst alpha-1.0-1-any.pkg.tar.zst.sig" "$titles" \ + "curl http://$REG/v2/it/client/x86_64/one/manifests/" + +#---------------------------------------------------------------------------# +# The four proxies, on ports they pick themselves +#---------------------------------------------------------------------------# +# proxy NAME MODE: start one and print its port +proxy() { + local p='' + "$RUNTIME" run -d --rm --name "$1" --network host -v "$FIX/fault-proxy.py:/proxy.py:ro" \ + "$PYTHON_IMAGE" python3 -I -u /proxy.py "$2" "$REG" > /dev/null + for _ in $(seq 1 30); do + p="$("$RUNTIME" logs "$1" 2>&1 | awk '$1 == "PORT" { print $2 }')" + [ -z "$p" ] || break + sleep 1 + done + printf '%s\n' "$p" +} +cut_port="$(proxy "$cut_name" cut)" +dead_port="$(proxy "$dead_name" 503)" +flip_port="$(proxy "$flip_name" flip)" +stall_port="$(proxy "$stall_name" stall)" +[ -n "$cut_port" ] && [ -n "$dead_port" ] && [ -n "$flip_port" ] && [ -n "$stall_port" ] || stop "the fault proxies start" \ + "cut '$cut_port' 503 '$dead_port' flip '$flip_port' stall '$stall_port'" "reproduce: $RUNTIME logs $cut_name" + +#---------------------------------------------------------------------------# +# The cases +#---------------------------------------------------------------------------# +# consumer CASE SERVER: one case in a fresh container; its output in +# $work/case-CASE.log +consumer() { + local rc=0 + "$RUNTIME" run --rm --network host \ + -v "$REPO_ROOT/client/pacman-oci:/in/pacman-oci:ro" \ + -v "$work/up/gnupg:/in/gnupg:ro" \ + -v "$FIX/consumer:/in/consumer:ro" \ + "$ARCH_IMAGE" bash /in/consumer "$1" "$2" > "$work/case-$1.log" 2>&1 || rc=$? + if [ "$rc" -ne 0 ]; then + fail "case $1 runs" "exit $rc" "$(tail -n 10 "$work/case-$1.log")" + fi +} + +# want CASE NAME EXPECTED DESCRIPTION +want() { + local got + got="$(awk -v n="$2" '$1 == "RESULT" && $2 == n { print $3 }' "$work/case-$1.log")" + if [ "$got" = "$3" ]; then + ok "$1: $4" + else + fail "$1: $4" "expected $2 = $3" "got: $2 = ${got:-nothing}" \ + "reproduce: IT_KEEP=1 tests/integration/20-client.sh, then read case-$1.log" + while IFS= read -r l; do diag "$l"; done < <(grep -v '^RESULT' "$work/case-$1.log" | tail -n 15) + fi +} + +# want_cmp CASE NAME OP NUMBER DESCRIPTION, with OP one of lt ge +want_cmp() { + local got pass=0 + got="$(awk -v n="$2" '$1 == "RESULT" && $2 == n { print $3 }' "$work/case-$1.log")" + case "$3:$got" in + *: | *:*[!0-9]*) ;; + lt:*) if [ "$got" -lt "$4" ]; then pass=1; fi ;; + ge:*) if [ "$got" -ge "$4" ]; then pass=1; fi ;; + esac + if [ "$pass" = 1 ]; then + ok "$1: $5" + else + fail "$1: $5" "expected $2 $3 $4" "got: $2 = ${got:-nothing}" \ + "reproduce: IT_KEEP=1 tests/integration/20-client.sh, then read case-$1.log" + fi +} + +for c in main-du main-root; do + consumer "$c" "$SERVER" + want "$c" sy_rc 0 "pacman -Sy of two repositories succeeds" + want "$c" sy_prefetch_get 4 "the first database request reads both manifests and both databases in one prefetch" + want "$c" sy_get 0 "no single request: the second database and both missing .db.sig come from the run store" + want "$c" sy_served 2 "both databases are handed over from the run store" + want "$c" s_rc 0 "pacman -S of three packages from two repositories succeeds" + want "$c" s_prefetch_get 3 "the prefetch reads the three packages; the signatures come from %PGPSIG%" + want "$c" s_get 0 "the serial pass makes no network request" + want "$c" s_served 6 "every package and every .sig is handed over from the run store" + want "$c" installed_alpha 0 "alpha is installed" + want "$c" installed_gamma 0 "gamma, from the second repository, is installed" + want "$c" fy_rc 0 "pacman -Fy succeeds" + want "$c" fy_served 2 "both .files databases come from one prefetch" + want "$c" fl 0:1 "pacman -Fl reads the files database" + want "$c" main_store_files 0 "no store file is left once pacman has ended" + want "$c" main_download_dirs 0 "no download directory is left behind" +done + +consumer noprefetch "$SERVER" +want noprefetch s_rc 0 "PACMAN_OCI_PREFETCH=0: pacman -S succeeds" +want noprefetch s_prefetch_get 0 "PACMAN_OCI_PREFETCH=0: no prefetch request" +want noprefetch s_get 8 "PACMAN_OCI_PREFETCH=0: two requests per file, as version 1 made them" +want noprefetch noprefetch_store_files 0 "PACMAN_OCI_PREFETCH=0: no store file is left" + +consumer registry "$SERVER" +want registry s_rc 0 "PACMAN_OCI_SIG=registry: pacman -S succeeds" +want registry s_sig_layers 3 "PACMAN_OCI_SIG=registry: each .sig is the registry's layer" +want registry s_get 0 "PACMAN_OCI_SIG=registry: the serial pass makes no network request" + +consumer provider "$SERVER" +want provider s_rc 0 "another provider: pacman -S virt succeeds" +want provider s_prefetched_default 1 "another provider: the prefetch took the first provider, as print mode does" +want_cmp provider s_get_chosen ge 2 "another provider: the chosen one is read alone" +want provider chosen_installed 0 "another provider: the chosen one is installed" + +consumer dead "oci+http://127.0.0.1:$dead_port/it/client/x86_64" +want dead dead_first_rc 1 "a registry that answers 503: the first file misses" +want dead dead_second_rc 1 "a registry that answers 503: the second file misses" +want dead dead_second_get 0 "a registry that answers 503: the second file makes no request" +want_cmp dead dead_second_ms lt 1000 "a registry that answers 503: the second file misses in under 1 s" +want_cmp dead dead_sig_get ge 1 "a registry that answers 503: a .sig is still asked for" + +big_hex="$(sha256sum "$work/up/one/big-1-1-any.pkg.tar.zst" | cut -d ' ' -f 1)" +for c in resume resume-serial; do + before="$("$RUNTIME" logs "$cut_name" 2>&1 | awk -v b="/blobs/sha256:$big_hex" 'index($0, b) && /range=bytes=/ { n++ } END { print n + 0 }')" + consumer "$c" "oci+http://127.0.0.1:$cut_port/it/client/x86_64" + after="$("$RUNTIME" logs "$cut_name" 2>&1 | awk -v b="/blobs/sha256:$big_hex" 'index($0, b) && /range=bytes=/ { n++ } END { print n + 0 }')" + want "$c" sy_rc 0 "every blob cut in half: pacman -Sy succeeds" + want "$c" s_rc 0 "every blob cut in half: pacman -S big succeeds" + want "$c" installed_big 0 "every blob cut in half: big is installed" + check "$c: the cut package was resumed with a Range request" 1 "$((after - before))" \ + "$RUNTIME logs $cut_name | grep $big_hex" +done +want resume s_get 0 "every blob cut in half: the prefetch resumed it, the serial pass made no request" + +consumer stall "oci+http://127.0.0.1:$stall_port/it/client/x86_64" +ranges="$("$RUNTIME" logs "$stall_name" 2>&1 | awk -v b="/blobs/sha256:$big_hex" 'index($0, b) && /range=bytes=65536-/ { n++ } END { print n + 0 }')" +want stall sy_rc 0 "a blob that stops after 64 KiB: pacman -Sy succeeds" +want stall s_rc 0 "a blob that stops after 64 KiB: pacman -S big succeeds" +want stall installed_big 0 "a blob that stops after 64 KiB: big is installed" +want stall s_get 0 "a blob that stops after 64 KiB: the prefetch got it, the serial pass made no request" +want_cmp stall s_ms lt 20000 "a blob that stops for 40 s: pacman -S big takes less than 20 s" +check "stall: the prefetch cut the stopped transfer and resumed it from 64 KiB" 1 "$ranges" \ + "$RUNTIME logs $stall_name | grep $big_hex" + +consumer flip "oci+http://127.0.0.1:$flip_port/it/client/x86_64" +want flip sy_rc 0 "one byte changed in each first answer: pacman -Sy succeeds" +want flip sy_refused 2 "one byte changed in each first answer: the prefetch keeps neither database" +want flip sy_get 4 "one byte changed in each first answer: each database is read again alone" +want flip s_rc 0 "one byte changed in each first answer: pacman -S alpha gamma succeeds" +want flip s_refused 2 "one byte changed in each first answer: the prefetch keeps neither package" +want flip s_get 4 "one byte changed in each first answer: each package is read again alone" +want flip installed_alpha 0 "one byte changed in each first answer: alpha is installed" +want flip installed_gamma 0 "one byte changed in each first answer: gamma is installed" + +summary diff --git a/tests/unit/31-run-store.sh b/tests/unit/31-run-store.sh new file mode 100644 index 0000000..252112a --- /dev/null +++ b/tests/unit/31-run-store.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +# +# The run store: one pacman run keeps its files beside OUTPUT under its own +# key, uses only files it owns, and leaves nothing behind. +# +# The parent process is faked with PACMAN_OCI_TEST_PROC: a directory laid out +# like /proc for this shell's pid, with comm, stat and cmdline. The wrapper is +# always run from this shell itself, never from a subshell, so its parent is +# this shell. Port 9 on the loopback refuses at once, so nothing here needs a +# network. +set -euo pipefail +# shellcheck source=../lib/harness.sh +. "$REPO_ROOT/tests/lib/harness.sh" + +W="$REPO_ROOT/client/pacman-oci" +work="$(work_dir)" +trap 'rm -rf "$work"' EXIT +mkdir -p "$work/tmp" +export TMPDIR="$work/tmp" +fake="$work/proc" +mkdir -p "$fake/$$" +: > "$fake/pacman-oci-test-proc" +export PACMAN_OCI_TEST_PROC="$fake" +U="oci+http://127.0.0.1:9/ns/mirror/x86_64/core" + +# parent START [COMM] [COMM IN STAT]: this shell, as the process COMM +# (default pacman) that started at START +parent() { + printf '%s\n' "${2:-pacman}" > "$fake/$$/comm" + printf '%s (%s) S 1 1 1 0 -1 4194560 837 0 0 0 0 0 0 0 20 0 1 0 %s 62676992 3236\n' \ + "$$" "${3:-${2:-pacman}}" "$1" > "$fake/$$/stat" +} +# argv ARG...: the parent's argument list, NUL separated as in /proc +argv() { + printf '%s\0' "$@" > "$fake/$$/cmdline" +} +# call OUTPUT URL: the wrapper as pacman calls it; its exit code in $rc, its +# debug output in $work/err +call() { + rc=0 + PACMAN_OCI_DEBUG=1 "$W" "$1" "$2" > /dev/null 2> "$work/err" || rc=$? +} +gets() { + awk '/^pacman-oci: GET / { n++ } END { print n + 0 }' "$work/err" +} +# entries DIR: the names of the run store files in DIR, sorted, on one line +entries() { + find "$1" -maxdepth 1 -name '.pacman-oci.*' | sed 's|.*/||' | sort | tr '\n' ' ' +} + +# A parent that prints nothing in print mode, so no prefetch has work. +cat > "$work/pacman" << 'EOF' +#!/bin/sh +exit 0 +EOF +chmod +x "$work/pacman" +ln -s "$work/pacman" "$fake/$$/exe" +argv pacman -S foo + +#---------------------------------------------------------------------------# +# The run key is the parent's pid and field 22 of its stat, counted after the +# last ")": a comm may hold ") " itself. +#---------------------------------------------------------------------------# +mkdir -p "$work/key" +parent 777 pacman 'pa) (cm) an' +call "$work/key/foo-1-1-any.pkg.tar.zst.part" "$U/foo-1-1-any.pkg.tar.zst" +check "the key is the pid and the start time after the last )" ".pacman-oci.$$-777.,down.127.0.0.1:9 .pacman-oci.$$-777.,open .pacman-oci.$$-777.,prefetch .pacman-oci.$$-777.,reaper " \ + "$(entries "$work/key")" "ls -a DIR after a call under a parent named pacman" + +mkdir -p "$work/notpacman" +parent 778 bash +call "$work/notpacman/foo-1-1-any.pkg.tar.zst.part" "$U/foo-1-1-any.pkg.tar.zst" +check "a parent that is not pacman: no store" "" "$(entries "$work/notpacman")" "ls -a DIR" + +mkdir -p "$work/short" +parent 779 +printf '%s (pacman) S 1 1\n' "$$" > "$fake/$$/stat" +call "$work/short/foo-1-1-any.pkg.tar.zst.part" "$U/foo-1-1-any.pkg.tar.zst" +check "a stat line with no field 22: no store" "" "$(entries "$work/short")" "ls -a DIR" + +mkdir -p "$work/nomark" +parent 779 +rm "$fake/pacman-oci-test-proc" +call "$work/nomark/foo-1-1-any.pkg.tar.zst.part" "$U/foo-1-1-any.pkg.tar.zst" +check "the test hook is ignored without its marker file" "" "$(entries "$work/nomark")" "ls -a DIR" +: > "$fake/pacman-oci-test-proc" + +#---------------------------------------------------------------------------# +# A ready entry becomes OUTPUT with no request; anything else is never served. +#---------------------------------------------------------------------------# +parent 800 +K="$$-800" +d="$work/ready" +mkdir -p "$d" +printf 'package bytes\n' > "$d/.pacman-oci.$K.ready-1-1-any.pkg.tar.zst" +printf 'stale part\n' > "$d/ready-1-1-any.pkg.tar.zst.part" +call "$d/ready-1-1-any.pkg.tar.zst.part" "$U/ready-1-1-any.pkg.tar.zst" +check "a ready package is OUTPUT, with no request" "0:package bytes:0" \ + "$rc:$(cat "$d/ready-1-1-any.pkg.tar.zst.part"):$(gets)" "client/pacman-oci DIR/ready...part URL" +check "the entry is moved, not copied" "" "$(find "$d" -name ".pacman-oci.$K.ready*")" "ls -a DIR" + +printf 'served through http\n' > "$d/.pacman-oci.$K.plain-1-1-any.pkg.tar.zst" +call "$d/plain-1-1-any.pkg.tar.zst.part" "http://127.0.0.1:9/core/os/x86_64/plain-1-1-any.pkg.tar.zst" +check "a ready package is OUTPUT whichever Server pacman asks" "0:served through http" \ + "$rc:$(cat "$d/plain-1-1-any.pkg.tar.zst.part")" "client/pacman-oci DIR/plain...part http://..." + +printf 'database bytes\n' > "$d/.pacman-oci.$K.core.db" +call "$d/core.db.part" "http://127.0.0.1:9/core/os/x86_64/core.db" +check "a ready database is never OUTPUT for a plain Server" 1 "$rc" "client/pacman-oci DIR/core.db.part http://..." +call "$d/core.db.part" "$U/core.db" +check "a ready database is OUTPUT for an oci:// Server" "0:database bytes" \ + "$rc:$(cat "$d/core.db.part")" "client/pacman-oci DIR/core.db.part oci+http://..." + +printf 'planted\n' > "$work/target" +ln -s "$work/target" "$d/.pacman-oci.$K.link-1-1-any.pkg.tar.zst" +call "$d/link-1-1-any.pkg.tar.zst.part" "$U/link-1-1-any.pkg.tar.zst" +check "a symbolic link is never served" 1 "$rc" "ln -s ... DIR/.pacman-oci.KEY.link...; client/pacman-oci" +check "the file a link points to is left as it was" planted "$(cat "$work/target")" "cat target" + +#---------------------------------------------------------------------------# +# Markers: a .sig the database manifest does not have misses with no request, +# and a host that failed is skipped for the rest of the run, never for a .sig. +#---------------------------------------------------------------------------# +parent 810 +K="$$-810" +d="$work/marks" +mkdir -p "$d" +: > "$d/.pacman-oci.$K.,open" +: > "$d/.pacman-oci.$K.,absent.core.db.sig" +call "$d/core.db.sig.part" "$U/core.db.sig" +check "an absent database signature misses with no request" "1:0" "$rc:$(gets)" "client/pacman-oci DIR/core.db.sig.part URL" + +call "$d/one-1-1-any.pkg.tar.zst.part" "$U/one-1-1-any.pkg.tar.zst" +check "a host that does not answer misses" 1 "$rc" "client/pacman-oci DIR/one...part oci+http://127.0.0.1:9/..." +check "and is marked down for this run" 1 "$(find "$d" -name ".pacman-oci.$K.,down.127.0.0.1:9" | wc -l | tr -d ' ')" "ls -a DIR" +call "$d/two-1-1-any.pkg.tar.zst.part" "$U/two-1-1-any.pkg.tar.zst" +check "the next package from that host misses with no request" "1:0" "$rc:$(gets)" "client/pacman-oci DIR/two...part URL" +call "$d/two-1-1-any.pkg.tar.zst.sig.part" "$U/two-1-1-any.pkg.tar.zst.sig" +check "a .sig is still asked of that host" "1:1" "$rc:$(gets)" "client/pacman-oci DIR/two...sig.part URL" + +#---------------------------------------------------------------------------# +# The sweep: entries of runs that are gone go; those of live runs, links, and +# names that are no key stay. +#---------------------------------------------------------------------------# +parent 820 +d="$work/sweep" +mkdir -p "$d" "$fake/4242" +printf '4242 (pacman) S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 5 1 1\n' > "$fake/4242/stat" +: > "$d/.pacman-oci.999999999-1.gone-1-1-any.pkg.tar.zst" +: > "$d/.pacman-oci.4242-5.live-1-1-any.pkg.tar.zst" +: > "$d/.pacman-oci.4242-6.reused-1-1-any.pkg.tar.zst" +: > "$d/.pacman-oci.notakey" +ln -s "$work/target" "$d/.pacman-oci.999999999-1.link" +call "$d/x.part" "file://$work/no-such-file" +check "the sweep keeps a live run, a link and a name that is no key" \ + ".pacman-oci.4242-5.live-1-1-any.pkg.tar.zst .pacman-oci.999999999-1.link .pacman-oci.notakey " \ + "$(entries "$d")" "ls -a DIR after one call" + +#---------------------------------------------------------------------------# +# The background remover: when the run ends, its entries go. +#---------------------------------------------------------------------------# +parent 830 +K="$$-830" +d="$work/reap" +mkdir -p "$d" +call "$d/foo-1-1-any.pkg.tar.zst.part" "$U/foo-1-1-any.pkg.tar.zst" +before="$(find "$d" -name ".pacman-oci.$K.*" | wc -l | tr -d ' ')" +parent 831 +left=1 +for _ in 1 2 3 4 5 6; do + left="$(find "$d" -name ".pacman-oci.$K.*" | wc -l | tr -d ' ')" + [ "$left" -ne 0 ] || break + sleep 0.5 +done +if [ "$before" -gt 0 ] && [ "$left" -eq 0 ]; then + ok "the run's entries are gone within 3 s of its end ($before before)" +else + fail "the run's entries are gone within 3 s of its end" "before: $before, after: $left" \ + "reproduce: tests/unit/31-run-store.sh, the reap block" +fi + +#---------------------------------------------------------------------------# +# Under dash, when it is here: the same key and the same serving. +#---------------------------------------------------------------------------# +if command -v dash > /dev/null 2>&1; then + parent 840 + d="$work/dash" + mkdir -p "$d" + printf 'dash bytes\n' > "$d/.pacman-oci.$$-840.d-1-1-any.pkg.tar.zst" + rc=0 + dash "$W" "$d/d-1-1-any.pkg.tar.zst.part" "$U/d-1-1-any.pkg.tar.zst" > /dev/null 2>&1 || rc=$? + check "under dash: a ready package is OUTPUT" "0:dash bytes" "$rc:$(cat "$d/d-1-1-any.pkg.tar.zst.part")" \ + "dash client/pacman-oci DIR/d...part URL" +else + ok "skipped: dash not installed" +fi + +summary diff --git a/tests/unit/32-prefetch-plan.sh b/tests/unit/32-prefetch-plan.sh new file mode 100644 index 0000000..e42d8ab --- /dev/null +++ b/tests/unit/32-prefetch-plan.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# +# The prefetch plan: which print run the wrapper asks pacman for, and what it +# makes of the lines pacman prints. +# +# pacman's argument list is rewritten for a print run: every refresh, +# --confirm, --noconfirm and --print-format go, everything after -- stays, and +# an argument list it cannot read with certainty means no prefetch at all. The +# print lines carry the URL, sha256, size and base64 signature of each file to +# download; a signature is written from that line, with no request. +# +# The parent process is faked with PACMAN_OCI_TEST_PROC (see 31-run-store.sh); +# its exe is a script that records its arguments and prints prepared lines. +# Port 9 on the loopback refuses at once, so nothing here needs a network. +set -euo pipefail +# shellcheck source=../lib/harness.sh +. "$REPO_ROOT/tests/lib/harness.sh" + +W="$REPO_ROOT/client/pacman-oci" +work="$(work_dir)" +trap 'rm -rf "$work"' EXIT +mkdir -p "$work/tmp" +export TMPDIR="$work/tmp" +fake="$work/proc" +mkdir -p "$fake/$$" +: > "$fake/pacman-oci-test-proc" +export PACMAN_OCI_TEST_PROC="$fake" +export FAKE_ARGS="$work/args" FAKE_LINES="$work/lines" FAKE_CONF="$work/conf.log" +U="oci+http://127.0.0.1:9/ns/mirror/x86_64/core" +TAB=$'\t' +FMT="%l$TAB%h$TAB%s$TAB%g" + +cat > "$work/pacman" << 'EOF' +#!/bin/sh +# Stands in for pacman's print run. +for a in "$@"; do + printf '%s\n' "$a" +done > "$FAKE_ARGS" +cat "$FAKE_LINES" +EOF +chmod +x "$work/pacman" +ln -s "$work/pacman" "$fake/$$/exe" +: > "$work/lines" + +start=1000 +# run ARG...: a fresh pacman run with these arguments, then one package request +# from it; the print run's arguments, if it ran, are in $work/args +run() { + start=$((start + 1)) + printf 'pacman\n' > "$fake/$$/comm" + printf '%s (pacman) S 1 1 1 0 -1 4194560 837 0 0 0 0 0 0 0 20 0 1 0 %s 62676992 3236\n' "$$" "$start" > "$fake/$$/stat" + printf '%s\0' pacman "$@" > "$fake/$$/cmdline" + rm -f "$work/args" + dl="$work/dl.$start" + mkdir -p "$dl" + rc=0 + PACMAN_OCI_DEBUG=1 "$W" "$dl/foo-1-1-any.pkg.tar.zst.part" "$U/foo-1-1-any.pkg.tar.zst" > /dev/null 2> "$work/err" || rc=$? +} + +# rewritten: the print run's arguments after the five the wrapper puts first +rewritten() { + if [ -f "$work/args" ]; then + sed -n '6,$p' "$work/args" | tr '\n' ' ' + else + printf 'no print run' + fi +} + +#---------------------------------------------------------------------------# +# The rewrite +#---------------------------------------------------------------------------# +run -Syu +check "the print run starts with --print and the format" "--print|--print-format|$FMT|--noconfirm|--noprogressbar" \ + "$(sed -n '1,5p' "$work/args" | paste -sd '|' -)" "PACMAN_OCI_TEST_PROC=DIR client/pacman-oci OUT URL; cat args" + +while IFS='|' read -r label want args; do + # shellcheck disable=SC2086 # the arguments are words on purpose + run $args + check "rewrite: $label" "$want" "$(rewritten)" "pacman $args, then the print run's arguments" +done << 'EOF' +-Syu drops the refresh|-Su |-Syu +-Syyu drops both refreshes|-Su |-Syyu +-S -y -u drops the lone -y|-S -u |-S -y -u +--sync --refresh drops --refresh|--sync |--sync --refresh +-Sb /x keeps b's value|-Sb /x |-Sb /x +-Syb/x drops the y before b and keeps b's value|-Sb/x |-Syb/x +a y after b is b's value|-Sb/y |-Sb/y +everything after -- stays|-S -- -y foo |-Sy -- -y foo +--noconfirm, --confirm and --print-format go|-S foo |-S --noconfirm --confirm --print-format %n foo +an option's value is never rewritten|-Su --ignore -yfoo |-Syu --ignore -yfoo +--config=FILE stays|--config=/etc/x.conf -Su |--config=/etc/x.conf -Syu +-r with its value apart stays|-Sr / foo |-Syr / foo +EOF + +while IFS='|' read -r label args; do + # shellcheck disable=SC2086 # the arguments are words on purpose + run $args + check "no print run: $label" "no print run" "$(rewritten)" "pacman $args, then ls args" +done << 'EOF' +-U is not sync|-U /tmp/x.pkg.tar.zst +-Sp prints already|-Sp foo +--print prints already|-S --print foo +an abbreviated option|-S --refr foo +an option pacman does not have|-S --frobnicate foo +a relative config|-S --config rel.conf foo +a relative cachedir|-S --cachedir=cache foo +a target read from stdin|-S - +an option with no value|-S foo --dbpath +EOF + +run -S "$(printf 'a\nb')" +check "no print run: an argument holds a newline" "no print run" "$(rewritten)" "pacman -S 'ab'" + +PACMAN_OCI_PREFETCH=0 run -Su +check "no print run: PACMAN_OCI_PREFETCH=0" "no print run" "$(rewritten)" "PACMAN_OCI_PREFETCH=0 ..." +PACMAN_OCI_JOBS=1 run -Su +check "no print run: PACMAN_OCI_JOBS=1" "no print run" "$(rewritten)" "PACMAN_OCI_JOBS=1 ..." +PACMAN_OCI_NO_DUMMY=1 run -Su +check "no print run: PACMAN_OCI_NO_DUMMY=1" "no print run" "$(rewritten)" "PACMAN_OCI_NO_DUMMY=1 ..." +run -Su +first="$(rewritten)" +rm -f "$work/args" +rc=0 +PACMAN_OCI_DEBUG=1 "$W" "$dl/bar-1-1-any.pkg.tar.zst.part" "$U/bar-1-1-any.pkg.tar.zst" > /dev/null 2> "$work/err" || rc=$? +check "one print run per pacman run" "-Su :no print run" "$first:$(rewritten)" "two requests in one run, then ls args" + +#---------------------------------------------------------------------------# +# The print lines. A signature comes from %g; the planned sha256 and size of +# each package go to the index; anything else is skipped. +#---------------------------------------------------------------------------# +H1="$(printf 'one' | sha256sum | cut -d ' ' -f 1)" +H2="$(printf 'two' | sha256sum | cut -d ' ' -f 1)" +SIG="$(printf 'SIGNATURE of good\001\000\377' | base64 | tr -d '\n')" +{ + printf 'file:///var/cache/pacman/pkg/cached-1-1-any.pkg.tar.zst\t%s\t0\t%s\n' "$H1" "$SIG" + printf '%s/good-1-1-any.pkg.tar.zst\t%s\t123\t%s\n' "$U" "$H1" "$SIG" + printf 'http://127.0.0.1:9/core/os/x86_64/plain-1-1-any.pkg.tar.zst\t%s\t45\t%s\n' "$H2" "$SIG" + printf '%s/badhex-1-1-any.pkg.tar.zst\txyz\t10\t%s\n' "$U" "$SIG" + printf '%s/badsig-1-1-any.pkg.tar.zst\t%s\t10\t!!not base64\n' "$U" "$H1" + printf '%s/nosig-1-1-any.pkg.tar.zst\t%s\t10\t\n' "$U" "$H2" + printf 'a line pacman prints for itself\n' + printf '%s/three-1-1-any.pkg.tar.zst\t%s\t10\n' "$U" "$H1" + printf '%s/core.db\t%s\t10\t%s\n' "$U" "$H1" "$SIG" + printf '%s/badsize-1-1-any.pkg.tar.zst\t%s\tlots\t%s\n' "$U" "$H1" "$SIG" +} > "$work/lines" +run -Su +K="$$-$start" +sigs="$(find "$dl" -name ".pacman-oci.$K.*.sig" | sed "s|.*/.pacman-oci.$K.||" | sort | tr '\n' ' ')" +check "a .sig is written for each package line with a valid %g" \ + "badhex-1-1-any.pkg.tar.zst.sig good-1-1-any.pkg.tar.zst.sig plain-1-1-any.pkg.tar.zst.sig " "$sigs" \ + "ls -a DIR after a print run with the prepared lines" +check "the index holds each package line with a valid %h" \ + "good-1-1-any.pkg.tar.zst $H1 123|plain-1-1-any.pkg.tar.zst $H2 45|badsig-1-1-any.pkg.tar.zst $H1 10|nosig-1-1-any.pkg.tar.zst $H2 10" \ + "$(tr '\t' ' ' < "$dl/.pacman-oci.$K.,index" | paste -sd '|' -)" "cat DIR/.pacman-oci.KEY.,index" + +rc=0 +PACMAN_OCI_DEBUG=1 "$W" "$dl/good-1-1-any.pkg.tar.zst.sig.part" "$U/good-1-1-any.pkg.tar.zst.sig" > /dev/null 2> "$work/err" || rc=$? +check "the .sig request is served from %g, decoded byte for byte, with no request" \ + "0:$(printf 'SIGNATURE of good\001\000\377' | sha256sum | cut -d ' ' -f 1):0" \ + "$rc:$(sha256sum < "$dl/good-1-1-any.pkg.tar.zst.sig.part" | cut -d ' ' -f 1):$(awk '/^pacman-oci: GET / { n++ } END { print n + 0 }' "$work/err")" \ + "client/pacman-oci DIR/good...sig.part URL" + +PACMAN_OCI_SIG=registry run -Su +check "PACMAN_OCI_SIG=registry: no .sig from %g" "" \ + "$(find "$dl" -name ".pacman-oci.$$-$start.*.sig")" "PACMAN_OCI_SIG=registry ...; ls -a DIR" +: > "$work/lines" + +#---------------------------------------------------------------------------# +# The database prefetch reads pacman's config through pacman-conf, never +# with --sysroot. +#---------------------------------------------------------------------------# +mkdir -p "$work/bin" +cat > "$work/bin/pacman-conf" << 'EOF' +#!/bin/sh +printf '%s\n' "$*" >> "$FAKE_CONF" +case "$*" in + *--repo-list*) printf 'core\n' ;; + *) printf 'oci+http://127.0.0.1:9/ns/mirror/x86_64/core\n' ;; +esac +EOF +chmod +x "$work/bin/pacman-conf" + +# dbrun ARG...: a fresh run with these arguments, then core.db +dbrun() { + start=$((start + 1)) + printf '%s (pacman) S 1 1 1 0 -1 4194560 837 0 0 0 0 0 0 0 20 0 1 0 %s 62676992 3236\n' "$$" "$start" > "$fake/$$/stat" + printf '%s\0' pacman "$@" > "$fake/$$/cmdline" + rm -f "$work/conf.log" + dl="$work/db.$start" + mkdir -p "$dl" + rc=0 + PATH="$work/bin:$PATH" PACMAN_OCI_DEBUG=1 "$W" "$dl/core.db.part" "$U/core.db" > /dev/null 2> "$work/err" || rc=$? +} +conf_calls() { + if [ -f "$work/conf.log" ]; then + paste -sd '|' - < "$work/conf.log" + else + printf 'no call' + fi +} +dbrun -Sy +check "the database prefetch reads /etc/pacman.conf by default" \ + "--config /etc/pacman.conf --repo-list|--config /etc/pacman.conf --repo core Server" "$(conf_calls)" "pacman -Sy; cat conf.log" +dbrun --config /etc/other.conf -Sy +check "the database prefetch reads the config pacman was given" \ + "--config /etc/other.conf --repo-list|--config /etc/other.conf --repo core Server" "$(conf_calls)" "pacman --config /etc/other.conf -Sy" +dbrun --sysroot /mnt -Sy +check "no database prefetch with --sysroot" "no call" "$(conf_calls)" "pacman --sysroot /mnt -Sy" +dbrun --arch aarch64 -Sy +check "no database prefetch with --arch" "no call" "$(conf_calls)" "pacman --arch aarch64 -Sy" + +summary From fab24f462440d179310591a78dc4e1afc262b792 Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 18:46:47 +0545 Subject: [PATCH 10/22] client: answer an absent database signature with an empty file under the sandbox of pacman 7.0 and 7.1 --- client/pacman-oci | 56 ++++++++++++++++++++++++++++++++-- tests/fixtures/client/consumer | 7 ++++- tests/integration/20-client.sh | 8 +++++ 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/client/pacman-oci b/client/pacman-oci index 085b97c..8a8b61e 100755 --- a/client/pacman-oci +++ b/client/pacman-oci @@ -608,6 +608,58 @@ store_open() { return 0 } +# sig_absent MESSAGE: a database signature that the registry does not hold. +# pacman sleeps 2 s after every download that fails, also for an optional +# signature, so each repository costs 2 s on every -Sy. Under the sandbox +# (OUTPUT in a download-XXXXXX directory), pacman 7.0 and 7.1 unlink a +# zero-byte file there and then read the signature as missing (libalpm +# dload.c finalize_download_file): an empty OUTPUT and exit 0 cost no pause. +# With no sandbox the empty file would stay as the signature, so this is a +# miss there, and for every pacman version not known to work this way. +# PACMAN_OCI_EMPTY_SIG=0 turns it off. +sig_absent() { + if [ "${PACMAN_OCI_EMPTY_SIG:-1}" != 0 ] && [ "$kind" = dbsig ] && sandboxed && pacman_unlinks_empty; then + if : > "$out"; then + debug "$1; OUTPUT is empty, and pacman removes it" + exit 0 + fi + fi + miss "$1" +} + +sandboxed() { + case ${out_dir##*/} in + download-?*) return 0 ;; + esac + return 1 +} + +# pacman_unlinks_empty: the parent pacman is a version whose sandbox removes a +# zero-byte download. Asked once per run; the answer is kept in the store. +pacman_unlinks_empty() { + [ -n "$key" ] || return 1 + if entry_ok ,pacver; then + IFS= read -r _pv < "$store,pacver" || _pv='' + else + _pv='' + argv_scan || debug "pacman's arguments: $scan_why" + if find_exe; then + need_work + _rc=0 + "$exe" --version > "$work/pacver" 2>&1 || _rc=$? + if [ "$_rc" = 0 ]; then + _pv=$(sed -n 's/.*Pacman v\([0-9][0-9.]*\).*/\1/p' "$work/pacver") + fi + fi + _t=$(mktemp "${store}tmp.XXXXXX") && printf '%s\n' "${_pv:-unknown}" > "$_t" && put ,pacver "$_t" + fi + case $_pv in + 7.0 | 7.0.* | 7.1 | 7.1.*) return 0 ;; + esac + debug "pacman ${_pv:-of unknown version}: a missing database signature is a miss" + return 1 +} + # serve NAME: a ready entry becomes OUTPUT, and the call ends serve() { entry_ok "$1" || return 1 @@ -1629,7 +1681,7 @@ if [ "$store_ok" = 1 ]; then if [ "$ours" = 1 ]; then serve "$file" if entry_ok ",absent.$file"; then - miss "no layer $file in the database manifest of this run" + sig_absent "no layer $file in the database manifest of this run" fi fi ;; @@ -1879,7 +1931,7 @@ case $idx in if [ "$kind" = pkgsig ]; then broken "$file: no signature layer in $host/$repo:$manifest_ref; pacman tries no other server for a .sig" fi - miss "$host/$repo:$manifest_ref has no layer named $file" + sig_absent "$host/$repo:$manifest_ref has no layer named $file" ;; esac digest=$(flat_get "$work_flat" "/layers/$idx/digest") diff --git a/tests/fixtures/client/consumer b/tests/fixtures/client/consumer index 8bf0f52..c9bbc1b 100644 --- a/tests/fixtures/client/consumer +++ b/tests/fixtures/client/consumer @@ -79,10 +79,14 @@ leftovers() { # sync LABEL ARGS...: one pacman call with the wrapper's debug output sync() { - local label="$1" rc=0 + local label="$1" rc=0 t0 t1 shift + t0="$(date +%s%N)" PACMAN_OCI_DEBUG=1 pac "$@" > "/tmp/$label.out" 2> "/tmp/$label.err" || rc=$? + t1="$(date +%s%N)" result "${label}_rc" "$rc" + result "${label}_ms" "$(((t1 - t0) / 1000000))" + result "${label}_empty_sig" "$(count 'OUTPUT is empty, and pacman removes it' "/tmp/$label.err")" result "${label}_get" "$(count 'pacman-oci: GET ' "/tmp/$label.err")" result "${label}_prefetch_get" "$(count 'pacman-oci: prefetch GET ' "/tmp/$label.err")" result "${label}_served" "$(count ' from the prefetch' "/tmp/$label.err")" @@ -103,6 +107,7 @@ case "$case_name" in fi conf "$du" 'Required DatabaseOptional' sync sy -Sy + result sy_db_sigs "$(find /var/lib/pacman/sync -maxdepth 1 -name '*.sig' | wc -l)" sync s -S alpha beta gamma installed alpha installed beta diff --git a/tests/integration/20-client.sh b/tests/integration/20-client.sh index 6f2525d..a405d08 100644 --- a/tests/integration/20-client.sh +++ b/tests/integration/20-client.sh @@ -186,6 +186,7 @@ for c in main-du main-root; do want "$c" sy_prefetch_get 4 "the first database request reads both manifests and both databases in one prefetch" want "$c" sy_get 0 "no single request: the second database and both missing .db.sig come from the run store" want "$c" sy_served 2 "both databases are handed over from the run store" + want "$c" sy_db_sigs 0 "no database signature is stored in the sync directory" want "$c" s_rc 0 "pacman -S of three packages from two repositories succeeds" want "$c" s_prefetch_get 3 "the prefetch reads the three packages; the signatures come from %PGPSIG%" want "$c" s_get 0 "the serial pass makes no network request" @@ -198,6 +199,13 @@ for c in main-du main-root; do want "$c" main_store_files 0 "no store file is left once pacman has ended" want "$c" main_download_dirs 0 "no download directory is left behind" done +# The 2 s pause pacman takes after each failed download: under the sandbox the +# two absent database signatures are empty files, which pacman removes, so -Sy +# pauses never. With no sandbox they stay misses, so -Sy pauses twice. +want main-du sy_empty_sig 2 "both absent database signatures are answered with an empty file" +want_cmp main-du sy_ms lt 1900 "-Sy of two repositories takes less than one 2 s pause" +want main-root sy_empty_sig 0 "with no sandbox, an absent database signature stays a miss" +want_cmp main-root sy_ms ge 4000 "with no sandbox, pacman pauses 2 s after each absent signature" consumer noprefetch "$SERVER" want noprefetch s_rc 0 "PACMAN_OCI_PREFETCH=0: pacman -S succeeds" From 2c3b48c646b126389f131c4b0a18a80dbee90aed Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 18:51:52 +0545 Subject: [PATCH 11/22] tests: client suite checks the -Sy pause and stored database signatures --- tests/fixtures/client/consumer | 4 ++-- tests/integration/20-client.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/fixtures/client/consumer b/tests/fixtures/client/consumer index c9bbc1b..6e0f6fc 100644 --- a/tests/fixtures/client/consumer +++ b/tests/fixtures/client/consumer @@ -85,7 +85,7 @@ sync() { PACMAN_OCI_DEBUG=1 pac "$@" > "/tmp/$label.out" 2> "/tmp/$label.err" || rc=$? t1="$(date +%s%N)" result "${label}_rc" "$rc" - result "${label}_ms" "$(((t1 - t0) / 1000000))" + result "${label}_wall_ms" "$(((t1 - t0) / 1000000))" result "${label}_empty_sig" "$(count 'OUTPUT is empty, and pacman removes it' "/tmp/$label.err")" result "${label}_get" "$(count 'pacman-oci: GET ' "/tmp/$label.err")" result "${label}_prefetch_get" "$(count 'pacman-oci: prefetch GET ' "/tmp/$label.err")" @@ -107,7 +107,7 @@ case "$case_name" in fi conf "$du" 'Required DatabaseOptional' sync sy -Sy - result sy_db_sigs "$(find /var/lib/pacman/sync -maxdepth 1 -name '*.sig' | wc -l)" + result sy_db_sigs "$(find /var/lib/pacman/sync -maxdepth 1 -name '*.sig' ! -name '.pacman-oci.*' | wc -l)" sync s -S alpha beta gamma installed alpha installed beta diff --git a/tests/integration/20-client.sh b/tests/integration/20-client.sh index a405d08..6807df3 100644 --- a/tests/integration/20-client.sh +++ b/tests/integration/20-client.sh @@ -203,9 +203,9 @@ done # two absent database signatures are empty files, which pacman removes, so -Sy # pauses never. With no sandbox they stay misses, so -Sy pauses twice. want main-du sy_empty_sig 2 "both absent database signatures are answered with an empty file" -want_cmp main-du sy_ms lt 1900 "-Sy of two repositories takes less than one 2 s pause" +want_cmp main-du sy_wall_ms lt 1900 "-Sy of two repositories takes less than one 2 s pause" want main-root sy_empty_sig 0 "with no sandbox, an absent database signature stays a miss" -want_cmp main-root sy_ms ge 4000 "with no sandbox, pacman pauses 2 s after each absent signature" +want_cmp main-root sy_wall_ms ge 4000 "with no sandbox, pacman pauses 2 s after each absent signature" consumer noprefetch "$SERVER" want noprefetch s_rc 0 "PACMAN_OCI_PREFETCH=0: pacman -S succeeds" From e363babeff62b64f04ccb8ab7b81c8524a266aa2 Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 18:58:39 +0545 Subject: [PATCH 12/22] sync: numeric inputs through fromJSON; accept a files database that leaves packages out --- .github/workflows/sync-target.yml | 4 ++-- .github/workflows/sync.yml | 11 +++++++---- scripts/plan | 16 ++++++++++++++-- scripts/verify-batch | 20 ++++++++++++++++++-- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/.github/workflows/sync-target.yml b/.github/workflows/sync-target.yml index 25ef43a..7d5b58e 100644 --- a/.github/workflows/sync-target.yml +++ b/.github/workflows/sync-target.yml @@ -54,7 +54,7 @@ jobs: packages: write strategy: fail-fast: false - max-parallel: ${{ inputs.max_shards }} + max-parallel: ${{ fromJSON(format('{0}', inputs.max_shards)) }} matrix: shard: ${{ fromJSON(inputs.shards) }} steps: @@ -149,7 +149,7 @@ jobs: contents: read strategy: fail-fast: false - max-parallel: ${{ inputs.max_batches }} + max-parallel: ${{ fromJSON(format('{0}', inputs.max_batches)) }} matrix: ${{ fromJSON(needs.stage.outputs.verify_matrix) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index ad77ae9..26335c3 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -261,6 +261,9 @@ jobs: # for its own shards and batches only. A job that calls a reusable workflow # takes no timeout-minutes (GitHub refuses the key there); every job of # sync-target.yml has its own. + # A dispatch input reaches an expression as a string, and max-parallel and a + # number input of a called workflow refuse a string ("Unexpected value '4'", + # run 38569214434). fromJSON(format('{0}', x)) gives a number in every case. target: needs: plan if: ${{ needs.plan.outputs.has_targets == 'true' }} @@ -269,15 +272,15 @@ jobs: packages: write strategy: fail-fast: false - max-parallel: ${{ inputs.max_targets || 4 }} + max-parallel: ${{ fromJSON(format('{0}', inputs.max_targets || 4)) }} matrix: ${{ fromJSON(needs.plan.outputs.targets) }} uses: ./.github/workflows/sync-target.yml with: target: ${{ matrix.target }} shards: ${{ toJSON(matrix.shards) }} - max_shards: ${{ inputs.max_shards || 3 }} - max_batches: ${{ inputs.max_batches || 3 }} - parallel: ${{ inputs.parallel || 6 }} + max_shards: ${{ fromJSON(format('{0}', inputs.max_shards || 3)) }} + max_batches: ${{ fromJSON(format('{0}', inputs.max_batches || 3)) }} + parallel: ${{ fromJSON(format('{0}', inputs.parallel || 6)) }} debloat: ${{ inputs.debloat || 'auto' }} report: diff --git a/scripts/plan b/scripts/plan index 5e062d7..431a093 100755 --- a/scripts/plan +++ b/scripts/plan @@ -355,10 +355,22 @@ plan_target() { warn "$name: $srv/$repo.files does not read as a tar archive" continue fi - if [ "$(cut -f 1,5 "$dir/files.tsv" | sort | sha256sum)" != "$(cut -f 1,5 "$dir/records.tsv" | sort | sha256sum)" ]; then - warn "$name: $srv/$repo.files names other packages than $repo.db" + # Every (file, sha256) of the files database must be in the database. A + # files database that leaves packages out is accepted and the packages are + # named: upstream serves it so to its own users too (Arch Linux ARM + # aarch64 aur.files left out localepurge, 2026-10-14), and pacman -F only + # knows less. One that names a package the database does not is refused. + cut -f 1,5 "$dir/files.tsv" | sort -u > "$dir/files-pairs.txt" + cut -f 1,5 "$dir/records.tsv" | sort -u > "$dir/db-pairs.txt" + comm -23 "$dir/files-pairs.txt" "$dir/db-pairs.txt" > "$dir/files-extra.txt" + comm -13 "$dir/files-pairs.txt" "$dir/db-pairs.txt" | cut -f 1 > "$dir/files-missing.txt" + if [ -s "$dir/files-extra.txt" ]; then + warn "$name: $srv/$repo.files names $(awk 'END { print NR + 0 }' "$dir/files-extra.txt") package file(s) that $repo.db does not, e.g. $(awk -F '\t' 'NR == 1 { print $1 " sha256 " $2 }' "$dir/files-extra.txt")" continue fi + if [ -s "$dir/files-missing.txt" ]; then + warn "$name: $srv/$repo.files leaves out $(awk 'END { print NR + 0 }' "$dir/files-missing.txt") package(s) of $repo.db, as upstream serves it: $(tr '\n' ' ' < "$dir/files-missing.txt")" + fi # A signature is absent only when the server says so with a 404; any other # failure would otherwise turn a signed database into an unsigned one. sig_done=1 diff --git a/scripts/verify-batch b/scripts/verify-batch index 5ba35dc..d972f95 100755 --- a/scripts/verify-batch +++ b/scripts/verify-batch @@ -410,8 +410,24 @@ if [ "$got" != "$files_sha" ]; then exit 1 fi echo "the synced $repo.files is the staged one, sha256 $got" -xargs -a /in/names.txt pacman --config /tmp/pacman.conf -Fl > /tmp/files-list.txt -echo "pacman -Fl listed $(wc -l < /tmp/files-list.txt) path(s) for $(wc -l < /in/names.txt) package(s)" +# plan accepts a files database that leaves out packages of the database (an +# upstream defect: Arch Linux ARM aarch64 aur.files, 2026-10-14). -Fl reads +# every batch name that the files database holds, and the rest are named. +bsdtar -tf "/var/lib/pacman/sync/$repo.files" | sed -n 's|/files$||p' | sort -u > /tmp/files-dirs.txt +pacman --config /tmp/pacman.conf -Sl "$repo" | awk '{ print $2 "\t" $2 "-" $3 }' > /tmp/name-dir.tsv +awk -F '\t' 'FILENAME == ARGV[1] { have[$1] = 1; next } + FILENAME == ARGV[2] { dir[$1] = $2; next } + { if (($0 in dir) && (dir[$0] in have)) print > "/tmp/fl-names.txt"; else print > "/tmp/fl-absent.txt" } +' /tmp/files-dirs.txt /tmp/name-dir.tsv /in/names.txt +touch /tmp/fl-names.txt /tmp/fl-absent.txt +if [ -s /tmp/fl-absent.txt ]; then + echo "the files database holds no entry for $(wc -l < /tmp/fl-absent.txt) package(s) of the batch, as upstream serves it: $(tr '\n' ' ' < /tmp/fl-absent.txt)" +fi +: > /tmp/files-list.txt +if [ -s /tmp/fl-names.txt ]; then + xargs -a /tmp/fl-names.txt pacman --config /tmp/pacman.conf -Fl > /tmp/files-list.txt +fi +echo "pacman -Fl listed $(wc -l < /tmp/files-list.txt) path(s) for $(wc -l < /tmp/fl-names.txt) package(s)" phase "done" echo "@@done" From 59b3412ebd09652d6b16d9438517a74fcc6821a3 Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 19:07:42 +0545 Subject: [PATCH 13/22] mirror-file: a 429 from the registry pauses every push of the shard and costs no attempt --- scripts/mirror-file | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/scripts/mirror-file b/scripts/mirror-file index 633921c..35c3da7 100755 --- a/scripts/mirror-file +++ b/scripts/mirror-file @@ -253,18 +253,56 @@ fi pushed=0 push_started="$(date -u +%s)" -for attempt in 1 2 3; do +# GHCR limits the requests of one token (measured: "toomanyrequests ... +# allowed: 2000/minute", run 38570044665), and every push of every job shares +# the workflow's token. A 429 costs no attempt. The first push of the shard +# that meets one writes $WORK/throttle, the time its pause ends, and every +# push of the shard waits for that time first. The pauses of one file may +# add up to MIRROR_RATE_WAIT seconds (default 1800). +RATE_WAIT="${MIRROR_RATE_WAIT:-1800}" +throttle_wait() { + local until now + [ -f "$WORK/throttle" ] || return 0 + until="$(cat "$WORK/throttle")" + now="$(date -u +%s)" + case "$until" in + '' | *[!0-9]*) return 0 ;; + esac + if [ "$until" -gt "$now" ]; then + sleep "$((until - now))" + fi +} +attempt=0 +waited=0 +while :; do + throttle_wait rc=0 (cd -- "$dir" && timeout --kill-after=30 "$deadline" "$ORAS" "${push_args[@]}" > "$dir/push.log" 2>&1) || rc=$? if [ "$rc" -eq 0 ]; then pushed=1 break fi + if grep -qiE 'toomanyrequests|429 Too Many' "$dir/push.log"; then + pause=$((20 + RANDOM % 41)) + waited=$((waited + pause)) + if [ "$waited" -gt "$RATE_WAIT" ]; then + warn "$file: the registry still limits pushes after ${waited}s of pauses" + break + fi + tmp="$(mktemp "$WORK/.throttle.XXXXXX")" + printf '%s\n' "$(($(date -u +%s) + pause))" > "$tmp" + mv -f -- "$tmp" "$WORK/throttle" + log "$file: the registry limits pushes (429); every push of the shard waits ${pause}s" + sleep "$pause" + continue + fi + attempt=$((attempt + 1)) warn "$file: push attempt $attempt ended $rc" "$(tail -n 3 "$dir/push.log")" + [ "$attempt" -lt 3 ] || break sleep $((attempt * attempt * 5)) done if [ "$pushed" -ne 1 ]; then - result failed "oras push failed three times, last: $(tail -n 1 "$dir/push.log")" + result failed "oras push failed ($attempt attempt(s), ${waited}s of 429 pauses), last: $(tail -n 1 "$dir/push.log")" exit 1 fi push_seconds=$(($(date -u +%s) - push_started)) From a1cf1847729ac922117c4096d4a7282af82e590d Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 21:00:34 +0545 Subject: [PATCH 14/22] workflows: deploy and restarts not held by skipped jobs; Worker check hourly in status; per-target artifact patterns --- .github/workflows/status.yml | 65 +++++++++++++++++++++++++++++-- .github/workflows/sync-target.yml | 6 +-- .github/workflows/sync.yml | 3 +- .github/workflows/watchdog.yml | 64 +++--------------------------- scripts/check-staleness | 4 +- scripts/dispatch-overdue | 13 +++++++ tests/integration/10-pipeline.sh | 4 +- tests/unit/45-dispatch-overdue.sh | 7 +++- 8 files changed, 98 insertions(+), 68 deletions(-) diff --git a/.github/workflows/status.yml b/.github/workflows/status.yml index 5904381..3ef329f 100644 --- a/.github/workflows/status.yml +++ b/.github/workflows/status.yml @@ -62,8 +62,10 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - scripts/dispatch-overdue sync.yml 240 - scripts/dispatch-overdue watchdog.yml 1560 + rc=0 + scripts/dispatch-overdue sync.yml 240 || rc=$? + scripts/dispatch-overdue watchdog.yml 1560 || rc=$? + exit "$rc" decide: runs-on: ubuntu-24.04 @@ -241,9 +243,12 @@ jobs: with: path: ${{ runner.temp }}/site + # Each of these names its needs' results: with no status function an if gets + # an implicit success() over every job before it, and bench is skipped in + # most runs. deploy: needs: build - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ !cancelled() && needs.build.result == 'success' && github.ref == 'refs/heads/main' }} runs-on: ubuntu-24.04 timeout-minutes: 15 permissions: @@ -283,3 +288,57 @@ jobs: id: deploy if: ${{ steps.pages.outputs.deploy == 'true' }} uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 + + # The Worker check reads the record build has just pushed to the status + # branch. Here, not in watchdog.yml: an hourly cron there would cancel a + # pending daily run, which shares its concurrency group. + worker: + needs: build + if: ${{ !cancelled() && needs.build.result == 'success' && github.ref == 'refs/heads/main' }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Raise worker-down after 2 failed status runs in a row + env: + GH_TOKEN: ${{ github.token }} + run: | + # status.yml probes the Worker each hour and counts the runs in a + # row with a failed probe (scripts/status-collect). This reads its + # last record from the status branch. + rc=0 + git ls-remote --exit-code --heads origin status > "$RUNNER_TEMP/heads.txt" || rc=$? + case "$rc" in + 0) ;; + 2) + echo "::notice::no status branch yet: status.yml has recorded no Worker probe" + exit 0 + ;; + *) + echo "::error::git ls-remote origin status ended $rc" + exit "$rc" + ;; + esac + git fetch --quiet --depth 1 origin status + git show FETCH_HEAD:status.json > "$RUNNER_TEMP/status.json" + git show FETCH_HEAD:worker.md > "$RUNNER_TEMP/worker.md" + read -r generated age failed ok <<< "$(jq -r '[.generated, (now - .generated_epoch | floor), .worker.failed_runs, .worker.ok] | map(tostring) | join(" ")' "$RUNNER_TEMP/status.json")" + echo "status.json from $generated, $age s ago: Worker ok $ok, failed $failed run(s) in a row" + # Three status runs missed: the record says nothing about now. + if [ "$age" -gt 10800 ]; then + echo "::error::the last Worker probe is from $generated, $age s ago; status.yml has stopped recording" + echo " reproduce: git fetch origin status && git show FETCH_HEAD:status.json | jq '{generated, worker}'" + exit 1 + fi + if [ "$ok" = true ]; then + scripts/track-issue clear worker-down "The Worker passes its probes again, measured $generated: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/status/worker.md" + elif [ "$ok" = false ] && [ "$failed" -ge 2 ]; then + scripts/track-issue raise worker-down "The Worker fails its probe" "$RUNNER_TEMP/worker.md" + else + echo "nothing to raise or clear: worker-down opens at 2 failed runs in a row" + fi diff --git a/.github/workflows/sync-target.yml b/.github/workflows/sync-target.yml index 7d5b58e..ec414e2 100644 --- a/.github/workflows/sync-target.yml +++ b/.github/workflows/sync-target.yml @@ -112,7 +112,7 @@ jobs: path: ${{ runner.temp }}/plan/${{ inputs.target }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - pattern: results-${{ inputs.target }}-* + pattern: results-${{ inputs.target }}-[0-9]* path: ${{ runner.temp }}/results merge-multiple: false - name: Stage @@ -197,7 +197,7 @@ jobs: path: ${{ runner.temp }}/stage - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - pattern: verify-${{ inputs.target }}-* + pattern: verify-${{ inputs.target }}-[0-9]* path: ${{ runner.temp }}/verify merge-multiple: false - name: Promote @@ -248,7 +248,7 @@ jobs: merge-multiple: true - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - pattern: verify-${{ inputs.target }}-* + pattern: verify-${{ inputs.target }}-[0-9]* path: ${{ runner.temp }}/verify merge-multiple: false - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 26335c3..df705e7 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -343,8 +343,9 @@ jobs: GH_TOKEN: ${{ github.token }} run: scripts/check-schedules-fired # Each scheduled workflow starts the others when theirs did not fire. + # always(): a check above that fails must not stop the restart. - name: Start an overdue status run - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ always() && github.ref == 'refs/heads/main' }} env: GH_TOKEN: ${{ github.token }} run: scripts/dispatch-overdue status.yml 120 diff --git a/.github/workflows/watchdog.yml b/.github/workflows/watchdog.yml index eb40cf1..b7815fb 100644 --- a/.github/workflows/watchdog.yml +++ b/.github/workflows/watchdog.yml @@ -1,20 +1,16 @@ # Measure the outcome, not the runs: has each target's latest stopped # following upstream? Have the schedules stopped firing? Does the Worker -# still answer? # # ⭐ sync.yml's keepalive job runs the same schedule checks, so each workflow # witnesses the other. A schedule that stops firing leaves no run behind to # fail; this is the run that notices. # -# Two crons. 41 7 runs every job once a day. 7 * runs only the Worker check, -# each hour, 20 minutes after status.yml records its probes: a Worker that -# fails its probe in 2 status runs in a row opens worker-down within the hour. +# Once a day. The hourly Worker check is a job of status.yml. name: Watchdog on: schedule: - cron: '41 7 * * *' - - cron: '7 * * * *' workflow_dispatch: permissions: {} @@ -29,7 +25,6 @@ defaults: jobs: staleness: - if: ${{ github.event_name != 'schedule' || github.event.schedule == '41 7 * * *' }} runs-on: ubuntu-24.04 timeout-minutes: 20 permissions: @@ -69,7 +64,6 @@ jobs: exit "$rc" schedules: - if: ${{ github.event_name != 'schedule' || github.event.schedule == '41 7 * * *' }} runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: @@ -88,59 +82,13 @@ jobs: GH_TOKEN: ${{ github.token }} run: scripts/check-schedules-fired # Each scheduled workflow starts the others when theirs did not fire. + # always(): a check above that fails must not stop the restart. - name: Start an overdue sync or status run - if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ always() && github.ref == 'refs/heads/main' }} env: GH_TOKEN: ${{ github.token }} run: | - scripts/dispatch-overdue sync.yml 240 - scripts/dispatch-overdue status.yml 120 - - worker: - runs-on: ubuntu-24.04 - timeout-minutes: 10 - permissions: - contents: read - issues: write - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Raise worker-down after 2 failed status runs in a row - env: - GH_TOKEN: ${{ github.token }} - run: | - # status.yml probes the Worker each hour and counts the runs in a - # row with a failed probe (scripts/status-collect). This reads its - # last record from the status branch. rc=0 - git ls-remote --exit-code --heads origin status > "$RUNNER_TEMP/heads.txt" || rc=$? - case "$rc" in - 0) ;; - 2) - echo "::notice::no status branch yet: status.yml has recorded no Worker probe" - exit 0 - ;; - *) - echo "::error::git ls-remote origin status ended $rc" - exit "$rc" - ;; - esac - git fetch --quiet --depth 1 origin status - git show FETCH_HEAD:status.json > "$RUNNER_TEMP/status.json" - git show FETCH_HEAD:worker.md > "$RUNNER_TEMP/worker.md" - read -r generated age failed ok <<< "$(jq -r '[.generated, (now - .generated_epoch | floor), .worker.failed_runs, .worker.ok] | map(tostring) | join(" ")' "$RUNNER_TEMP/status.json")" - echo "status.json from $generated, $age s ago: Worker ok $ok, failed $failed run(s) in a row" - # Three status runs missed: the record says nothing about now. - if [ "$age" -gt 10800 ]; then - echo "::error::the last Worker probe is from $generated, $age s ago; status.yml has stopped recording" - echo " reproduce: git fetch origin status && git show FETCH_HEAD:status.json | jq '{generated, worker}'" - exit 1 - fi - if [ "$ok" = true ]; then - scripts/track-issue clear worker-down "The Worker passes its probes again, measured $generated: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/status/worker.md" - elif [ "$ok" = false ] && [ "$failed" -ge 2 ]; then - scripts/track-issue raise worker-down "The Worker fails its probe" "$RUNNER_TEMP/worker.md" - else - echo "nothing to raise or clear: worker-down opens at 2 failed runs in a row" - fi + scripts/dispatch-overdue sync.yml 240 || rc=$? + scripts/dispatch-overdue status.yml 120 || rc=$? + exit "$rc" diff --git a/scripts/check-staleness b/scripts/check-staleness index a60caee..ed494a5 100755 --- a/scripts/check-staleness +++ b/scripts/check-staleness @@ -79,7 +79,9 @@ added_date() { "reproduce: git -C $dir rev-parse --is-shallow-repository" return 0 fi - git -C "$dir" log --diff-filter=A --format=%cs -- "$base" | awk 'NR == 1' + # --first-parent: the date CONF reached this branch (the merge or squash + # commit), not the date of a commit on the branch it came from. + git -C "$dir" log --first-parent --diff-filter=A --format=%cs -- "$base" | awk 'NR == 1' } worst=0 diff --git a/scripts/dispatch-overdue b/scripts/dispatch-overdue index b6375b6..25ce746 100755 --- a/scripts/dispatch-overdue +++ b/scripts/dispatch-overdue @@ -52,6 +52,19 @@ esac work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT +# A workflow that is not active is not started: GitHub refuses the dispatch. +# One a person disabled stays so (scripts/keep-schedules-enabled leaves it +# too), and one GitHub disabled for inactivity is enabled by that script. +rc=0 +gh api "repos/$slug/actions/workflows/$wf" > "$work/workflow.json" || rc=$? +[ "$rc" -eq 0 ] || die "could not read the state of $wf (gh ended $rc)" \ + "reproduce: gh api repos/$slug/actions/workflows/$wf --jq .state" +state="$(jq -r '.state // "unknown"' "$work/workflow.json")" +if [ "$state" != active ]; then + echo "::notice::dispatch-overdue: $wf is $state; it is not started" + exit 0 +fi + rc=0 gh api "repos/$slug/actions/workflows/$wf/runs?branch=main&per_page=10" > "$work/runs.json" || rc=$? [ "$rc" -eq 0 ] || die "could not list the runs of $wf (gh ended $rc)" \ diff --git a/tests/integration/10-pipeline.sh b/tests/integration/10-pipeline.sh index d5f02cc..c8f43f5 100644 --- a/tests/integration/10-pipeline.sh +++ b/tests/integration/10-pipeline.sh @@ -214,7 +214,9 @@ sync() { > "$o/collect-$t.log" 2>&1 done rc=0 - "$REPO_ROOT/scripts/report" "$o/plan" "$o/summaries" > "$o/report.md" 2> "$o/report.log" || rc=$? + # report writes its table to GITHUB_STEP_SUMMARY when that is set, as it is + # on a runner; here the table must go to stdout. + env -u GITHUB_STEP_SUMMARY "$REPO_ROOT/scripts/report" "$o/plan" "$o/summaries" > "$o/report.md" 2> "$o/report.log" || rc=$? printf '%s\n' "$rc" > "$o/report.rc" return 0 } diff --git a/tests/unit/45-dispatch-overdue.sh b/tests/unit/45-dispatch-overdue.sh index 994a040..44112b6 100644 --- a/tests/unit/45-dispatch-overdue.sh +++ b/tests/unit/45-dispatch-overdue.sh @@ -18,10 +18,13 @@ cat > "$work/bin/gh" << 'EOF' #!/usr/bin/env bash printf 'gh %s\n' "$*" >> "$STUB_DIR/calls.log" case "$1 $2" in - "api repos/"*) + "api repos/"*/runs\?*) [ "${STUB_FAIL:-0}" = 1 ] && exit 4 cat "$STUB_DIR/runs.json" ;; + "api repos/"*) + printf '{"state":"%s"}\n' "${STUB_STATE:-active}" + ;; "workflow run") exit 0 ;; *) echo "stub gh: no answer for: gh $*" >&2 @@ -65,5 +68,7 @@ case_run "a workflow name that is not a file name" '{"workflow_runs":[]}' 2 no ' case_run "a limit that is not a number" '{"workflow_runs":[]}' 2 no sync.yml soon STUB_FAIL=1 case_run "the runs list does not read" '{"workflow_runs":[]}' 1 no sync.yml 240 +STUB_STATE=disabled_manually case_run "a workflow a person disabled" '{"workflow_runs":[]}' 0 no sync.yml 240 +STUB_STATE=disabled_inactivity case_run "a workflow GitHub disabled for inactivity" '{"workflow_runs":[]}' 0 no sync.yml 240 summary From 48ad4519e59621988373ca0aecf8ea29a098a6c5 Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 21:17:50 +0545 Subject: [PATCH 15/22] docs: ASD-STE100 rewrite; remove narrative history --- .github/workflows/status.yml | 3 +- .github/workflows/sync.yml | 2 +- .github/workflows/watchdog.yml | 2 +- HISTORY/CONTINUE.md | 90 -------- HISTORY/design.md | 272 ------------------------- HISTORY/tests-seen-to-fail.md | 103 ---------- README.md | 361 ++++++++++++++++++++------------- client/pacman-oci | 7 +- docs/client.md | 247 ++++++++++++++++++++++ docs/conventions.md | 89 ++++++++ docs/design.md | 236 +++++++++++++++++++++ docs/measurements.md | 240 ++++++++++++++++++++++ docs/operations.md | 190 +++++++++++++++++ docs/tests.md | 334 ++++++++++++++++++++++++++++++ docs/worker.md | 155 ++++++++++++++ scripts/check-schedules-fired | 2 +- scripts/cron-tolerance | 2 +- scripts/install-oras | 2 +- scripts/mirror-file | 3 +- scripts/report | 2 +- scripts/status-collect | 2 +- 21 files changed, 1731 insertions(+), 613 deletions(-) delete mode 100644 HISTORY/CONTINUE.md delete mode 100644 HISTORY/design.md delete mode 100644 HISTORY/tests-seen-to-fail.md create mode 100644 docs/client.md create mode 100644 docs/conventions.md create mode 100644 docs/design.md create mode 100644 docs/measurements.md create mode 100644 docs/operations.md create mode 100644 docs/tests.md create mode 100644 docs/worker.md diff --git a/.github/workflows/status.yml b/.github/workflows/status.yml index 3ef329f..20e8bd1 100644 --- a/.github/workflows/status.yml +++ b/.github/workflows/status.yml @@ -10,12 +10,13 @@ # branch status as one orphan commit, and the site to Pages as an # artifact # deploy actions/deploy-pages, on main, when Pages takes a workflow build +# worker on main: raises or clears worker-down from the record build pushed # # ⚠ A person enables Pages once, with an admin token. GITHUB_TOKEN cannot: # gh api -X POST repos/pkgforge-dev/mirror-archlinux/pages -f build_type=workflow # Until then deploy says so in a notice and deploys nothing. # -# The status branch holds the history the next run reads. watchdog.yml reads +# The status branch holds the history the next run reads. The worker job reads # status.json and worker.md from it to raise worker-down. name: Status diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index df705e7..0dcbcb9 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -6,7 +6,7 @@ # registry and every file new since the promoted database has passed pacman's # own checks through the wrapper. Targets are independent: each runs its own # chain, and a slow or held target never delays or holds another. -# HISTORY/design.md has the whole design. +# docs/design.md has the whole design. # # promote runs only on main. A run from any other branch does everything else, # writes only write-once package tags and its own staging- tag, and diff --git a/.github/workflows/watchdog.yml b/.github/workflows/watchdog.yml index b7815fb..f692154 100644 --- a/.github/workflows/watchdog.yml +++ b/.github/workflows/watchdog.yml @@ -1,5 +1,5 @@ # Measure the outcome, not the runs: has each target's latest stopped -# following upstream? Have the schedules stopped firing? Does the Worker +# following upstream? Have the schedules stopped firing? # # ⭐ sync.yml's keepalive job runs the same schedule checks, so each workflow # witnesses the other. A schedule that stops firing leaves no run behind to diff --git a/HISTORY/CONTINUE.md b/HISTORY/CONTINUE.md deleted file mode 100644 index 1f5acde..0000000 --- a/HISTORY/CONTINUE.md +++ /dev/null @@ -1,90 +0,0 @@ -# Continue here - -What anyone picking this repository up needs first: the conventions, where -things stand, and what is left. - -## Conventions - -- One squashed commit per piece of work on `main`, by the maintainer's own - identity. No co-author or generated-by lines. -- Scripts have no extension, are executable in the git index (except - `scripts/lib.sh`, which is sourced), start with `set -euo pipefail`, and say in - their header what they do, how to call them and what their exit codes mean. - `client/pacman-oci` is POSIX sh and is tested under bash, dash and busybox. -- A failure is never hidden: no `|| true`, no `2>/dev/null` on anything that - matters, no `set +e`, no `continue-on-error`. An exit code is captured as - `rc=0; cmd || rc=$?`. Every curl has a connect deadline and a transfer or - stall deadline; every oras call runs under `timeout`. `tests/static/30` checks - all of it. -- Every failure message says what was measured and how to reproduce it. A - script that sources `scripts/lib.sh` also reports, through an ERR trap, the - command that ended it under `set -e`, so no exit is silent. -- Every test has been seen to fail: `HISTORY/tests-seen-to-fail.md`. -- Docs: no em dashes, no marketing adjectives, and only the markers ⛔ ⭐ ⚠. - -## Where things stand (2026-10-14) - -- x86_64 core is mirrored to `ghcr.io/pkgforge-dev/mirror-archlinux/x86_64/core` - every three hours, verified by pacman through the wrapper before promotion. - The first promotion from `main` was run 38490975315 on 2026-10-14: `latest` - and `2026.10.14` name the same snapshot, and a stock Arch pacman with it as - core's only Server ran `-Sy` in 4.0 s and `-S nano` in 5.1 s from a - residential network. - The package is public: the organisation creates packages public. -- Tests: the static suite (shellcheck, actionlint, seven policy files), the unit - suite under bash and gawk, dash and mawk, and busybox, and the integration - suite, which drives the whole mirror against a local registry and installs - through the wrapper with and without `DownloadUser`. Every test has been seen - to fail (`HISTORY/tests-seen-to-fail.md`). -- The design was reviewed before it was built, and the implementation after - (`HISTORY/design.md`, both review sections); every confirmed finding is fixed - and tested. -- docker-archlinux is unchanged. Nothing reads the mirror yet but its own tests. - -## Maintainer actions - -Done 2026-10-13, with the maintainer's approval: - -- `default_workflow_permissions = read`, `can_approve_pull_request_reviews = false`. -- `sha_pinning_required = true`: only actions pinned by commit run. - -Left to the maintainer: - -- A GHCR package created by the workflow may come out private, and GitHub has - no API to change that. `visibility-` opens with the settings link - when it does; everything is pushed and verified meanwhile. -- `replaced-`: a file whose tag already holds other bytes. Arch never - republishes a filename with new bytes, so this means something unusual - upstream or on the registry. The issue names the file and both digests. -- Renovate proposes digest updates for the verify image and the test images, - and action updates weekly; nothing merges on its own here. - -## Phase 2, in order - -1. **ppc.** Add `powerpc64le-base`, `powerpc64le-base-any` and the powerpc - and powerpc64 equivalents with `db_signed = yes` (below), and a verify - platform for them (binfmt and QEMU on the runner, which the ubuntu image - lacks). -2. **`db_signed = yes`.** Plan already refuses a database whose `.sig` answers - anything but 404, and keeps the `.sig` it finds. Still to build when the - first signed target arrives: refuse a missing `.sig` (done in plan), check - it with the pinned keyring before staging, verify with `DatabaseRequired` - (done in verify-batch), and tell that repository's consumers to set it. - Measure how often a consumer meets the `.db` and `.db.sig` race before - adding any pinning to the wrapper. -3. **docker-archlinux.** Decide with the maintainer whether images enable the - `XferCommand` for every repository, only for repositories whose upstream is - blocked, or only in the build stage. A global one costs about 2 s per - repository per `-Sy` and makes downloads sequential (HISTORY/design.md). - Add the mirror's `latest` age to docker-archlinux's freshness check, so a - schedule stop here is seen from there too. -4. **x86_64 extra.** 107.6 GiB, 15041 files: about 20 shards at the current - limits, and probably two runs, since upstream supersedes files during a long - seed. Its keyring is in core: add a `keyring_from = x86_64-core` key that puts - the mirrored core into verify's pacman.conf. -5. **A staleness rule for targets a runner cannot read**, where the watchdog - cannot compare with upstream: the age of `fetched` alone. - -## Open measurements - -`HISTORY/design.md`, "Open, to be measured". diff --git a/HISTORY/design.md b/HISTORY/design.md deleted file mode 100644 index 62f74d2..0000000 --- a/HISTORY/design.md +++ /dev/null @@ -1,272 +0,0 @@ -# Design - -Arch Linux repositories on GHCR as OCI artifacts, read by pacman through an -`XferCommand`. Written 2026-10-13, when the mirror was built; every number in -it was measured that day unless it says otherwise. - -## Why - -GitHub's runners are refused by upstream mirrors more and more. ArchPOWER, the -only publisher of the PowerPC ports, answers every datacentre address with a -Cloudflare challenge (HTTP 403), measured from runners on 2026-08-28 and -2026-09-18. docker-archlinux held its ppc images back from 2026-09-18 for it -(its `HISTORY/outage-2026-09.md`). A self-run mirror on a rented server was the -alternative, and was set aside for cost. GHCR serves GitHub's own runners, and -public packages cost nothing to store. - -## Decisions by the maintainer (2026-10-13) - -1. One GHCR repository per arch and repo: - `ghcr.io/pkgforge-dev/mirror-archlinux//`, `` being - pacman's `$arch`. -2. Each package file is one tag, its escaped filename; its signature is a - second layer of the same manifest. The database is tag `latest`, plus a - dated tag per UTC day. -3. Consumers: `XferCommand = /usr/local/bin/pacman-oci %o %u` and - `Server = oci://ghcr.io/pkgforge-dev/mirror-archlinux/$arch/$repo`, with the - upstream Servers after it as the fallback. -4. Keep everything; nothing is ever deleted. -5. Runners are debloated when the work needs the room, and the work runs in - parallel as far as it usefully goes. -6. Phase 1 mirrors x86_64 core only, sharded anyway, and changes nothing in - docker-archlinux. ppc ingest is a later phase. -7. The repository's Actions settings were hardened on 2026-10-13: the default - token is read-only, Actions cannot approve pull requests, and only actions - pinned by commit may run. - -## Measurements that shaped it - -| what | measured | -| --- | --- | -| GHCR, `Authorization: Bearer QQ==` | 200 on a public manifest and blob; no header 401; an unrecognised bearer 403 with no challenge | -| GHCR blob GET | 307 to a signed URL on `pkg-containers.githubusercontent.com`; curl does not carry the Authorization header across hosts, and the URL needs none; ranges work | -| GHCR blobs | scoped per repository: another repository's blob answers 404 | -| GHCR tags | resolved without regard to case: `latest`, `LATEST` and `Latest` name one manifest | -| GHCR, a 129 character reference | 303 to an HTML page on github.com | -| GHCR limits (docs) | 10 GB per layer, a 10 minute upload | -| GHCR visibility | no REST endpoint changes it (the packages API offers GET, DELETE and restore only) | -| pacman 7.1.0 `XferCommand` | fork and execvp, no shell; `%o` and `%u` replaced only as whole words; exec failure is 127, like any failed download | -| pacman, after a failure | sleeps 2 s, tries the next Server; never deletes the `.part` | -| pacman, downloads | one at a time under an `XferCommand` | -| pacman, a package's `.sig` | asked only of the server that served the package; when the signature is required, a failed `.sig` ends the transaction | -| pacman, with `DownloadUser` | the `XferCommand` runs as that user (uid switch and NO_NEW_PRIVS only), from `/download-XXXXXX/` | -| pacman, signatures | checked against the database's `%PGPSIG%` when present, else the detached `.sig`; `-Sw` checks both sha256 and signatures | -| pacman 7.1, a package with `%PGPSIG%` | still asks the `XferCommand` for its `.sig` and stores it in the cache, unread (2026-10-14) | -| `%PGPSIG%` coverage | Arch x86_64 core and extra 100%; aarch64, armv7h 100%; loong64 core 263/295; riscv64 core 9/294; ArchPOWER base-powerpc64le 0/3735, base-any 262/2200 | -| `%PGPSIG%` against `.sig` | base64 decoded, byte identical (zlib) | -| filenames | about 44000 across nine databases, all `[A-Za-z0-9@._+:-]`, longest 92; `:` in 761, `+` in 246; a few dozen with upper case | -| newest `%BUILDDATE%` | not monotonic: archived core of 2026-10-02 and 10-03 differ and share it, so do 10-10 and 10-11 | -| per package `%BUILDDATE%` | over all eleven consecutive daily pairs 10-01 to 10-12, the newer database never had an older package and the older always had one when the bytes differed | -| superseded files | gone from every Arch mirror within a day, in `os/` and `pool/` alike | -| sizes | x86_64 core 299 packages, 1.85 GiB; extra 15041, 107.6 GiB, 10680 under 1 MiB, largest 3.29 GB | -| runners (docs and maintainers) | 4 vCPU; about 87 GB free on `/`, 14 GB guaranteed; `/mnt` not guaranteed | -| pkgforge-dev | GitHub Free: 20 concurrent jobs shared by every repository of the organisation | - -## Storage layout - -``` -ghcr.io/pkgforge-dev/mirror-archlinux/x86_64/core - : one package: the file, and its signature - :staging the candidate database of the run on main - :staging- the candidate of a run from another branch - :latest the promoted database - :YYYY.MM.DD the first database promoted that UTC day; never moves - :db- the promoted database by content - :files- -``` - -### Tags - -A package tag is the filename with `+` as `_p`, `:` as `_c`, `@` as `_a`, and a -literal `_` as `_u` only when the next character is one of `p c a u`, so -`x86_64` stays `x86_64` and the mapping is injective. Tags are lower case -because GHCR folds case: a filename with upper case, or one the mapping cannot -carry within 128 characters, is stored under `fn-` and the hex sha256 of the -name. That form has no dot, and every escaped package name has `.pkg.tar`, so -the two never meet, and neither meets `latest`, `staging` or a date. The -mapping lives in `client/pacman-oci --tag` and nowhere else; the scripts call -it. - -The judge of the design review asked for `_` as `__` (the draft's rule). The -rule above was kept instead: it is injective too, keeps the names readable as -the maintainer approved them, and is lower case throughout. - -### Manifests - -Package: artifact type `application/vnd.pkgforge.archlinux.package.v1`, layer 0 -the package (`application/vnd.pkgforge.archlinux.pkg.v1`), layer 1 its -signature (`application/pgp-signature`), each titled with its filename. -Database: artifact type `application/vnd.pkgforge.archlinux.db.v1`, layers -`.db` and `.files` and any `.sig` upstream publishes beside them, -byte for byte, annotated with the upstream server, both sha256 values, the -package count, the newest build date and the ISO 8601 time it was fetched. - -## The invariant - -⛔ A database reaches `latest` only after every file it names is on the -registry and every file new since the promoted database has passed pacman's -own checks through the wrapper. - -## Rules, and the failure each one prevents - -- **The database comes from the distribution's own servers only.** Arch - databases are unsigned, and whoever supplies the database chooses which - validly signed packages are installed. Package bytes may come from any - server; each is checked against the database. -- **Never behind.** A candidate is dropped when some package in it was built - before the same package in the promoted database, or in another candidate. - `allow_older` lifts the first comparison for a deliberate upstream downgrade. - A candidate that differs from the promoted database only by packages added - or dropped shows no build date either way, and may be an older copy from a - server that lags: while a db_server still serves the promoted database, that - one is kept, and the change is taken once every db_server agrees. - Promote then refuses a staged snapshot fetched before the one `latest` names, - so a re-run or a second writer cannot roll consumers back. -- **A package tag is written once.** A tag holding other bytes under a name is - never overwritten: the promoted database or an older snapshot may name those - bytes, and pacman refuses a file whose sha256 it does not expect, with no - other Server tried. A run that meets one holds the target and opens - `replaced-` for a person. A signature missing beside the right package - bytes is repaired by pushing again with the same blob. -- **A package never lacks its signature when the database has one.** The layer - is the database's `%PGPSIG%`, decoded; with no `%PGPSIG%` the detached `.sig` - is downloaded, and a file is pushed unsigned only when every server answers - 404 for it, as upstream ships it. -- **The `.sig` a consumer stores is the one pacman would check.** pacman checks - against `%PGPSIG%` and stores the served `.sig` unread, so verify compares - every stored `.sig` with the decoded `%PGPSIG%`; a mismatch fails the batch - and names the file. Without it, a wrong layer would pass verify and break a - later `pacman -U` from the cache. -- **Every field read from a database is checked first.** A filename with `/`, a - space, a leading `-` or `.`, a control character or a byte above 0x7e is - refused with the whole database. Names makepkg allows that the tag mapping - cannot carry go to the `fn-` tag. -- **Absent means the registry said so.** Only 404 with `NAME_UNKNOWN` or - `MANIFEST_UNKNOWN` is absent; anything else fails the target for that run. -- **A file upstream superseded during a run** holds the target for that run; - the next run plans from the newer database. No archive server is consulted. -- **A file that never succeeds** holds its target until upstream drops or - rebuilds it; the watchdog's `stale-` issue is what surfaces it, and - each run's summary names the file and its error. -- **No exit is silent.** Every script that sources `scripts/lib.sh` has an ERR - trap that names the command which ended it under `set -e`. It found the - first intermittent fault on its first run: `oras version | awk '... exit'` - in `scripts/install-oras`, where awk closed the pipe while oras still wrote, - the pipeline failed with SIGPIPE (141), and a promote ended with no message. - No pipeline in a script ends in a reader that stops early - (`tests/static/35-pipeline-traps.sh`). - -## Jobs - -`sync.yml`, every three hours and on dispatch: - -| job | permissions | timeout | does | -| --- | --- | --- | --- | -| plan | contents, packages read | 15 | `scripts/plan`: candidates, floor, skip or todo, shards | -| mirror | contents, packages write | 120 | `scripts/mirror-shard` per shard: `mirror-file` over the lines, several at a time | -| stage | contents, packages write | 20 | `scripts/stage`: every new file accounted for, the database under staging, the anonymous read, verify batches | -| verify | contents | 90 | `scripts/verify-batch` per batch: pacman in the target's image | -| promote | contents, packages write | 15 | `scripts/promote` per staged target, on main only | -| report | contents, issues write | 10 | `scripts/report`: the summary, `visibility-` and `replaced-` issues | -| keepalive | actions write | 10 | `keep-schedules-enabled`, `check-schedules-fired` | - -`max_parallel` (default 8) caps both matrices: the organisation's 20 concurrent -jobs are shared with every other repository, docker-archlinux's builds -included. Shards are at most 512 MiB and 400 files, verify batches 2 GiB and -400 files, both packed largest first onto the lightest. - -`watchdog.yml`, daily: `scripts/check-staleness` for every target, through the -wrapper's token flow (`PACMAN_OCI_NO_DUMMY=1`), so that path runs against GHCR -on schedule too, and the two schedule checks. A target is stale when upstream's -database differs from the one `latest` serves and that snapshot was fetched -longer ago than the tolerance `scripts/cron-tolerance` derives from sync.yml's -cron. That tolerance is in days: a three-hourly cron gets three days. ⚠ That is -coarse, and accepted: it is a measured routine reused, and a mirror that stops -is still seen within the week. - -### Disk - -The work goes to the largest writable filesystem among `$RUNNER_TEMP`, `/mnt` -and `/tmp`, counted once each. Each file is deleted once pushed, so a shard -needs room only for its `PARALLEL` largest files; `scripts/debloat-runner` runs -only when that does not fit, and `ci.yml` runs it on every push so it is seen to -work. If the room is still short, `PARALLEL` comes down. - -## Running outside GitHub Actions - -Every script reads only `MIRROR_REGISTRY`, `MIRROR_NAMESPACE`, `REGISTRY_USER`, -`REGISTRY_TOKEN`, `TARGETS_DIR` and its arguments, so a source runners cannot -read can be ingested from another host. - -- Every GHCR repository is first created by `sync.yml` with the workflow token, - which links it to this repository. A package first pushed from outside is not - linked, and the workflow token can then never write to it. -- An outside host never uses a maintainer's token: GHCR accepts only classic - tokens, which can write every package their owner can. It uses a dedicated - account granted write on these packages alone. -- Every script logs in through `--password-stdin` into a registry - configuration inside its own temporary directory, removed on exit. -- A push must keep above GHCR's 10 minute upload limit: the largest x86_64 - extra file needs 5.5 MB/s. - -## The first runs (2026-10-13, branch wip) - -| what | measured | -| --- | --- | -| first sync, x86_64 core | plan 25 s; 4 shards of 74 to 75 files each, 53 s to 3 min 10 s; stage 35 s; verify 6.5 min | -| pushes | 299 of 299 read back right on the first try: GHCR read-after-push was consistent | -| upstream spread | 75, 75, 75 and 74 files from the four servers: the per-shard rotation works | -| push rate | up to 34 MB/s for a single file | -| verify | pacman, as `alpm`, downloaded 1888 MiB through the wrapper from GHCR, then checked the keyring and every package's integrity | -| visibility | the package came out public: the pkgforge-dev organisation creates packages public | -| second sync, nothing new | every file `present`, nothing pushed; shards 12 to 17 s | -| consumer, residential network | stock Arch config, `DownloadUser = alpm`, the mirror as core's only Server: `pacman -Sy` 6.0 s (2 s of it pacman's pause after the absent `core.db.sig`), `pacman -S nano` 4.3 s | -| the token path | `PACMAN_OCI_NO_DUMMY=1` against GHCR: 401, anonymous token, 200 | -| debloat on a runner | 12 of 12 directories removed in 45 s, 85 GiB free to 120 GiB | -| first promotion, from `main` (2026-10-14) | run 38490975315: `latest`, `2026.10.14`, `db-` and `files-` name one snapshot of 299 packages; a consumer's synced `core.db` hashes to its `db-` tag; `-Sy` 4.0 s, `-S nano` 5.1 s | - -## The code review - -The implementation was reviewed on 2026-10-13 by four independent reviewers -(the wrapper, the pipeline, the workflows, the tests), each proving a defect in -a container rather than arguing it, and a judge ruled on 26 findings: 10 -confirmed, 13 refuted because the fix had already landed, 3 not worth their -cost. Every confirmed one is fixed and tested, among them: a manifest far -larger than any real one could keep pacman waiting in the parser (now capped at -64 KiB, given up at once); a TMPDIR the download user cannot write broke every -download (now falls back to pacman's own directory); an outage cost up to 60 s -a request (now 15 s a try); a lagging server that only dropped a package could -move `latest` back (the rule above). - -Running every fault again afterwards found one gap the reviewers had not: no -fixture database carried `%PGPSIG%`, because pacman 7's `repo-add` embeds -signatures only with `--include-sigs`, so the suite never drove the path every -real core package takes. The fixture embeds them now, and the suite asserts it. -Driving that path then showed verify passing a wrong signature layer, which -pacman stores without reading; verify now compares it -(`HISTORY/tests-seen-to-fail.md`, "What the faults found"). - -## The design review - -The draft was reviewed on 2026-10-13 by five independent reviewers (pacman, -registry, CI, failure modes, security), each measuring rather than guessing, -and a judge ruled on all 50 findings: 3 accepted, 45 accepted with changes, 2 -rejected. The rules above are the result. The two rejected: a separate runner -per platform (no non-amd64 target yet), and a consumer-side age warning in the -wrapper (an idle upstream would trip it; the watchdog measures staleness at the -source instead). - -## Open, to be measured - -- Whether GHCR read-after-push stays consistent at seed scale: 299 of 299 read - back on the first try for core; count read-back retries in the mirror - results for extra. -- GHCR request volume at seed scale (about 15k manifest reads and pushes for - x86_64 extra), 429s, token requests. -- Wrapper latency per package against pacman's own downloader. -- Matrix queue time under the organisation's job limit, at `max_parallel` 8. -- Runner disk at shard start, and whether debloat ever runs in production. -- Docker Hub pulls of the verify image from runners. -- `DownloadUser` with every sandbox feature disabled, in a container, before - the README's warning is taken as measured (it is read from source). -- The signed-database race, once a signed target exists. diff --git a/HISTORY/tests-seen-to-fail.md b/HISTORY/tests-seen-to-fail.md deleted file mode 100644 index afc456b..0000000 --- a/HISTORY/tests-seen-to-fail.md +++ /dev/null @@ -1,103 +0,0 @@ -# Tests seen to fail - -⛔ A test that has never failed has not been shown to test anything. Each fault -below was injected into a fresh copy of the repository on 2026-10-13 or -2026-10-14, the test that must catch it was run, and it failed with the -assertion named. The tree itself was never changed. Every integration fault was -run again on 2026-10-14, after the fixture change below, and each was caught -again by the assertion named. - -How each was run: copy the repository, replace one exact piece of text in one -file, run the one test file with `REPO_ROOT` set to the copy. Integration -faults ran with docker or podman and their own registry and upstream ports. - -## client/pacman-oci - -| fault | test | failed with | -| --- | --- | --- | -| a literal `_` before `p c a u` no longer escaped | unit/10-tag-encoding | `tag of a_pb-1-1-any.pkg.tar.zst` | -| upper case kept in tags, which GHCR folds together | unit/10-tag-encoding | `a name outside the mapping hashes: lshw-B.02.20-3-x86_64.pkg.tar.zst` | -| the 128 character tag limit off by one | unit/10-tag-encoding | `a name of 129 characters is hashed` | -| an escaped newline or tab accepted in a JSON string | unit/20-json-parser | `refused: an escaped newline, which could forge an output line` | -| data after the JSON document ignored | unit/20-json-parser | `refused: trailing data` | -| an uncompressed `.pkg.tar` read as a database | unit/30-wrapper-urls | `an uncompressed package` | -| a database signature read from the moving tag | unit/30-wrapper-urls | `core.db.sig is read by the content of the core.db beside it` | -| a failed attempt leaves its `.part` behind | unit/30-wrapper-urls | `a failed attempt removes the .part it was given` | -| a blob's sha256 no longer checked | integration/10-pipeline | `the wrapper refuses a blob whose sha256 is not its descriptor's` | - -## scripts - -| fault | test | failed with | -| --- | --- | --- | -| a tag holding other bytes is pushed over (decide_push) | unit/40-lib | `write-once: other bytes under this name are never overwritten` | -| mirror-file pushes over a tag holding other bytes | integration/10-pipeline | `a file whose bytes changed under its own name holds the target` | -| older and newer swapped in the behind rule | unit/40-lib | `core of 10-03 is not behind 10-02` | -| plan takes a database behind the promoted one | integration/10-pipeline | `an older upstream database is refused as behind` | -| a `/` allowed in a filename from a database | unit/40-lib | `db field check refuses a / alone, after a valid prefix` | -| staleness waits ten times too long | unit/40-lib | `stale: different, fetched longer ago than the tolerance` | -| an earlier snapshot may be promoted over a later one | unit/40-lib | `a later snapshot may be promoted` | -| promote no longer refuses an older snapshot | integration/10-pipeline | `promoting an older run's snapshot again is refused` | -| an item that left no result counts as pushed | unit/40-lib | `an item that left no result is counted as failed` | -| an unknown key in a target file ignored | unit/40-lib | `refused: an unknown key` | -| verify downloads and checks nothing | integration/10-pipeline | `verify fails it, and names the package` | -| plan lets a lagging db_server that only dropped a package win | integration/10-pipeline | `a lagging db_server that only drops a package does not move latest` | -| with no `%PGPSIG%`, the detached `.sig` is never fetched | integration/10-pipeline | `a database without %PGPSIG% stages` | -| the signature layer is not the database's `%PGPSIG%` (the decode loses its last base64 group) | integration/10-pipeline | `alpha-1:1.0+r2-1-any.pkg.tar.zst signature layer is its %PGPSIG%, decoded` | - -## What the faults found - -The fault for the detached `.sig` path was first caught by the first sync, not -by the scenario written for it: no fixture database carried `%PGPSIG%` at all. -pacman 7's `repo-add` embeds signatures only when given `--include-sigs`, so -the suite had driven only the detached path, while every package of Arch's core -carries `%PGPSIG%` (299 of 299, measured 2026-10-14). The fixture now passes -`--include-sigs`, and the suite asserts that its database carries a `%PGPSIG%` -for every package before it relies on it. - -It found a second gap. With the decode broken, verify still passed: pacman 7.1 -checks a package against the database's `%PGPSIG%`, yet still asks the wrapper -for its `.sig` and stores it in the cache unread (measured against the -promoted core, 2026-10-14), where a later `pacman -U` from the cache reads it. -Verify now compares every stored `.sig` with the decoded `%PGPSIG%`, and the -decode fault is refused there, before promotion. - -## Policy - -| fault | test | failed with | -| --- | --- | --- | -| an unquoted expansion in scripts/plan | static/10-shellcheck | `shellcheck -s bash scripts/plan` | -| a job reads an output that does not exist | static/20-actionlint | `actionlint .github/workflows/sync.yml` | -| a failure swallowed with `\|\| true` | static/30-failures-stay-visible | `scripts/stage hides no failure` | -| a curl call with no deadline | static/30-failures-stay-visible | `scripts/install-oras: every curl has a connect deadline and a transfer or stall deadline` | -| an oras call with no deadline | static/30-failures-stay-visible | `scripts/promote: every oras call has a deadline` | -| a script committed without its executable bit | static/40-executable-bits | `scripts/plan is mode 100755 in the index` | -| a job without timeout-minutes | static/50-workflow-policy | `.github/workflows/watchdog.yml: every job has timeout-minutes` | -| a dispatch input expanded inside a `run:` block | static/50-workflow-policy | `.github/workflows/sync.yml: no untrusted value is expanded inside a run: block` | -| an action pinned by tag | static/50-workflow-policy | `.github/workflows/ci.yml: every action is pinned by commit` | -| a call to the packages DELETE API | static/50-workflow-policy | `no script or workflow deletes a package or a version` | -| a pipeline that ends in a reader that stops early | static/35-pipeline-traps | `scripts/install-oras: no pipeline ends in a reader that stops early` | -| a verify image pinned without a tag | static/60-targets | `targets/x86_64-core.conf pins its verify image as name:tag@sha256` | - -## Upstream faults the integration suite drives on every run - -These are not code faults but hostile or broken inputs, built by -`tests/fixtures/build-repo` or served by a plain web server standing in for a -registry that lies. The suite fails if the mirror accepts any of them. - -| input | must end as | -| --- | --- | -| an upstream database older than the promoted one | `behind`, latest unmoved | -| a file whose bytes change under its own name | `replaced`, the tag unmoved | -| a new file whose tag already holds other bytes | `replaced`, the tag unmoved | -| two db_servers each ahead of the other on one package | `held` | -| a db_server, listed first, that lags by one dropped package | `current`, latest unmoved | -| a database with a `%PGPSIG%` for every package, as core's | each signature layer is that `%PGPSIG%`, decoded | -| a database with no `%PGPSIG%` | each new package's signature layer is upstream's detached `.sig`, and pacman verifies against it | -| a package signed by a key nobody trusts | verify `failed` naming it, never promoted | -| an older run's stage promoted again | refused, latest unmoved | -| a manifest whose bytes are not the digest the URL pins | refused by the wrapper | -| a manifest with two layers of one name | refused by the wrapper | -| a package manifest with no signature layer | refused by the wrapper, loudly | -| a blob whose size, or sha256, is not its descriptor's | refused by the wrapper | -| a manifest over 64 KiB | given up within 20 s | -| a database signature whose snapshot moved on | not paired, the wrapper misses | diff --git a/README.md b/README.md index c4a9604..8cdfe0a 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,50 @@ # mirror-archlinux -Arch Linux repositories stored on GHCR, and read by pacman through one small -shell script. - -GitHub's runners are refused by a growing number of upstream mirrors: ArchPOWER -answers every datacentre address with a bot challenge, and docker-archlinux -held its PowerPC images back for weeks because of it. GHCR does not refuse -GitHub's own runners. This repository copies each repository's database, -packages and signatures to `ghcr.io/pkgforge-dev/mirror-archlinux//`, -checks every copy with pacman itself before it is published, and ships -`client/pacman-oci`, the pacman `XferCommand` that reads it back. - -| mirrored | from | state | -| --- | --- | --- | -| `x86_64/core` | Arch Linux | every three hours | +Arch Linux and three of its ports, stored on GHCR and checked by pacman before each snapshot is published. + +[![all](https://pkgforge-dev.github.io/mirror-archlinux/badges/all.svg)](https://pkgforge-dev.github.io/mirror-archlinux/) +[![x86_64](https://pkgforge-dev.github.io/mirror-archlinux/badges/x86_64.svg)](https://pkgforge-dev.github.io/mirror-archlinux/) +[![aarch64](https://pkgforge-dev.github.io/mirror-archlinux/badges/aarch64.svg)](https://pkgforge-dev.github.io/mirror-archlinux/) +[![armv7h](https://pkgforge-dev.github.io/mirror-archlinux/badges/armv7h.svg)](https://pkgforge-dev.github.io/mirror-archlinux/) +[![loong64](https://pkgforge-dev.github.io/mirror-archlinux/badges/loong64.svg)](https://pkgforge-dev.github.io/mirror-archlinux/) +[![riscv64](https://pkgforge-dev.github.io/mirror-archlinux/badges/riscv64.svg)](https://pkgforge-dev.github.io/mirror-archlinux/) +[![speed](https://pkgforge-dev.github.io/mirror-archlinux/badges/speed.svg)](https://pkgforge-dev.github.io/mirror-archlinux/) + +![Share of upstream files in the mirror, per target](https://pkgforge-dev.github.io/mirror-archlinux/charts/coverage.svg) + +![Snapshot age per target, against the sync interval](https://pkgforge-dev.github.io/mirror-archlinux/charts/freshness.svg) + +The dashboard is at . It shows every target, the history of the last 30 days and the daily speed measurement. + +## Use the mirror + +There are three ways to read the mirror. Each way reads the same snapshots. Pick one per machine. + +### 1. The Worker, with pacman's own downloader + +Add the Worker as the first Server of each repository in `/etc/pacman.conf`: + +```ini +[core] +Server = https://ghcr.mirror.archlinux.pkgforge.dev/$arch/$repo +Include = /etc/pacman.d/mirrorlist +``` -## Using it +The Worker sends no package bytes. It answers a request for a file with a redirect to GitHub's blob host. pacman uses its own downloader, so `ParallelDownloads` and resume work as for any mirror. + +### 2. The registry, with the wrapper + +Install the wrapper: ```bash -curl -fsSL -o /usr/local/bin/pacman-oci https://raw.githubusercontent.com/pkgforge-dev/mirror-archlinux/main/client/pacman-oci +curl -fsSL --connect-timeout 10 --max-time 120 -o /usr/local/bin/pacman-oci \ + https://raw.githubusercontent.com/pkgforge-dev/mirror-archlinux/main/client/pacman-oci chmod 0755 /usr/local/bin/pacman-oci ``` -`/etc/pacman.conf`: +⚠ An `XferCommand` replaces pacman's own downloader for every Server of every repository. The wrapper then fetches every URL that is not `oci://` with curl. + +Set the wrapper as the `XferCommand`, and add the registry as the first Server: ```ini [options] @@ -33,147 +55,212 @@ Server = oci://ghcr.io/pkgforge-dev/mirror-archlinux/$arch/$repo Include = /etc/pacman.d/mirrorlist ``` -The `oci://` Server is read from the registry. Every other URL is handed to -curl, so the ordinary mirrors after it stay the fallback: when the registry -does not have a file, pacman moves on to the next Server, as it does for any -mirror. +The wrapper reads the registry directly. It asks pacman for the whole download list once per run and fetches it in parallel. [docs/client.md](docs/client.md) tells how. -⚠ Pin by digest before you use the mirror for a build anyone must reproduce, so -it cannot move under you: +### 3. Both, in fallback order ```ini -Server = oci://ghcr.io/pkgforge-dev/mirror-archlinux/$arch/$repo@sha256: +[options] +XferCommand = /usr/local/bin/pacman-oci %o %u + +[core] +Server = oci://ghcr.io/pkgforge-dev/mirror-archlinux/$arch/$repo +Server = https://ghcr.mirror.archlinux.pkgforge.dev/$arch/$repo +Include = /etc/pacman.d/mirrorlist ``` -| Server form | gives | -| --- | --- | -| `oci://.../$arch/$repo` | `latest`, which follows upstream | -| `oci://.../$arch/$repo:2026.10.13` | the first snapshot promoted that UTC day; it never moves once it exists | -| `oci://.../$arch/$repo@sha256:HEX` | exactly one snapshot. ⭐ The only real pin: the wrapper hashes the manifest it is given and refuses one that is not HEX | +pacman tries each Server in turn when a file is not found. Keep `oci://` first: the prefetch reads only the first Server of each repository. The Worker reads the same GHCR repository, so it is a second path to the same snapshot, not a second copy. -Tags are labels; `@sha256:` is the pin. The digest a tag names now: +### An absent database signature -```bash -curl -sI -H 'Authorization: Bearer QQ==' -H 'Accept: application/vnd.oci.image.manifest.v1+json' \ - https://ghcr.io/v2/pkgforge-dev/mirror-archlinux/x86_64/core/manifests/latest | grep -i docker-content-digest -``` +⚠ Under an `XferCommand`, pacman pauses 2 s after each failed download, also for an optional `.db.sig`. No upstream of this mirror publishes a database signature. -### What it costs - -Measured with pacman 7.1.0: - -- With an `XferCommand`, pacman downloads one file at a time; `ParallelDownloads` - is ignored. -- pacman waits 2 s after every failed attempt. Arch publishes no - `.db.sig`, so each repository costs about 2 s on every `-Sy` and every - `-Fy`, from the mirror and from an ordinary mirror alike, and each file that - falls through to the next Server costs 2 s more. -- During a GHCR outage, each manifest request gives up within about 45 s (15 s - a try, no new try after 30 s), and a file then falls through to the next - Server. -- `latest` can move between pacman's request for `.db` and its request for - `.db.sig`. The wrapper pairs a database signature with the very database - pacman just took (by its content), so this needs a snapshot that disappears - between the two requests; a consumer that needs the pair atomic pins the - digest. - -⚠ With `DownloadUser` set, keep at least one sandbox feature enabled: with -`DownloadUser` and every `DisableSandbox*` option set together, pacman downloads -into root-owned directories while still running the `XferCommand` as the -download user, and every download fails. - -### What the wrapper checks - -Everything it reads is checked before pacman sees it, and pacman then checks -what it always checks. - -1. A manifest asked for by digest must hash to that digest. -2. Exactly one layer must carry the name pacman asked for. -3. The blob's size and sha256 must equal that layer's descriptor. -4. pacman checks the file's sha256 against the database and its signature - against the distribution's keyring. - -A package's signature is always in the same manifest as the package: pacman -asks for a `.sig` from the server that served the file and from no other, so a -missing signature would end the install rather than fall through. - -The wrapper is one POSIX sh file. It needs curl, awk, sed, sha256sum and -mktemp, and the test suite runs it under bash with gawk, dash with mawk, and -busybox. +With `DownloadUser` set (stock Arch) and pacman 7.0 or 7.1, the wrapper answers an absent database signature with an empty file. pacman removes that file and reads the signature as absent, with no pause. `PACMAN_OCI_EMPTY_SIG=0` turns this answer off. -## How the mirror works +With no `DownloadUser`, or with another pacman version, add this line to each `oci://` repository section: +```ini +SigLevel = Required DatabaseNever ``` -plan ──> mirror (shards) ──> stage ──> verify (batches) ──> promote ──> report -``` -1. **plan** reads the database from the distribution's own servers only, and - refuses one that is malformed, or behind the promoted database or another - candidate (some package built before the same package in the other). Files - the promoted database already names with the same sha256 cost nothing; the - rest are split into shards. -2. **mirror** downloads each file from any listed server, checks its size and - sha256 against the database, and pushes it with its signature under its own - tag. ⛔ A tag is written once: a tag already holding other bytes is never - overwritten, and holds the target. -3. **stage** pushes the database under a staging tag once every new file is on - the registry, and checks that it reads with no credential. -4. **verify** runs pacman in the target's own Arch image, with the wrapper as its - `XferCommand` and the staged digest as its only Server, and downloads every - new file: pacman checks every sha256 and signature as a consumer would. - Each `.sig` pacman stored must also be the database's `%PGPSIG%`, which - pacman itself checks against without reading the stored one. -5. **promote** tags the staged digest: the day's tag (once), the content tags, - then `latest`, never backwards, and reads it back through the wrapper. -6. **report** writes the run's summary, and keeps one issue per problem that - does not heal by itself: a package that is private, or a tag that would - have to change. `watchdog.yml` raises one when a target stops following - upstream. - -Targets are independent. One that is held never holds another, and a held -target keeps serving its last promoted snapshot. - -The design, every decision in it and the measurements behind them: -[`HISTORY/design.md`](HISTORY/design.md). - -## Adding a target - -A file `targets/-.conf`, where `` is pacman's `$arch`: +pacman then asks for no database signature and still checks every package signature. `SigLevel` applies to every Server of the section, the mirrorlist too. + +⚠ With `DownloadUser` set, do not turn off both parts of pacman's sandbox (`DisableSandbox`, or `DisableSandboxFilesystem` with `DisableSandboxSyscalls`). pacman then runs the `XferCommand` as the download user in a root-owned directory, and every download fails. + +### Pin a snapshot + +⚠ Pin by digest when a build must give the same result later. `latest` moves with upstream. The wrapper and the Worker check the hash of a digest. Nothing checks what a tag names. + +| Server form | Snapshot | `oci://` (wrapper) | `https://` (Worker) | +| --- | --- | --- | --- | +| `.../$arch/$repo` | `latest`: follows upstream | yes | yes | +| `.../$arch/$repo:latest` | the same | yes | yes | +| `.../$arch/$repo:YYYY.MM.DD` | the first promoted snapshot that was fetched on that UTC date; it never moves | yes | yes | +| `.../$arch/$repo:db-HEX` | the snapshot whose `.db` has sha256 HEX | yes | yes | +| `.../$arch/$repo:files-HEX` | the snapshot whose `.files` has sha256 HEX | yes | yes | +| `.../$arch/$repo@sha256:HEX` | one manifest. ⭐ The wrapper and the Worker hash it and refuse a manifest that is not HEX | yes | yes | +| `.../$arch/$repo:staging` and any other tag | a snapshot that is not verified yet, or not a snapshot | yes | no: 404 | + +A digest is the digest of one repository's manifest. Write a pinned Server line in each repository section, with that repository's digest: ```ini -arch = aarch64 -repo = core -# The distribution's own servers: the database comes only from these. -db_server = https://example.org/$arch/$repo -# More servers package bytes may come from. Every byte is checked against the -# database before it is pushed, so any mirror will do. -server = https://mirror.example.org/$arch/$repo -# tag: base -verify_image = docker.io/library/archlinux:base@sha256: -verify_platform = linux/amd64 -keyring = archlinux-keyring +[core] +Server = oci://ghcr.io/pkgforge-dev/mirror-archlinux/$arch/$repo@sha256: ``` -The first push creates the GHCR package. If it comes out private, the run says -so and opens an issue with the settings link: GitHub has no API for package -visibility, so making it public is one click. +Read the digest that `latest` names now: + +```bash +curl -sI --connect-timeout 10 --max-time 30 \ + -H 'Authorization: Bearer QQ==' -H 'Accept: application/vnd.oci.image.manifest.v1+json' \ + https://ghcr.io/v2/pkgforge-dev/mirror-archlinux/x86_64/core/manifests/latest | grep -i docker-content-digest +``` + +### What each way does and costs + +| | 1. Worker | 2. Wrapper | +| --- | --- | --- | +| Downloader | pacman's own | `client/pacman-oci`, one call per file | +| Parallel downloads | `ParallelDownloads` | one package prefetch and one database prefetch per run: up to 32 transfers at once (`PACMAN_OCI_JOBS`), then one rename per file | +| Package signature | the registry's `.sig` layer, through a redirect | decoded from the database's `%PGPSIG%` with no request, else the registry's layer | +| Resume | pacman's own | `--continue-at -`, then a sha256 check | +| A host that fails | pacman's server error limit | skipped for the rest of the run, never for a `.sig` | +| Absent `.db.sig` | a 404 with no pause | an empty file under the sandbox (pacman 7.0, 7.1), else a 2 s pause | +| Sandbox, with `DownloadUser` | pacman's own (Landlock, seccomp) | uid switch and `NO_NEW_PRIVS` only | +| Programs on the host | none | `curl`, `awk`, `sed`, `grep`, `sha256sum`, `mktemp`, `base64` | +| Reads from | the Worker, then GitHub's blob host | GHCR, then GitHub's blob host | +| Trust root | GitHub, this repository, the Worker and its Cloudflare account | GitHub and this repository | + +The measured times of each way are in [docs/measurements.md](docs/measurements.md). The Worker is described in [docs/worker.md](docs/worker.md). + +## Targets + +A target is one repository of one architecture. The table comes from `targets/*.conf`. The Server path goes after `oci://ghcr.io/pkgforge-dev/mirror-archlinux/` or after `https://ghcr.mirror.archlinux.pkgforge.dev/`. + +| arch | repo | upstream distribution | keyring packages | Server path | +| --- | --- | --- | --- | --- | +| x86_64 | core | Arch Linux | archlinux-keyring | `x86_64/core` | +| x86_64 | extra | Arch Linux | archlinux-keyring | `x86_64/extra` | +| x86_64 | multilib | Arch Linux | archlinux-keyring | `x86_64/multilib` | +| aarch64 | core | Arch Linux ARM | archlinuxarm-keyring | `aarch64/core` | +| aarch64 | extra | Arch Linux ARM | archlinuxarm-keyring | `aarch64/extra` | +| aarch64 | alarm | Arch Linux ARM | archlinuxarm-keyring | `aarch64/alarm` | +| aarch64 | aur | Arch Linux ARM | archlinuxarm-keyring | `aarch64/aur` | +| armv7h | core | Arch Linux ARM | archlinuxarm-keyring | `armv7h/core` | +| armv7h | extra | Arch Linux ARM | archlinuxarm-keyring | `armv7h/extra` | +| armv7h | alarm | Arch Linux ARM | archlinuxarm-keyring | `armv7h/alarm` | +| armv7h | aur | Arch Linux ARM | archlinuxarm-keyring | `armv7h/aur` | +| loong64 | core | Arch Linux for Loong64 (LCPU) | archlinux-keyring, archlinux-lcpu-keyring | `loong64/core` | +| loong64 | extra | Arch Linux for Loong64 (LCPU) | archlinux-keyring, archlinux-lcpu-keyring | `loong64/extra` | +| riscv64 | core | Arch Linux RISC-V | archlinux-keyring | `riscv64/core` | +| riscv64 | extra | Arch Linux RISC-V | archlinux-keyring | `riscv64/extra` | + +pacman checks each package against the keyring of its distribution. A consumer needs that keyring in its own pacman keyring. The badges show which targets have a promoted snapshot. + +## How the mirror works + +`.github/workflows/sync.yml` runs every 3 hours. Each target with work runs its own chain (`.github/workflows/sync-target.yml`), so one target never holds another. + +1. **plan** (`scripts/plan`) reads each database from the distribution's own servers only. It refuses a database that is malformed or behind the promoted one, and puts the new files into shards per target. +2. **mirror** (`scripts/mirror-shard`, `scripts/mirror-file`) downloads each new file from any listed server. It checks size and sha256, then pushes the file with its signature under a tag of its own. +3. **stage** (`scripts/stage`) pushes the database under a staging tag once every file it names is on the registry. It reads the database back and splits the new files into verify batches. +4. **verify** (`scripts/verify-batch`) runs pacman in the x86_64 Arch image with the wrapper as its `XferCommand`. pacman downloads every new file and checks its sha256 and signature. +5. **promote** (`scripts/promote`, branch `main` only) tags the staged digest: the date tag once, `db-HEX`, `files-HEX`, then `latest`. `latest` never moves backwards. +6. **report** (`scripts/report`) writes one table for the run. On `main`, it keeps the `visibility-` and `replaced-` issues current, then starts the status workflow. +7. **status** (`scripts/status-collect`, `scripts/status-render`, every hour) measures each target against upstream and against the Worker. It publishes the dashboard, the badges and the charts. + +[docs/design.md](docs/design.md) gives each rule and the failure it prevents. + +## Trust and checks + +⛔ A database reaches `latest` only after every file it names is on the registry. Every file new since the promoted database must also pass pacman's own checks through the wrapper. + +- The database comes only from the `db_server` hosts of the target, which the distribution runs. These databases are unsigned. The source of a database chooses which validly signed packages get installed. +- The wrapper checks each file it reads from the registry before pacman sees it: + 1. A manifest asked for by digest must hash to that digest. + 2. Exactly one layer must carry the name pacman asked for. + 3. The blob's size and sha256 must equal that layer's descriptor, or the database's `%SHA256SUM%`. +- pacman then checks each package's sha256 against the database, and each signature against the keyring. +- Push access to this repository is the trust root of the registry: any workflow on any branch can ask for `packages: write`. Every workflow starts from `permissions: {}`. Only `main` promotes. Actions are pinned by commit. +- The Worker and its Cloudflare account join the trust root for the consumers of ways 1 and 3. Whoever can change the Worker can change the database a consumer gets. +- Renovate pushes its own branches, so it holds the same trust. Its small updates merge when CI is green. ⚠ The keyring images of the targets and every major update wait for a person. + +## Add a target + +1. Write `targets/-.conf`, where `` is pacman's `$arch`. Use `key = value` lines and `#` comments. +2. Run `tests/run.sh static 60-targets`. +3. Make sure `sync.yml` has a plan upload slot for each enabled target. There are 16 slots. +4. Start a run for the target: `gh workflow run sync.yml -f targets=-`. + +| key | count | grammar | meaning | +| --- | --- | --- | --- | +| `arch` | 1, required | `[a-z0-9]+(_[a-z0-9]+)*` | pacman's `$arch`; the file name starts with it | +| `repo` | 1, required | `[a-z0-9]+([.-][a-z0-9]+)*` | the repository; the file name ends with it | +| `db_server` | 1 or more, required | an `https://` URL; `$arch` and `$repo` are expanded | the distribution's own servers, tried in order; the database comes only from these | +| `server` | 0 or more | an `https://` URL; `$arch` and `$repo` are expanded | more sources of package bytes, after the db_servers | +| `verify_image` | 1, required | `name:tag@sha256:<64 hex>` under a `# tag: ` line | the image pacman verifies in: the one x86_64 Arch image of every target | +| `verify_platform` | 1, required | `linux/amd64` | the platform of `verify_image` | +| `keyring` | 0 or 1 | package names, space separated, each `[a-z0-9@_+][a-z0-9@._+-]*` | keyring packages that verify installs before it checks anything | +| `keyring_from` | 0 or 1 | `-`: an enabled target of the same arch, not this one | the target whose `latest` holds the keyring packages; needs `keyring` | +| `keyring_image` | 0 or 1 | `ghcr.io/pkgforge-dev/archlinux:tag@sha256:<64 hex>` under a `# tag: ` line | an image whose `/usr/share/pacman/keyrings/*` adds to the trust root of verify | +| `keyring_platform` | 1 with `keyring_image` | `linux/` or `linux//v` | the platform to pull `keyring_image` for | +| `enabled` | 0 or 1 | `yes` or `no`; default `yes` | `no` leaves the target out of the scheduled sync, the status page and the watchdog | +| `db_signed` | 0 or 1 | `yes` or `no`; default `no` | `yes`: plan requires `.db.sig` and `.files.sig`, and verify uses `DatabaseRequired` | + +An unknown key is an error. A key of count 0 or 1 may not repeat. `scripts/lib.sh` `target_check` and `tests/static/60-targets.sh` apply these rules. ⚠ Neither one checks the value of `enabled`. A value other than `yes` or `no` stops `plan`, `status-collect` and `check-staleness` when they read it. + +The first push creates the GHCR package. If the package is private, the run opens `visibility-` with the settings link. GitHub has no API to change the visibility, so a person makes it public. ## Tests ```bash -scripts/install-test-tools ./bin -PATH="$PWD/bin:$PATH" tests/run.sh all +scripts/install-test-tools /tmp/mirror-archlinux-tools +PATH="/tmp/mirror-archlinux-tools:$PATH" tests/run.sh static +tests/run.sh unit +CONTAINER_RUNTIME=podman tests/run.sh integration +tests/run.sh worker +WORKER_LIVE=1 tests/run.sh worker live ``` -`static` lints and checks policy, `unit` needs no network, and `integration` -runs the whole mirror against a local registry with docker or podman. Every -test has been seen to fail: [`HISTORY/tests-seen-to-fail.md`](HISTORY/tests-seen-to-fail.md). +`tests/run.sh all` runs the four suites in turn. [docs/tests.md](docs/tests.md) tells what each suite needs and lists the tests seen to fail. + +## Maintenance + +These run by themselves: + +- Sync every 3 hours. Status, with the Worker check, every hour. The staleness and schedule checks every day. +- The scheduled workflows start each other when a newest run is overdue, and keep every schedule enabled. +- Renovate merges digest, pin, patch and minor updates when every check is green. +- `worker.yml` deploys the Worker after a push to `main` that changes `worker/`, when the `CLOUDFLARE_API_TOKEN` secret is set. -## Trust +These issues open by themselves. Each one opens once, keeps its body current, and closes when the problem clears: -Push access to this repository is the mirror's trust root: any workflow on any -branch can ask for `packages: write`. Every workflow starts from -`permissions: {}`, only `main` promotes, and actions are pinned by commit, which -the repository requires. ⚠ Renovate, when it is enabled for the repository, -pushes its own branches, so it is trusted with that root too; its pull requests -wait for a person, and no other bot is given push access. +| issue | raised when | raised by | +| --- | --- | --- | +| `visibility-` | the package reads only with the workflow's token | `scripts/report` | +| `replaced-` | a package tag holds other bytes than the database names | `scripts/report` | +| `stale-` | `latest` differs from upstream for longer than the tolerance, 3 days for the 3-hour cron | `watchdog.yml` (`scripts/check-staleness`) | +| `worker-down` | the Worker failed its probe in 2 status runs in a row | `status.yml` | + +These need a person: + +- Make a private package public, with the link in its `visibility-` issue. +- Decide each `replaced-` issue: wait for a new upstream filename, or accept the new bytes. +- Review and merge the keyring image updates and the major updates from Renovate. +- Enable Pages once: `gh api -X POST repos/pkgforge-dev/mirror-archlinux/pages -f build_type=workflow`. +- Set the `CLOUDFLARE_API_TOKEN` secret to deploy the Worker. Without it, nothing is deployed. +- Start a workflow by hand when every schedule stops at once. GitHub stops the schedules of a repository after 60 days with no activity. + +[docs/operations.md](docs/operations.md) has the schedules, the issues and the settings in full. + +## Documents + +| document | contents | +| --- | --- | +| [docs/design.md](docs/design.md) | the invariant, the rules, the storage layout, the jobs, verify | +| [docs/client.md](docs/client.md) | the wrapper: run store, prefetch, single-file path, environment, exit codes | +| [docs/worker.md](docs/worker.md) | the Worker: paths, references, caching, cost, deploy, trust | +| [docs/operations.md](docs/operations.md) | workflows, schedules, issues, Renovate, secrets, Pages, the status branch | +| [docs/measurements.md](docs/measurements.md) | measured facts, each with its date and source | +| [docs/tests.md](docs/tests.md) | the suites, how to run them, the tests seen to fail | +| [docs/conventions.md](docs/conventions.md) | the code and text conventions | diff --git a/client/pacman-oci b/client/pacman-oci index 8a8b61e..4d04846 100755 --- a/client/pacman-oci +++ b/client/pacman-oci @@ -53,8 +53,8 @@ # DownloadUser set (stock Arch) it runs as that user, from a download-XXXXXX # directory inside the sync or cache directory; without it, as root. # -# Needs curl, awk, sed, sha256sum, mktemp, base64 (coreutils or busybox). Uses -# pacman, pacman-conf and timeout when they are there. +# Needs curl, awk, sed, grep, sha256sum, mktemp, base64 (coreutils or busybox). +# Uses pacman, pacman-conf and timeout when they are there. # # Environment: # PACMAN_OCI_DEBUG=1 print every request and decision to stderr @@ -66,6 +66,8 @@ # PACMAN_OCI_NO_DUMMY=1 skip the dummy token GHCR accepts for public # packages and use the registry's token flow, with no # prefetch; it exists so that flow runs on schedule +# PACMAN_OCI_EMPTY_SIG=0 no empty answer for an absent database signature +# under the sandbox of pacman 7.0 and 7.1 # PACMAN_OCI_TEST_PROC=DIR for the test suite only: read the parent process # from DIR, not /proc. Used only when DIR holds a # file named pacman-oci-test-proc. @@ -110,6 +112,7 @@ debug() { # ,down.HOST HOST failed once its retries were spent # ,absent.FILE the database manifest of this run has no layer FILE # ,index FILESHA256SIZE of each package the prefetch planned +# ,pacver the version of the parent pacman, asked once per run # ,reaper the background remover's output # tmp.* files being written #---------------------------------------------------------------------------# diff --git a/docs/client.md b/docs/client.md new file mode 100644 index 0000000..8a0d838 --- /dev/null +++ b/docs/client.md @@ -0,0 +1,247 @@ +# The wrapper + +`client/pacman-oci` lets pacman read a repository that is stored on an OCI registry. It is one POSIX sh file, version 2. pacman runs it as its `XferCommand`. + +```ini +[options] +XferCommand = /usr/local/bin/pacman-oci %o %u + +[core] +Server = oci://ghcr.io/pkgforge-dev/mirror-archlinux/$arch/$repo +Include = /etc/pacman.d/mirrorlist +``` + +pacman runs the `XferCommand` for every file it downloads, from each Server in turn. It moves to the next Server when the command exits non-zero. The wrapper reads an `oci://` Server from the registry. It hands every other URL to curl, so the mirrors after it stay the fallback. + +## Calls + +| call | does | +| --- | --- | +| `pacman-oci OUTPUT URL` | the `XferCommand` form: the file at URL into OUTPUT | +| `pacman-oci --tag FILENAME` | prints the tag a package file is stored under | +| `pacman-oci --tags` | the same for the filenames on stdin; prints `FILENAMETAG` | +| `pacman-oci --json FILE` | the JSON parser alone; prints `POINTERVALUE` | +| `pacman-oci --version` | prints `pacman-oci 2` | + +| exit status | means | +| --- | --- | +| 0 | the file is at OUTPUT | +| 1 | not here, or a check failed: pacman tries the next Server | +| 2 | usage | +| 3 | the `--json` input is not JSON | + +## URL forms + +| URL | reads | +| --- | --- | +| `oci://HOST/REPOSITORY/FILE` | the database from tag `latest` | +| `oci://HOST/REPOSITORY:TAG/FILE` | the database from a tag | +| `oci://HOST/REPOSITORY@sha256:HEX/FILE` | the database from one manifest, which must hash to HEX | +| `oci+http://HOST/...` | the same over plain http, for a local test registry only | + +A package file (`*.pkg.tar`, `*.pkg.tar.*`) and its `.sig` are always read from the package's own tag, whichever database the URL names. A database signature is read from the manifest of the database pacman took just before it. + +The wrapper sorts each request into one kind: + +| kind | file | read from | +| --- | --- | --- | +| `pkg` | a package | its own tag, layer `FILE` | +| `pkgsig` | its signature | the package's tag, layer `FILE`; never optional | +| `db` | `.db`, `.files` | the tag, digest or `latest` of the URL | +| `dbsig` | `.db.sig`, `.files.sig` | the manifest of the database beside OUTPUT, found by its content (`db-HEX` or `files-HEX`) | + +## Requirements + +The wrapper needs `curl`, `awk`, `sed`, `grep`, `sha256sum`, `mktemp` and `base64`, and the other usual tools of coreutils or busybox. It uses `pacman`, `pacman-conf` and `timeout` when they are there. The test suite runs it under bash with gawk, dash with mawk, and busybox. + +pacman runs it with fork and execvp, never through a shell. With `DownloadUser` set, it runs as that user, from a `download-XXXXXX` directory in the sync or cache directory. Without `DownloadUser`, it runs as root, in the sync or cache directory. + +## The run store + +A run of pacman makes many calls of the wrapper, one per file. The run store lets one call prepare files for the calls that follow. + +- The key of a run is the parent's pid and its start time: `-`. The start time is field 22 of `/proc//stat`, counted after the last `)`. So a pid that the kernel reuses is another run. +- There is a store only when `/proc//comm` starts with `pacman`. Any other parent gets the single-file path with no store. +- Entries are flat files beside OUTPUT: `.pacman-oci..`. Each one is made under a `mktemp` name in the same directory, then renamed. A call uses an entry only when it is a regular file of this user and not a symbolic link. +- A ready entry `` passed its size and sha256 check before it got its name. A ready `.sig` made from `%PGPSIG%` is the decoded value, with no other check. Serving an entry is one `mv` to OUTPUT, with no copy and no network. +- Each entry is in OUTPUT's own directory. So `,prefetch`, `,dbprefetch` and `,down.` apply to one run in one directory. The sync directory and the cache directory each have their own. + +| entry | means | +| --- | --- | +| `` | a file ready for pacman | +| `,open` | the store of this run is open in this directory | +| `,prefetch` | the package prefetch of this run ran here | +| `,dbprefetch` | the database prefetch of this run ran here | +| `,down.` | the host failed once its retries were spent | +| `,absent.` | the database manifest of this run has no layer `` | +| `,index` | `FILESHA256SIZE` of each package the prefetch planned | +| `,pacver` | the pacman version, asked once per run | +| `,reaper` | the output of the background remover | +| `tmp.*` | files that are being written | + +Three things remove the entries: + +1. pacman unlinks every regular file of its `download-XXXXXX` directory when it removes that directory. +2. The call that opens the store starts a background remover. It waits until the pacman run ends, then removes the run's entries. +3. Until the store of its run is open in a directory, each call removes the entries of runs that are gone there. A run is gone when its pid is missing or its start time differs. + +⚠ Entries stay flat in OUTPUT's directory, never in a subdirectory. pacman cannot remove a `download-XXXXXX` directory that holds a subdirectory, and leaves it behind. The private work directory goes there only when `TMPDIR` is not writable, and the wrapper removes it on exit. + +## Package prefetch + +Under an `XferCommand`, pacman asks for one file at a time. The first package request of a run asks pacman itself for the whole list and fetches it in parallel. + +The prefetch runs once per run when all of these are true: + +- the run has a store +- the requested file name is printable ASCII and does not start with `,` +- `PACMAN_OCI_PREFETCH` is not `0` +- `PACMAN_OCI_JOBS` is more than 1 +- `PACMAN_OCI_NO_DUMMY` is not `1` +- OUTPUT's directory holds no `"`, `\`, tab or newline + +### The print run + +1. The wrapper reads pacman's arguments from `/proc//cmdline`. It refuses, and does no prefetch, when it cannot read them with certainty: + - an argument holds a newline + - an option is not known (getopt also takes abbreviations) + - a path option (`--config`, `--dbpath`, `--root` and others) is relative + - a target is read from stdin (`-`) + - the operation is not a sync (`-S` or `--sync`), or `--print` or `-p` is present +2. It rewrites the arguments. It drops every refresh: `--refresh`, and each `y` of a short group before its first `b` or `r`. It drops `--confirm`, `--noconfirm` and `--print-format`. It keeps everything after `--`. +3. It finds the pacman binary from `/proc//exe`, else from `argv[0]`. The download user cannot read `/proc//exe`. +4. It runs pacman with `--print --print-format '%l%h%s%g' --noconfirm --noprogressbar`, under `timeout 120` when `timeout` exists. `pacman -S --print` takes no lock and needs no root. + +Each line names the first Server URL (`%l`), the sha256 (`%h`), the download size (`%s`) and the base64 `%PGPSIG%` (`%g`). A `file://` line is a cached file and is skipped. A line is work when all of these are true: + +- it starts with `oci://`, `oci+http://`, `https://` or `http://` +- it names a package file, not a `.sig`, and `%s` is a number +- for `https://` or `http://`: `%h` is a valid sha256, and the URL holds no `"`, space or `\` +- for `oci://` or `oci+http://`: the host and the repository are valid + +Print mode picks the first provider and never prompts. When the run chooses another provider, the serial pass fetches that one alone. + +### Signatures + +With `PACMAN_OCI_SIG=auto` (the default), the prefetch writes `.sig` from `%g`, decoded, with no request. pacman checks the package against `%PGPSIG%` anyway. When `%g` is empty, the signature of an `oci://` line comes from the registry's layer. The prefetch fetches no `.sig` for an `http://` line. The single-file path then reads `.sig`. + +With `PACMAN_OCI_SIG=registry`, the signature of an `oci://` line always comes from the registry's layer. Verify uses this mode, so it checks the stored layers. + +### Blob addresses + +With `auto`, a valid `%h` and a valid `%g`, the prefetch reads `/blobs/sha256:` directly, with no manifest request. Otherwise it reads the package manifest by its tag, then the layers `FILE` and `FILE.sig`. A layer whose digest is not `%h` is not used. + +### Transfers + +One curl process runs each list with `--parallel --parallel-max $PACMAN_OCI_JOBS` and a generated `--config` file. Each transfer has its own block with its URL, output, headers and `--max-filesize`. There are at most two phases: the manifests, when any are needed, then the blobs. + +- Each finished transfer is checked by size and sha256, then renamed to its ready entry. A file that fails the check is removed. +- A transfer that is cut goes on in the next pass. Cut means: its time limit, 408, 429, 5xx, or a connection that broke after the answer started. With bytes on disk, it resumes with `--continue-at -`. Each pass is a new curl process, so it has new connections. +- A transfer with no answer at all, another 4xx, or a file over its size is not tried again in the prefetch. +- A resumed file that fails its sha256 check starts again from zero, once. +- A pass after the first that makes no file ready and moves less than 30 KiB is the last. There are at most 100 passes. +- Each transfer of pass N gets 3 s times N plus 1 s per MiB, at most 30 s. A file that needs longer goes on in the next pass. So a slow connection costs at most 30 s. + +⭐ The prefetch never fails a call. Any file that it did not make ready goes through the single-file path. + +On a terminal, or with `PACMAN_OCI_DEBUG=1`, the prefetch prints one line: the file count, the MiB and the seconds. It prints nothing when it made no file ready. + +## Database prefetch + +The first `oci://` request for a `.db` or `.files` file of a run does the same for databases, once. It reads only the extension that pacman asked for. + +1. It reads `--config` from pacman's arguments (default `/etc/pacman.conf`). It does nothing with `--sysroot` or `--arch`, or without `pacman-conf`. +2. It lists the repositories with `pacman-conf --config C --repo-list`, and the first Server of each with `pacman-conf --config C --repo R Server`. +3. For each first Server that is `oci://` or `oci+http://`, it reads the manifests in parallel. +4. A manifest asked for by digest must hash to that digest. A manifest with more than one signature layer is not used. +5. It reads each database blob in parallel, except when the copy pacman holds already has that sha256. pacman downloads every database again on each `-Sy` under an `XferCommand`. +6. A manifest with a signature layer gives its `.sig` too. When a database is ready and its manifest has no signature layer, the wrapper writes an `,absent` entry. The request for that `.sig` then gets the answer for an absent database signature (below), with no network. + +## The single-file path + +A file the store does not hold is read alone. + +1. A host with a `,down.` entry misses at once, for a `pkg` or `db` request. A `.sig` request is never skipped: pacman asks for a `.sig` only from the server that served its file. +2. A URL that is not `oci://` goes to curl (below). +3. The wrapper reads the manifest. It sends `Authorization: Bearer QQ==`, which GHCR accepts for public packages, and saves a token round trip. +4. A 401 or 403 makes it read the Bearer challenge and fetch the anonymous pull token once. Manifest and token requests follow no redirect. +5. A 303 is asked again once, after 1 s. GHCR answers a reference it cannot parse with 303 to an HTML page. +6. A manifest asked for by digest must hash to that digest. +7. Exactly one layer must carry the file's name. A package manifest with no signature layer is an error: pacman tries no other server for a `.sig`. +8. A database signature must sit beside the very database pacman holds. Its manifest is found by the database's content: `db-HEX` for a `.db.sig`, `files-HEX` for a `.files.sig`. When that tag is missing, the URL's own manifest is used only when it names that database. With no database beside OUTPUT, the URL's own manifest is used. +9. A file that pacman already holds with the right size and sha256 is copied to OUTPUT, with no download. +10. The blob is read in up to 3 attempts. The blob's size and sha256 must equal the layer's descriptor. + +### Resume + +A blob download keeps OUTPUT between attempts and resumes with `--continue-at -`. A resumed file that fails the sha256 check is removed and fetched once more from zero. + +An attempt is tried again after HTTP 408, 429 or 5xx. It is also tried again after curl exits 6, 16, 18, 28, 35, 52, 55, 56 or 92. These are a name, TLS, HTTP/2 or time failure, an empty answer, or a cut transfer. After curl exit 33 (a range the server refuses), OUTPUT is removed and the attempt is tried again. Any other failure misses at once. The waits between attempts are 1 s, then 2 s. + +When the size of the file is known, each attempt has a time limit: + +| attempt | time limit | +| --- | --- | +| 1 | 3 s plus 1 s per MiB | +| 2 | 30 s plus 1 s per 128 KiB | +| 3 | none | + +No attempt starts after 30 s, unless the one before it added bytes. + +### Plain URLs + +pacman's own downloader is off when an `XferCommand` is set. So the wrapper also fetches the fallback mirrors, with curl. A package resumes what it got. When the prefetch of the run planned the package, its `,index` line bounds the size, and a resumed file must have its sha256. + +### The host blacklist + +A request that ends with no answer (HTTP 000), 429 or 5xx, once its own retries are spent, writes `,down.`. Later `pkg` and `db` requests of the same run, in the same directory, to that host miss at once. pacman then tries the next Server with no wait for timeouts. + +### An absent database signature + +⚠ pacman sleeps 2 s after every download that fails, also for an optional signature. No upstream of this mirror publishes a database signature, so each repository costs 2 s on every `-Sy`. + +Under the sandbox, OUTPUT is in a `download-XXXXXX` directory. pacman 7.0 and 7.1 unlink a zero-byte file there, and then read the signature as missing (libalpm `dload.c` `finalize_download_file`). So when the manifest has no signature layer, the wrapper writes an empty OUTPUT and exits 0. There is then no pause. + +The wrapper does this only when all of these are true: + +- the file is a database signature +- OUTPUT is in a `download-XXXXXX` directory +- the parent pacman says it is version 7.0 or 7.1 +- `PACMAN_OCI_EMPTY_SIG` is not `0` + +With no sandbox, the empty file would stay as the signature. So the wrapper misses there, and pacman pauses. Set `SigLevel = Required DatabaseNever` in each `oci://` repository section to remove the request. + +### Deadlines + +| request | limits | +| --- | --- | +| manifest, token, single-file path | `--connect-timeout 10 --max-time 15`, `--retry 2 --retry-max-time 30`, 64 KiB at most | +| manifest, prefetch | `--connect-timeout 10 --max-time 15`, 64 KiB at most, no retry | +| blob, single-file path | `--connect-timeout 20 --speed-limit 1024 --speed-time 60`, plus the attempt limits above | +| prefetch transfer | `--connect-timeout 10 --speed-limit 1024 --speed-time 60`, plus the pass limit above | +| print run | `timeout 120` | + +A manifest far larger than any real one fails at once. The parser is plain awk. + +## Environment + +| variable | default | effect | +| --- | --- | --- | +| `PACMAN_OCI_DEBUG` | `0` | `1` prints each request and decision to stderr | +| `PACMAN_OCI_PREFETCH` | `1` | `0` turns the prefetch off: each file is one request | +| `PACMAN_OCI_JOBS` | `32` | transfers at once in a prefetch, at most 64; `1`, or a value that is not a number, turns the prefetch off | +| `PACMAN_OCI_SIG` | `auto` | `registry` reads each package signature from the registry's layer, never from `%PGPSIG%` | +| `PACMAN_OCI_NO_DUMMY` | `0` | `1` skips `Bearer QQ==` and uses the registry's token flow, with no prefetch; the watchdog uses it | +| `PACMAN_OCI_EMPTY_SIG` | `1` | `0` turns off the empty answer for an absent database signature | +| `PACMAN_OCI_TEST_PROC` | unset | for the test suite only: read the parent from DIR, not `/proc`; used only when DIR holds a file named `pacman-oci-test-proc` | +| `TMPDIR` | `/tmp` | where the private work directory goes; OUTPUT's directory when TMPDIR is not writable | + +## Security of the store + +- The wrapper serves no package or database that a call did not check. A ready entry passed its size and sha256 check, against `%SHA256SUM%` or the layer descriptor, before it got its name. A `.sig` entry made from `%PGPSIG%` is the value that pacman checks the package against. +- An entry is used only when it is a regular file owned by the user that runs the wrapper, and not a symbolic link. +- Under `DownloadUser`, the store sits in the `download-XXXXXX` directory, mode 0700, owned by the download user. pacman removes that directory with the run. +- Without `DownloadUser`, the store sits in the sync or cache directory, which root owns. The wrapper runs as root there. +- A store lasts one run of pacman. There is no cache across runs. A package tag can be pushed again to repair its signature layer. So a cache keyed by tag could serve a stale `.sig`. pacman's own `CacheDir` keeps packages across runs. +- A filename never reaches a shell as code. sed and awk read it as data, on stdin or through the environment. On the `oci://` path, the wrapper refuses a name that makepkg could never make. That is an empty name, `.`, `..`, or a name with a space, a control character or a byte above 0x7e. +- The test hook `PACMAN_OCI_TEST_PROC` works only with its marker file, so it cannot trigger by accident. diff --git a/docs/conventions.md b/docs/conventions.md new file mode 100644 index 0000000..f5d757b --- /dev/null +++ b/docs/conventions.md @@ -0,0 +1,89 @@ +# Conventions + +Keep these conventions in every change. The static suite enforces most of them. + +## Commits + +- Make one squashed commit per piece of work on `main`. +- Commit with the maintainer's own identity. Add no co-author line and no "generated with" line. + +## Scripts + +- A script has no extension. Its header says what it does, how to call it, and what its exit codes mean. +- A bash script starts with `set -euo pipefail`. `client/pacman-oci` is the one POSIX sh file. It must run under bash, dash and busybox sh. +- A python script (`scripts/status-render`) uses the standard library only. It runs as `python3 -I`. Its first line names python3. +- A script with a `#!` line is mode 100755 in the git index: `git update-index --chmod=+x FILE`. `scripts/lib.sh` is sourced, has no `#!` line, and stays 100644. `tests/static/40-executable-bits.sh` checks this. +- Every file under `client/` and `scripts/` is shell or python. `tests/static/70-python.sh` checks this. +- A script reads its inputs as data. A target file is parsed, never sourced. Each field of a database is checked against its grammar before it reaches a path, a command line or a tag. + +## Failures stay visible + +⛔ Never hide a failure. + +- Do not write `|| true`, `set +e` or `continue-on-error`. +- Do not send the output of anything that matters to `/dev/null` (`2>/dev/null`, `>/dev/null 2>&1`). +- Capture an exit status as `rc=0; cmd || rc=$?`, then act on it. +- Give every curl call `--connect-timeout`, and `--max-time` or `--speed-time`, on its own command line. +- Run every oras call under `timeout`. + +`tests/static/30-failures-stay-visible.sh` checks these rules in the shipped scripts and the workflows. It checks the oras rule in `scripts/` only. It does not check the `rc=0` form. + +⛔ Do not end a pipeline in a reader that stops early. Examples are `head -n1`, `awk '... exit'`, and `grep -q` on a producer that keeps writing. Under `pipefail`, SIGPIPE (status 141) then ends the script. Read everything and decide at the end. `tests/static/35-pipeline-traps.sh` checks this rule. + +- Each failure message says what was measured and how to reproduce it. `scripts/lib.sh` `die` prints one item per line. In Actions, its first line is also an error annotation. +- A script that sources `scripts/lib.sh` has an ERR trap. It names the command that ended the script under `set -e`, so no exit is silent. + +## Workflows + +`tests/static/50-workflow-policy.sh` checks each rule: + +- `permissions: {}` at the top. Each job asks for what it needs. +- `pages: write` and `id-token: write` only in job `deploy` of `status.yml`. `contents: write` only in job `build` of `status.yml`. +- `shell: bash` by default, so each `run:` block has `-eo pipefail`. +- `timeout-minutes` on every job. A job that calls a reusable workflow cannot have one; the jobs of the called workflow have their own. +- Actions pinned by commit, with the version in a comment. The test checks the pin, not the comment. A local path (`./.github/workflows/x.yml`) is the same commit, so it counts as pinned. +- `persist-credentials: false` on every checkout. +- No `${{ inputs.* }}`, `${{ matrix.* }}`, `${{ needs.* }}` or event value inside a `run:` block. Pass the value through `env:`. +- No `pull_request_target` or `workflow_run` trigger. +- No use of the packages DELETE API. Nothing is ever deleted. +- No unquoted `if: !...`, which YAML reads as a tag. +- Every workflow with a `schedule:` is one that `scripts/keep-schedules-enabled` enables. + +## Images + +- Pin each image as `name:tag@sha256:<64 hex>`. +- Write a `# tag: ` line at most 3 lines before the pin, with the pin's own tag. Renovate reads that line. `tests/static/75-renovate.sh` checks each pin and its line. +- In a target file, write the `# tag:` line directly before the `verify_image` or `keyring_image` line. `tests/static/60-targets.sh` checks it. + +## Tests + +- ⛔ See each new test fail once. Break the code in a copy, watch the test fail, and keep the tree as it was. Record the test, the fault and what was observed in [tests.md](tests.md). +- In a test that needs a tool which may be absent, call `need_tool`. +- An integration test picks free ports at run time and gives each container a unique name. Several runs can then share one host. +- A failed assertion prints what was measured and the command that reproduces it. + +## Measurements + +- Every number in the documents comes from the code or from a measurement. [measurements.md](measurements.md) gives the date and the source of each measurement. +- ⚠ Do not take timings in Git Bash on Windows. A process start costs about 165 ms there (20 runs of `/usr/bin/true` took 3297 ms). + +## Text + +Comments, messages, documents and test names use ASD-STE100 Simplified Technical English. + +- Write one idea per sentence. +- Keep a procedural sentence to 20 words or fewer, and a descriptive sentence to 25 words or fewer. +- Use the active voice. Use the imperative for an instruction. Use simple tenses. +- Use simple, approved words: use, do, make, start, stop, show. Do not use utilize, leverage, ensure, robust, seamless or comprehensive. +- Use one name for one thing: "target", "snapshot", "the wrapper", "the Worker". +- Put a warning before the step it applies to. +- State the fact, the rule and the measurement. Write no narrative history: no "we found", no "the previous session". Dates of events go in [measurements.md](measurements.md) only. +- Do not use em dashes or marketing adjectives. + +Use only these three markers: + +| marker | means | +| --- | --- | +| ⛔ | a rule that must not be broken | +| ⭐ | a property the design depends on | +| ⚠ | a caution or a known limit | diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..7ccf307 --- /dev/null +++ b/docs/design.md @@ -0,0 +1,236 @@ +# Design + +This document gives the invariant of the mirror, each rule with the failure it prevents, the storage layout, the jobs and the verify step. The measurements behind the rules are in [measurements.md](measurements.md). + +## The invariant + +⛔ A database reaches `latest` only after every file it names is on the registry. Every file new since the promoted database must also pass pacman's own checks through the wrapper. + +Each rule below keeps this invariant true for one way it could fail. + +## Rules, and the failure each one prevents + +| rule | failure it prevents | where | +| --- | --- | --- | +| The database comes only from the target's `db_server` hosts, which the distribution runs. Package bytes can come from any `server`. | The databases are unsigned. The source of a database chooses which validly signed packages get installed, old ones too. | `scripts/plan` | +| Every field of a database is checked against its grammar first. A filename with `/`, a space, a leading `-` or `.`, a control character or a byte above 0x7e refuses the whole database. | A hostile name becomes a path, an option or a tag. | `scripts/lib.sh` `db_validate` | +| Each package directory gives exactly one record. A `desc` member of NUL bytes only, or empty, is counted and named; at most 16 are accepted. Any other unreadable member refuses the database. | A database that drops packages without a word. An upstream sparse member that holds a target for ever. | `scripts/lib.sh` `db_check` | +| Each (file, sha256) pair of `.files` must be in `.db`. A files database that names another pair is refused, and plan tries the next `db_server` that served the same database. A files database that leaves packages out is accepted, and plan names them. | A files database that describes other packages makes `pacman -F` wrong. A refusal of a files database that only knows less holds the target for an upstream defect that upstream's own users also get. | `scripts/plan` | +| Never behind: a candidate is dropped when one of its packages was built before the same package in the promoted database or in another candidate. `ALLOW_OLDER` lifts the first comparison. | A lagging server rolls consumers back to older builds. | `scripts/plan` | +| A candidate with no package built later than in the promoted database waits while a `db_server` still serves the promoted database. A candidate that only adds or drops packages is one. `ALLOW_OLDER` lifts this wait too. | A server that lags by one dropped package moves `latest`. | `scripts/plan` | +| `latest` never moves backwards. Just before it tags, promote reads the `fetched` time of the snapshot `latest` names, and refuses a staged snapshot fetched earlier. | A re-run, or a second writer, rolls consumers back. | `scripts/promote` | +| ⛔ A package tag is written once. A tag that holds other bytes under a name is never written again. The target's state is `replaced`, and on `main` the `replaced-` issue opens. | The promoted or an older snapshot names those bytes. pacman refuses a file whose sha256 it does not expect, and tries no other Server. | `scripts/lib.sh` `decide_push`, `scripts/mirror-file` | +| A signature layer that is missing or wrong beside the right package bytes is repaired: the file is pushed again with the same package blob. plan skips a file only when its sha256 and its `%PGPSIG%` equal the promoted record. | A file that upstream signs again keeps a stale signature layer for ever. | `scripts/plan`, `scripts/mirror-file` | +| A package never lacks its signature when the database has one. The layer is the decoded `%PGPSIG%`. With no `%PGPSIG%`, it is the detached `.sig`, which must be one OpenPGP signature packet under 64 KiB. Any other answer is an error, not an absence. A file is pushed unsigned only when every server answers 404 for its `.sig`. | pacman asks for a `.sig` only from the server that served the package. A missing `.sig` ends the install. | `scripts/mirror-file` | +| Verify compares every `.sig` pacman stored with the decoded `%PGPSIG%`. | pacman checks against `%PGPSIG%` and stores the served `.sig` unread. A wrong layer passes verify and breaks a later `pacman -U` from the cache. | `scripts/verify-batch` | +| Package manifests are the same bytes whoever pushes them. `org.opencontainers.image.created` is the package's `%BUILDDATE%`, and no annotation names the source server. | Two writers replace each other's tag with a manifest of another digest. | `scripts/mirror-file` | +| Absent means the registry says so: only a 404 with `NAME_UNKNOWN` or `MANIFEST_UNKNOWN`. On GHCR, with a token set, a read tries `Bearer QQ==` first, and only its 200 counts. Any other answer is read again with the real token. | GHCR answers `QQ==` with 404 for a repository it does not show. A private package reads as absent, and its tags are pushed over. | `scripts/lib.sh` `registry_manifest` | +| A file that upstream supersedes during a run holds the target for that run. A file that never succeeds holds its target until upstream drops or rebuilds it. | A database is staged that names a file the registry does not hold. | `scripts/stage` | +| Each target runs its own chain and reads only its own artifacts. Every artifact name carries the target. A chain reads its shard results and verdicts by the pattern `--[0-9]*`. So a target name that extends another target name with a letter never matches. ⚠ An extension that starts with `-` and a digit, as in `x86_64-core-2`, still matches. | One slow or failed target holds every other target. A chain reads the verdicts of another target. | `.github/workflows/sync-target.yml` | +| A push attempt gets 300 s plus the time at `MIRROR_MIN_RATE` (2 MiB/s), and never more than 600 s. | Attempts run past GHCR's 10-minute upload limit and are lost. | `scripts/mirror-file` | +| A 429 from the registry costs no attempt. Each push that gets one writes `$WORK/throttle`, the time its pause of 20 to 60 s ends. Every push of the shard waits for that time first. The pauses that one file takes after its own 429s add up to at most `MIRROR_RATE_WAIT` (1800 s). | GHCR limits the requests of one token, and every push of every job shares the workflow token. Three attempts, 5 s and 20 s apart, all fall inside the limit, and the file fails. | `scripts/mirror-file` | +| Each item of a shard writes its result line the moment it ends. A work line with no result counts as failed. | A shard that runs out of time loses the results of the files it pushed. | `scripts/mirror-shard`, `scripts/mirror-file` | +| The verify budgets nest. The pulls and the container end inside `VERIFY_BUDGET` (4800 s). That ends inside the step's 85 minutes, which end inside the job's 95. The verdict is written by an EXIT trap. | The runner ends the job, and no verdict is left. | `scripts/verify-batch`, `sync-target.yml` | +| The verify container asks no keyserver: its `gpg.conf` names `hkp://127.0.0.1:9`, which refuses at once. | A missing or expired key makes verify wait on a host outside the run. | `scripts/verify-batch` | +| No exit is silent. Each script that sources `scripts/lib.sh` has an ERR trap that names the command which ended it. No pipeline ends in a reader that stops early. | A step ends with status 141 (SIGPIPE) and no message. | `scripts/lib.sh`, `tests/static/35-pipeline-traps.sh` | + +## Storage layout + +``` +ghcr.io/pkgforge-dev/mirror-archlinux// + : one package: the file and its signature + :staging the candidate database of the run on main + :staging- the candidate database of a run from another branch + :latest the promoted database + :YYYY.MM.DD the first promoted database fetched on that UTC date; never moves + :db-.db> the promoted database by its content + :files-.files> the same, by the files database +``` + +`` is pacman's `$arch`. One GHCR repository holds one target. Nothing is ever deleted: `tests/static/50-workflow-policy.sh` refuses every use of the packages DELETE API. + +`` is the branch name in lower case. Each character other than `a-z` and `0-9` becomes `-`, a run of `-` becomes one, and a leading or trailing `-` is removed. Then the name is cut to 40 characters. An empty result is `branch`. + +## Tags + +A package tag is the filename with this mapping: + +- `+` becomes `_p`, `:` becomes `_c` and `@` becomes `_a`. +- A literal `_` becomes `_u` only when the next character is `p`, `c`, `a` or `u`. So `x86_64` stays `x86_64`, and the mapping is injective. + +GHCR resolves tags with no regard to case, so every tag is lower case. Some filenames are stored under `fn-` and the hex sha256 of the name. These are a name with upper case or another character outside `[a-z0-9@._+:-]`, and a name that the mapping cannot carry in 128 characters. That form has no dot. Every escaped package name has `.pkg.tar`. So the two forms never meet, and neither meets `latest`, `staging`, a date, `db-` or `files-`. + +The mapping has one implementation: `client/pacman-oci --tag` and `--tags`. The scripts call it. The Worker has a copy, `tagOf`. `worker/test/tags.test.js` compares the two for every name in `tests/fixtures/filenames.txt`. + +## Manifests + +A package manifest: + +- artifact type `application/vnd.pkgforge.archlinux.package.v1` +- layer 0: the package, `application/vnd.pkgforge.archlinux.pkg.v1`, titled with its filename +- layer 1: its signature, `application/pgp-signature`, titled `.sig`, when the file has one +- annotations: `org.opencontainers.image.created` (the `%BUILDDATE%`), `org.opencontainers.image.source`, `org.opencontainers.image.title`, and `dev.pkgforge.mirror.arch`, `.repo`, `.pkgname`, `.sha256` + +A database manifest: + +- artifact type `application/vnd.pkgforge.archlinux.db.v1` +- layers `.db` (`application/vnd.pkgforge.archlinux.db.v1`) and `.files` (`application/vnd.pkgforge.archlinux.files.v1`), byte for byte as upstream serves them +- a `.db.sig` and a `.files.sig` layer, only when upstream publishes them +- annotations: `dev.pkgforge.mirror.arch`, `.repo`, `.upstream` (the server), `.db-sha256`, `.files-sha256`, `.packages`, `.bytes`, `.newest-builddate`, `.fetched` (ISO 8601 UTC), `.run`, and the OCI `source`, `title` and `revision` + +## Jobs + +`.github/workflows/sync.yml` runs on the cron `17 */3 * * *` and on dispatch. + +| job | permissions | timeout (min) | does | +| --- | --- | --- | --- | +| `plan` | contents read, packages read | 30 | `scripts/plan` for every enabled target, or for the targets of the `targets` input; uploads `plan-` for each pending target | +| `target` | contents read, packages write | (each called job has its own) | one call of `sync-target.yml` per pending target, at most `max_targets` (4) at once | +| `report` | actions write, contents read, issues write | 10 | `scripts/report`; on `main` it starts `status.yml` | +| `keepalive` | actions write, contents read | 10 | `keep-schedules-enabled`, `check-schedules-fired`, and `dispatch-overdue status.yml 120` on `main` | + +`.github/workflows/sync-target.yml` is the chain of one target: + +| job | permissions | timeout (min) | does | +| --- | --- | --- | --- | +| `mirror` | contents read, packages write | 120 (step 110) | `scripts/mirror-shard` per shard, at most `max_shards` (3) at once | +| `stage` | contents read, packages write | 20 | `scripts/stage` for the target | +| `verify` | contents read | 95 (step 85) | `scripts/verify-batch` per batch, at most `max_batches` (3) at once | +| `promote` | contents read, packages write | 15 | `scripts/promote`, on `main` only | +| `summary` | contents read | 10 | `scripts/report collect`: a small `summary-` artifact for the report | + +All runs share one concurrency group, `sync`, on every ref. Two runs never write the same tags at once. + +The dispatch inputs are `targets`, `max_targets` (4), `max_shards` (3), `max_batches` (3), `parallel` (6 files at once in a shard), `debloat` (`auto`, `always`, `never`) and `allow_older`. + +A dispatch input reaches an expression as a string. `max-parallel` and a number input of a called workflow refuse a string. So `sync.yml` passes each number input through `fromJSON(format('{0}', x))`, which gives a number from a string or a number. `sync-target.yml` passes its two `max-parallel` values the same way. + +### Shards + +plan gives each target shards of its own. The count is k = the larger of ceil(bytes / `SHARD_BYTES`) and ceil(files / `SHARD_FILES`). k is at least 1, at most `MAX_SHARDS`, and at most the file count. Files go largest first onto the lightest shard. plan logs when `MAX_SHARDS` binds, with the GiB and files per shard. Each shard reads the servers in an order that starts at its own index, so the shards spread across the mirrors. + +| variable | default | +| --- | --- | +| `SHARD_BYTES` | 536870912 (512 MiB) | +| `SHARD_FILES` | 400 | +| `MAX_SHARDS` | 32, per target | + +⚠ GHCR allows one token 2000 requests a minute, and every push of every job shares the workflow token. That limit, not the parallelism, bounds a seed: about 200 files a minute ([measurements.md](measurements.md#ghcr)). + +plan writes `OUT/targets.json` for the pending targets with files to mirror, and `OUT/pending.json` for pending targets with no file to mirror. A database that only drops packages is such a target. It still stages and verifies. + +### Batches + +stage splits the files new since the promoted database into verify batches. The count is the larger of ceil(bytes / `VERIFY_BATCH_BYTES`) and ceil(files / `VERIFY_BATCH_FILES`). When that is more than `VERIFY_MAX_BATCHES`, both limits grow until it is not, and the log says so. + +| variable | default | +| --- | --- | +| `VERIFY_BATCH_BYTES` | 2147483648 (2 GiB) | +| `VERIFY_BATCH_FILES` | 400 | +| `VERIFY_MAX_BATCHES` | 40, per target | + +Batch 0 also holds the canaries: the smallest package, and each keyring package that the target's own database holds. So a run with nothing new still proves the path a consumer takes. + +### Disk + +A shard works in the directory with the most free space among `$RUNNER_TEMP`, `/mnt` and `/tmp`, each filesystem counted once. Each file is deleted once it is pushed. So a shard needs room only for its `PARALLEL` largest files, plus 5 % and 1 GiB. + +| `DEBLOAT` | `scripts/debloat-runner` | +| --- | --- | +| `auto` (the default) | runs only when the files do not fit | +| `always` | runs on each shard | +| `never` | does not run | + +If the files still do not fit, `PARALLEL` comes down. A shard fails when its largest file does not fit. The `debloat` job of `ci.yml` runs `debloat-runner` in each CI run, so it is seen to work. + +## Verify + +`scripts/verify-batch STAGE TARGET BATCH OUT` checks one batch the way a consumer reads it. + +### One image, no emulation + +Every target verifies in one image: `docker.io/library/archlinux:base`, pinned by digest, on `linux/amd64`. pacman.conf sets `Architecture` to the target's arch. pacman on x86_64 then downloads and checks the packages of another arch. No target needs QEMU. `tests/static/60-targets.sh` checks that every target names the same image and platform. + +verify writes its own pacman.conf: + +```ini +[options] +Architecture = +CheckSpace +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +DownloadUser = alpm +XferCommand = /usr/local/bin/pacman-oci %o %u + +[] +Server = oci://ghcr.io/pkgforge-dev/mirror-archlinux//@ +``` + +With `db_signed = yes`, the SigLevel is `Required DatabaseRequired`. With `keyring_from`, and when that target has a `latest`, a second section follows (below). + +### The trust root + +1. `pacman-key --init`. +2. When the target names `keyring_image`, verify pulls it for `keyring_platform`. It runs `create` and `cp` only: nothing in the image runs, so no emulation is needed. It accepts only plain keyring files. It adds each one to `/usr/share/pacman/keyrings/`, never over a file the verify image has. +3. `pacman-key --populate`. +4. verify installs each `keyring` package with `pacman -Sdd`. The packages come from the staged repository, or from the second section of `keyring_from`. The files copied in step 2 are removed first, because the packages own them. +5. `pacman-key --populate` again, so a key added upstream is signed locally before any package is checked. + +`keyring_from` names another target of the same arch. Its `latest` is read when verify starts, and its section is pinned by that digest. When it has no `latest` yet, verify logs a warning and skips steps 4 and 5. The trust root of steps 1 to 3 still applies, and pacman still checks every signature. Any other answer from the registry fails the batch: no answer is not an absence. + +### Phases + +Each phase is logged with its UTC time. + +| phase | does | +| --- | --- | +| `pull` | the verify image, and `keyring_image` when named: two attempts of 300 s each | +| `keyring` | the trust root, above | +| `canaries` | batch 0 only: the canaries through the wrapper's single-file path (`PACMAN_OCI_PREFETCH=0`), into a cache of their own | +| `batch` | every package through the wrapper's default path, with `PACMAN_OCI_SIG=registry`, so the stored signatures are the registry's layers | +| `refusals` | after a download that failed: each package alone, to name the ones at fault | +| `sig check` | each stored `.sig` must be the decoded `%PGPSIG%` | +| `files db` | `pacman -Fy`: the synced `.files` must have the staged sha256. `pacman -Fl` reads each name of the batch that the files database holds. The log names the other names of the batch. | + +The canaries and the batch run both paths of the wrapper on every sync. + +### Upstream-refused + +pacman refuses some files from upstream too. Examples are a package signed by a key that expired, and a package with no signature anywhere. These are not the mirror's fault when the mirror serves exactly upstream's file. + +A package that pacman refuses alone, for a reason in its signature, is upstream-refused when all of these are true: + +- its tag holds the database's sha256 and size +- its signature layer is the decoded `%PGPSIG%`; or, with no `%PGPSIG%`, the `.sig` that upstream serves now; or upstream has no signature, the tag has none, and every server answers 404 for the `.sig` + +`scripts/lib.sh` `refusal_reason` reads pacman's output and the wrapper's exit status: + +| class | means | +| --- | --- | +| `sig-missing` | no signature could be read | +| `sig-refused` | pacman read a signature and refused it: an expired or unknown key, a trust it does not accept, a signature not valid for the bytes | +| `other` | anything else; an `other` line wins over any signature line | + +Only `sig-missing` and `sig-refused` can be upstream-refused. They go to `OUT/refused--.tsv`. The batch passes when every failure is upstream-refused, with the count in the verdict. promote accepts it, and the report lists the files. + +### The trust guard + +⛔ Upstream-refused is believed only when the trust root is seen to work. That means verify installed a keyring package, or pacman accepted a package of the batch. A trust root that accepts nothing refuses every package. With no keyring package installed and every package refused, verify fails the batch and excuses nothing. + +### NUL-only members + +Two upstream databases hold a `desc` member that is all NUL bytes: a GNU sparse member with no data. pacman lists such a package with every field `None` and cannot download it. plan accepts these members, at most 16 per database, and names them in `OUT//nul.txt`, in `why` and in the summary. Any other unreadable member refuses the database. + +### The verdict + +`OUT/verify--.tsv` holds `target batch result detail digest`. The result is `passed` or `failed`. The digest is the staged manifest the verdict is about. promote counts a batch with no verdict, or a verdict about another digest, as failed. The EXIT trap writes the verdict and the packages at fault so far, also when the step is killed. When the container ends with status 124 or 137, the verdict says that the script's own budget ran out. + +## Running outside GitHub Actions + +The scripts take the registry from `MIRROR_REGISTRY`, `MIRROR_NAMESPACE`, `REGISTRY_USER` and `REGISTRY_TOKEN`, and the targets from `TARGETS_DIR`. So another host can ingest a source that runners cannot read. + +- `sync.yml` must create each GHCR repository first, with the workflow token. That links the package to this repository. A package first pushed from outside is not linked, and the workflow token can never write to it. +- ⚠ An outside host must not use a maintainer's token. GHCR accepts only classic tokens, which can write every package of their owner. Use a dedicated account with write access to these packages only. +- Each script that pushes (`mirror-shard`, `stage`, `promote`) logs in with `--password-stdin`. The registry configuration is in its own temporary directory, which the script removes on exit. +- ⚠ `scripts/debloat-runner` runs only on a GitHub-hosted Linux runner, and fails elsewhere. Set `DEBLOAT=never` on another host. `PARALLEL` is 4 there by default, not 6. +- A push must stay inside GHCR's 10-minute upload limit. The largest file of a target sets the lowest upload rate it can use. diff --git a/docs/measurements.md b/docs/measurements.md new file mode 100644 index 0000000..b8397b6 --- /dev/null +++ b/docs/measurements.md @@ -0,0 +1,240 @@ +# Measurements + +Measured facts only. Each table gives its date and its source. A row that comes from documentation says so. Times are UTC unless a table says otherwise. + +## pacman + +Source: pacman v7.1.0 (commit 5683f847), read from source. Also pacman 7.1.0.r9.g54d9411-2, run in `archlinux:base` under podman with no network, with an `XferCommand` that logs each call. Date: 2026-10-14. + +| behaviour | measured | +| --- | --- | +| downloads under an `XferCommand` | one file at a time, in order; 2 packages with `ParallelDownloads = 5` were fetched back to back | +| `ParallelDownloads`, server error limit | not used under an `XferCommand` | +| after a failed `XferCommand` call | `sleep(2)`, also for an optional `.sig`; 4 failing servers took 8.28 s | +| `SigLevel = Required DatabaseOptional`, no `.db.sig` on the server | 2.16 s for `-Sy` | +| `SigLevel = Required DatabaseNever` | no `.db.sig` request; 0.10 s for `-Sy` | +| `%o`, `%u` | replaced only when they are a whole argument | +| the `XferCommand` process | a direct child of the pacman main process; pacman waits only on its pid | +| the `XferCommand` under `DownloadUser` | uid switch and `NO_NEW_PRIVS` only; no Landlock, no seccomp; `CapEff` 0 | +| what it can write under `DownloadUser` | its `download-XXXXXX` directory (mode 0700) and `/tmp`; not the cache or sync directory | +| `download-XXXXXX` | one per operation: `-Syu` makes two | +| cleanup of `download-XXXXXX` | payload files and their `.sig` are moved out; other regular files are unlinked; a subdirectory makes the cleanup fail and the directory stays | +| a zero-byte `%o` under the sandbox | unlinked; the old file is kept; pacman exits 0 | +| a zero-byte `%o` with no sandbox | replaces the database; pacman exits 0 | +| `DownloadUser` with both sandbox parts off | the `XferCommand` runs as the download user in a root-owned directory: permission denied | +| `-Syy` with no `DownloadUser` | the database is deleted before the `XferCommand` runs | +| a database under an `XferCommand` | downloaded again on every `-Sy`: no If-Modified-Since | +| `pacman -Sp` | takes no lock and needs no root; `-Syp` needs root | +| `--print-format` | `%l` the first Server or `file://` when cached, `%h` sha256, `%s` download size (0 when cached), `%g` base64 `%PGPSIG%` | +| print mode | never prompts; takes the first provider | +| a cached package with required signatures and no cached `.sig` | the package and its `.sig` are downloaded again | +| native downloader, redirects | follows a 302 or 307 to another host; at most 10 redirects; `file://` refused | +| native downloader, 404 on an optional `.db.sig` | no error line and no pause (0.04 s) | +| native downloader, Range on resume | sent again to the redirect target; 206 completed the file | +| native `DownloadUser` sandbox in podman on WSL2, no `CAP_SYS_ADMIN` | fails: Landlock ruleset not applied | + +| behaviour | date | source | +| --- | --- | --- | +| pacman on x86_64 with `Architecture = ` downloads and checks another arch's packages | 2026-10-14 | `scripts/verify-batch` in `archlinux:base` | +| pacman 7.1 checks a package against `%PGPSIG%`, and still asks for its `.sig` and stores it unread | 2026-10-14 | pacman against the promoted x86_64 core | +| pacman 7.1 takes an expired primary key for a missing key: it asks a keyserver, then says "required key missing from keyring" | 2026-10-14 | the integration suite and loong64 extra | +| pacman 7.1 under an `XferCommand` names no file when a package's `.sig` cannot be served: only "failed to retrieve some files" | 2026-10-14 | the integration suite | + +## GHCR + +| fact | date | source | +| --- | --- | --- | +| `Authorization: Bearer QQ==` gives 200 on a public manifest and blob | 2026-10-13 | curl against ghcr.io | +| no Authorization header gives 401 with a Bearer challenge | 2026-10-14 | curl against ghcr.io | +| an unknown bearer gives 403 with no challenge | 2026-10-13 | curl against ghcr.io | +| `Bearer QQ==` for a repository that does not exist gives 404 `MANIFEST_UNKNOWN` | 2026-10-14 | curl against ghcr.io | +| a blob GET gives 307 to a signed URL on `pkg-containers.githubusercontent.com`; that URL needs no Authorization | 2026-10-13 | curl against ghcr.io | +| a Range request on a blob gives 206; a resumed download was byte-identical | 2026-10-14 | curl against ghcr.io, a 199 MiB blob | +| blobs are scoped per repository: another repository's blob gives 404 | 2026-10-13 | curl against ghcr.io | +| tags resolve with no regard to case: `latest`, `LATEST` and `Latest` name one manifest | 2026-10-13 | curl against ghcr.io | +| a reference of 129 characters gives 303 to an HTML page on github.com | 2026-10-14 | curl against ghcr.io | +| 30 manifest GETs at once all gave 200 | 2026-10-14 | curl `--parallel` against ghcr.io | +| a 200 carries no RateLimit or Retry-After header | 2026-10-14 | curl against ghcr.io | +| tag list: 100 per page by default; `n` is capped at 1000 | 2026-10-14 | curl against ghcr.io | +| a package that `GITHUB_TOKEN` creates from this public repository reads anonymously | 2026-10-14 | curl against ghcr.io | +| 299 of 299 pushes read back right on the first try | 2026-10-13 | the first sync of x86_64 core | +| 10 GB per layer and a 10-minute upload limit (documented); no rate limit documented | 2026-10-14 | GitHub docs, container registry | +| no REST endpoint changes a package's visibility (documented) | 2026-10-13 | GitHub REST API docs | +| oras 1.3.4 keeps the `created` annotation it is given: two writers pushed manifests of one digest, 3 of 3 | 2026-10-14 | the integration suite | +| pushes from runners got 429 `toomanyrequests` with "allowed: 2000/minute"; 4 targets x 3 shards x 6 pushes ran at once on one workflow token | 2026-10-14 | run 38570044665 | +| that limit bounds a seed to about 200 files a minute per token, whatever the parallelism | 2026-10-14 | run 38570044665 | + +GHCR latency from one host. Host: Windows, Git Bash, curl 8.21.0 over HTTP/1.1, residential network in Nepal; GHCR answered from edge `centralindia`. Date: 2026-10-14. + +| request | median | +| --- | --- | +| manifest GET, new connection (n=20) | 0.410 s | +| manifest GET, reused connection | 0.267 s | +| blob GET to the 307 only (n=10) | 0.392 s | +| blob GET, following the redirect (n=10) | 0.722 s | +| blob GET, reused connection | 0.569 s | +| 20 manifests with `--parallel` | 1.003 s in all | + +## GitHub Actions and Pages + +Source: GitHub docs, read from github/docs at commit d67ced72, and `gh api` reads of the organisation. Date: 2026-10-14. + +| fact | value | +| --- | --- | +| organisation plan | Free; 619 public repositories | +| concurrent jobs, shared by every repository of the organisation | 20 | +| jobs per matrix | 256 | +| time per job on a GitHub-hosted runner | 6 hours | +| REST requests of `GITHUB_TOKEN` | 1000 per hour per repository | +| artifact retention in a public repository | 1 to 90 days; at most 500 artifacts per job | +| `ubuntu-24.04-arm` | free for public repositories: 4 vCPU, 16 GB RAM, 14 GB SSD | +| enabling Pages | needs an admin token; `GITHUB_TOKEN` has no administration permission | +| Pages limits | 1 GB per site, 10 minutes per deployment, 100 GB per month soft bandwidth | + +| fact | date | source | +| --- | --- | --- | +| a job that calls a reusable workflow takes no `timeout-minutes` | 2026-10-14 | actionlint 1.7.12 | +| a dispatch input reaches an expression as a string; `max-parallel` refused it: "Unexpected value '4'" | 2026-10-14 | run 38569214434 | +| `keyring_image` with `create` and `cp` only, for `linux/loong64` and `linux/arm64`, worked on an amd64 runner | 2026-10-14 | run 38570044665 | +| a public-repository `ubuntu-24.04` runner starts with about 87 GB free on `/`; GitHub guarantees 14 GB | 2026-10-13 | runner image documentation | +| `scripts/debloat-runner` removed 12 of 12 directories in 45 s: 85 GiB free became 120 GiB | 2026-10-13 | a CI run on a runner | + +## Cloudflare Workers + +Source: Cloudflare pricing page (updated 2026-10-02) and limits page (updated 2026-10-12), documented. + +| fact | value | +| --- | --- | +| Workers Paid | $5 per month per account, 10 million requests included, then $0.30 per million | +| CPU | 30 million CPU-ms included, then $0.02 per million CPU-ms | +| subrequests, egress | not billed | +| Cache API | per data centre, not replicated; objects up to 512 MB | +| `limits.cpu_ms` | a cap per invocation, enforced on Cloudflare only | + +The Worker run locally. Source: `npx wrangler@4.150.0 dev --local` (workerd) against live GHCR, from the host in Nepal. Date: 2026-10-14. + +| request | measured | +| --- | --- | +| `core.db`, cold | 0.53 to 0.88 s | +| `core.db`, repeat (one cache lookup) | 4 to 6 ms | +| a package, cold | 0.29 to 0.79 s (manifest about 260 ms, blob about 250 to 310 ms) | +| a package, repeat | 3 to 5 ms | +| a `.sig` asked 2 s after its package | 3 to 4 ms | +| a `.sig` asked about 50 ms after its package | 0.25 to 0.29 s: the prewarm lost the race by about 5 ms | +| `wrangler deploy --dry-run` | 14.08 KiB, gzip 4.69 KiB, no warnings | + +| fact | date | source | +| --- | --- | --- | +| the deployed version 1 answered `core.db` with 302 in 1.05 to 1.33 s | 2026-10-14 | curl from the host in Nepal | +| 16 of 16 Worker probes passed; median 0.41 s | 2026-10-14 | `scripts/status-collect`, live | +| the GHCR `latest` manifest, read anonymously: median 0.43 s | 2026-10-14 | `scripts/status-collect`, live | + +## Upstream + +Source: each database downloaded from a `db_server` and read with `scripts/lib.sh` `db_records` and `db_validate`. The x86_64 core row comes from `scripts/status-collect`, live. Date: 2026-10-14. + +| target | packages | total `%CSIZE%` (bytes) | GiB | largest file (bytes) | `%PGPSIG%` | +| --- | --- | --- | --- | --- | --- | +| x86_64-core | 299 | 1981090710 | 1.85 | not measured | 299 of 299 | +| x86_64-extra | 15041 | 115541940123 | 107.61 | 3290190315 | 15041 of 15041 | +| x86_64-multilib | 182 | 371948848 | 0.35 | 121321827 | 182 of 182 | +| aarch64-core | 313 | 1576463280 | 1.47 | 172772972 | 313 of 313 | +| aarch64-extra | 12994 readable of 12995 | 56935118832 | 53.02 | 3405583252 | 12994 of 12994 | +| aarch64-alarm | 75 | 142390596 | 0.13 | 35327900 | 75 of 75 | +| aarch64-aur | 12 | 10992812 | 0.01 | 6974820 | 12 of 12 | +| armv7h-core | 326 | 1622069404 | 1.51 | 172776152 | 326 of 326 | +| armv7h-extra | 12555 | 45784791156 | 42.64 | 3405570792 | 12555 of 12555 | +| armv7h-alarm | 125 | 190947120 | 0.18 | 35377156 | 125 of 125 | +| armv7h-aur | 11 | 10670656 | 0.01 | 6563988 | 11 of 11 | +| loong64-core | 295 | 1904375301 | 1.77 | 208134504 | 263 of 295; the other 32 have a detached `.sig` | +| loong64-extra | 14184 | 78262056692 | 72.89 | 3064267589 | 12965 of 14184; 1218 have a detached `.sig`, 1 has none | +| riscv64-core | 294 | 2121172682 | 1.98 | 208134524 | 9 of 294; the other 285 have a detached `.sig` | +| riscv64-extra | 14716 readable of 14717 | 89248639018 | 83.12 | 3290190302 | 80 of 14716; the other 14636 have a detached `.sig` | + +| fact | date | source | +| --- | --- | --- | +| 15 targets: 71,422 upstream files, 395,704,667,230 bytes (368.5 GiB) | 2026-10-14 | `scripts/status-collect`, live | +| no target publishes `.db.sig` or `.files.sig`: 404 on every server tried | 2026-10-14 | curl on each `db_server` | +| 98 of 98 `db_server` and `server` URLs answered 200 for `.db` | 2026-10-14 | curl on each URL | +| aarch64 extra `findnewest-0.3-4/desc`: a GNU sparse member of 1271 NUL bytes; pacman lists it with every field None and cannot download it | 2026-10-14 | the database on every mirror; pacman 7.1 in the aarch64 image | +| riscv64 extra `ntk-1.3.1001-4/desc`: a GNU sparse member of 576 NUL bytes, on every current mirror | 2026-10-14 | the database on each mirror | +| Arch Linux ARM aarch64 `aur.files` leaves out `localepurge-0.7.3.4-1`, which `aur.db` names | 2026-10-14 | `scripts/plan` on the upstream databases | +| loong64 extra: pacman refuses `compiler-rt18-18.1.8-1`, `x11-ssh-askpass-1.2.4.1-8` and `yt6801-dkms-1.0.28-3` (keys expired) and `python-xarray-2026.07.0-1` (no signature anywhere) | 2026-10-14 | pacman 7.1 in `archlinux:base`, `Architecture = loong64` | +| loong64 extra: 16 packages are signed by a subkey that expires on 2026-12-31 | 2026-10-14 | the signature issuers | +| Arch Linux ARM: one key, `68B3537F39A313B3E574D06777193F152BDBE6A6`, made every signature of the 8 targets | 2026-10-14 | the signature issuers | +| Arch Linux RISC-V: every signature is by `B5971F2C5C10A9A08C60030F786C63F330D7CB92`, which is in `archlinux.gpg` | 2026-10-14 | the signature issuers | +| about 44000 filenames across nine databases, all `[A-Za-z0-9@._+:-]`; the longest is 92 characters | 2026-10-13 | the databases | +| x86_64 extra: 12 filenames have upper case and map to `fn-` tags; 15041 unique tags; the longest tag is 91 characters | 2026-10-14 | `client/pacman-oci --tags` | +| `%PGPSIG%`, base64 decoded, is byte-identical to the detached `.sig` | 2026-10-13 | the x86_64 core database | +| the newest `%BUILDDATE%` of a database is not monotonic: archived core of 10-02 and 10-03 share it, and so do 10-10 and 10-11 | 2026-10-13 | the Arch Linux archive | +| over eleven pairs of consecutive days (10-01 to 10-12), the newer core never had an older package | 2026-10-13 | the Arch Linux archive | +| a superseded file is gone from every Arch mirror within a day, in `os/` and `pool/` | 2026-10-13 | the mirrors | + +## The pipeline + +| fact | date | source | +| --- | --- | --- | +| first sync of x86_64 core: plan 25 s; 4 shards of 74 to 75 files, 53 s to 3 min 10 s; stage 35 s; verify 6.5 min | 2026-10-13 | the first run, on a branch | +| a single push ran at up to 34 MB/s | 2026-10-13 | the first run | +| the four upstream servers gave 75, 75, 75 and 74 files | 2026-10-13 | the first run | +| a sync with nothing new: every file present; shards took 12 to 17 s | 2026-10-13 | the second run | +| Sync run 38490975315: 8 min 40 s in all; plan script 2.7 s; mirror shards 6.56 to 12.80 s; stage 3.25 s; verify 426.3 s for 299 packages and 1888.10 MiB; promote 6.0 s | 2026-10-14 | the run's logs | +| the same verify step took 333 s and 443 s in two other runs | 2026-10-14 | runs 38435488665 and 38440550631 | +| a wrapper call in verify cost at most 0.56 s: 333 s for 598 calls, the image pull included | 2026-10-14 | run 38435488665 | +| verify jobs: loong64-core 82 s for 295 packages and 1.8 GB, its `batch` phase 24 s through the wrapper's prefetch; aarch64-aur 36 s; riscv64-core 67 s; x86_64-multilib 39 s | 2026-10-14 | run 38570044665, on GitHub runners | +| mirror time per file, first seed: 3.19 s plus 0.0328 s per MiB (n=299, `PARALLEL=4`) | 2026-10-14 | the first seed's results | +| `batch_count` on the x86_64 extra seed (15041 files, 115541940123 bytes): it wants 54 batches and grows the limits to 572 files and 3057647619 bytes, for 38 batches | 2026-10-14 | `scripts/lib.sh` `batch_count` | +| the plan artifact of x86_64 core: 2,451,850 bytes; a simulated x86_64 extra seed: 122.4 MB | 2026-10-14 | run 38490975315 and a simulation | +| `history.jsonl`: 1261 bytes per line for 15 targets, about 2.5 MB at 2000 lines | 2026-10-14 | `scripts/status-collect`, live | + +## The wrapper + +| fact | date | source | +| --- | --- | --- | +| 40 small packages through pacman, median of 8 runs, from Nepal: 3.8 s with 8 jobs, 7.8 s with 16, 3.0 s with 24, 2.7 s with 32 | 2026-10-14 | the comment above `PACMAN_OCI_JOBS` in `client/pacman-oci` | +| a connection to the blob host can stall: 150 small blobs on one HTTP/2 connection took 425 s and 440 s, and 4 to 6 s on a new connection | 2026-10-14 | a WSL2 VM in Nepal; the comment above `deadline` in `client/pacman-oci` | +| one 208 MB blob ran at 330 KB/s for 625 s | 2026-10-14 | the same | +| about 1 run in 4 of 40 small blobs sent at a few KB/s for 10 s or more; a normal run takes 1 to 2 s | 2026-10-14 | the same | +| a 3 s transfer limit, not 10 s, took a run of 40 small packages with a stall from 11 to 17 s down to 6.5 to 9 s | 2026-10-14 | the comment above `slice` in `client/pacman-oci` | +| one limit of 3 s plus 1 MiB/s for the whole file: 3 large packages (540 MB) took 238 s, where the official mirror took 40 s | 2026-10-14 | the same | +| version 1, 40 files driven directly: 75.1 s with 1 job, 20.1 s with 4, 17.1 s with 8, 13.8 s with 16 | 2026-10-14 | the Windows host in Nepal | +| the decoded `%PGPSIG%` equalled the served `.sig` for 20 of 20 packages | 2026-10-14 | x86_64 core | +| version 1 against a registry that does not answer: a file failed after 35.35 s (three 10 s connect timeouts) | 2026-10-14 | a blackholed address | +| version 1 with `PACMAN_OCI_NO_DUMMY=1`: 4 requests per file (manifest 401, token, manifest 200, blob), 3.0 s | 2026-10-14 | the Windows host in Nepal | +| version 1, the first promoted core as the only Server, from a residential network: `pacman -Sy` 4.0 s, `pacman -S nano` 5.1 s | 2026-10-14 | a stock Arch pacman | + +## Bench + +Source: `scripts/bench-consumer`, with `client/pacman-oci` version 2. Host: "Ajam (Linux x86_64), in archlinux:base through podman". pacman 7.1.0.r9.g54d9411-2. Date: 2026-10-14T12:38:28Z. + +Each case uses `ParallelDownloads = 5` and `SigLevel = Required DatabaseNever`. `sy` is `pacman -Sy` into an empty database path. `sw-small` is `pacman -Swdd` of the 40 smallest core packages, and `sw-large` of the 3 largest. The file counts include the signatures. Seconds is the median of 3 rounds. + +| path | op | seconds | rounds | files | bytes | ok | +| --- | --- | --- | --- | --- | --- | --- | +| official | sy | 0.727 | 0.539, 0.743, 0.727 | 1 | 130359 | yes | +| official | sw-small | 4.853 | 6.442, 2.639, 4.853 | 80 | 665107 | yes | +| official | sw-large | 55.077 | 55.077, 42.611, 63.287 | 6 | 539912381 | yes | +| worker | sy | 2.238 | 3.502, 1.484, 2.238 | 1 | 130359 | yes | +| worker | sw-small | 10.552 | 14.476, 8.622, 10.552 | 80 | 665107 | yes | +| worker | sw-large | 49.173 | 261.048, 49.173, 35.617 | 6 | 539912381 | no | +| oci | sy | 1.615 | 2.016, 1.615, 1.596 | 1 | 130359 | yes | +| oci | sw-small | 2.735 | 2.735, 6.756, 2.592 | 80 | 665107 | yes | +| oci | sw-large | 72.612 | 72.612, 77.378, 38.745 | 6 | 539912381 | yes | +| oci-serial | sy | 1.574 | 2.056, 1.153, 1.574 | 1 | 130359 | yes | +| oci-serial | sw-small | 126.064 | 142.665, 121.073, 126.064 | 80 | 665107 | yes | +| oci-serial | sw-large | 348.617 | 71.521, 860.701, 348.617 | 6 | 539912381 | yes | + +`official` is `geo.mirror.pkgbuild.com` with pacman's downloader. `worker` is the Worker with pacman's downloader. `oci` is the wrapper with its prefetch, and `oci-serial` the wrapper with `PACMAN_OCI_PREFETCH=0`. `ok` is no when a round failed or downloaded another number of files. + +## Tools and images + +| fact | date | source | +| --- | --- | --- | +| oras 1.3.4: the sha256 of each archive, from the release's `checksums.txt` | 2026-10-13 | `scripts/install-oras` | +| ShellCheck 0.11.0, actionlint 1.7.12, jq 1.8.2: release checksum lists and GitHub asset digests agreed | 2026-10-13 | `scripts/install-test-tools` | +| curl 8.22.0 (stunnel/static-curl), musl static: the GitHub asset digest, the release notes and `SHA256SUMS` agreed | 2026-10-14 | `scripts/install-test-tools` | +| bash 5.2.21-2ubuntu4 (Ubuntu noble `bash-static`, release pocket): the Packages index and Launchpad agreed | 2026-10-14 | `scripts/install-test-tools` | +| `docker.io/library/node:24-alpine` is `lts-alpine`, digest `sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1`, node v24.21.0 | 2026-10-14 | Docker Hub | +| `node --test worker/test/` fails on node 26.7.0; the quoted glob works on node 22, 24 and 26 | 2026-10-14 | node in containers | +| the integration suite `10-pipeline.sh`: 98 of 98 in 314 to 490 s, in a 10-vCPU WSL VM with podman | 2026-10-14 | `tests/run.sh integration` | +| the static suite: about 40 s in a container | 2026-10-14 | `tests/run.sh static` | diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..b109de3 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,190 @@ +# Operations + +This document tells what runs by itself, which issues the automation raises and clears, and what a person must do. + +## Workflows and schedules + +| workflow | triggers | does | +| --- | --- | --- | +| `sync.yml` | cron `17 */3 * * *`, dispatch | plan, one chain per pending target (`sync-target.yml`), report, keepalive | +| `sync-target.yml` | called by `sync.yml` | mirror, stage, verify, promote (on `main`), summary, for one target | +| `status.yml` | cron `47 * * * *`, dispatch, and `report` after each sync on `main` | measures the mirror and the Worker, renders the site, pushes branch `status`, deploys Pages, raises or clears `worker-down` | +| `watchdog.yml` | cron `41 7 * * *`, dispatch | staleness and schedules | +| `ci.yml` | push to `main`, pull request (not to `status`), dispatch | the test suites | +| `worker.yml` | push to `main` that changes `worker/**` or the workflow, dispatch | tests and deploys the Worker | + +### The jobs of the watchdog + +| job | does | +| --- | --- | +| `staleness` | `scripts/check-staleness`; raises or clears `stale-` | +| `schedules` | `scripts/keep-schedules-enabled`, `scripts/check-schedules-fired`, and on `main` `dispatch-overdue` for sync and status | + +### The jobs of the status workflow + +| job | does | +| --- | --- | +| `kick` | on `main`: `dispatch-overdue sync.yml 240` and `dispatch-overdue watchdog.yml 1560` | +| `decide` | a bench on dispatch with `bench=true`; else a bench on the first run from 05:00 to 08:59 UTC with no bench of the day on branch `status` | +| `bench` | `scripts/bench-consumer`, when `decide` says yes | +| `build` | `scripts/status-collect`, then `scripts/status-render`; on `main` it pushes branch `status` and uploads the Pages artifact | +| `deploy` | on `main`, when `build` passed and Pages takes a workflow build: `actions/deploy-pages` | +| `worker` | on `main`, when `build` passed: reads `status.json` from branch `status`; raises or clears `worker-down` | + +`build` runs when `decide` passed, also when `bench` was skipped or failed. `deploy` and `worker` name the result of `build` in their `if`. An `if` with no status function gets an implicit `success()` over every job before it. `bench` is skipped in most runs, so that implicit check would skip `deploy` and `worker` too. + +The Worker check is a job of `status.yml`, not of `watchdog.yml`. An hourly cron in `watchdog.yml` would cancel a pending daily run, because both share one concurrency group. + +## Each scheduled workflow starts the others + +⛔ GitHub can delay or drop a scheduled run. A dropped run leaves nothing behind that could notice. So each scheduled workflow runs `scripts/dispatch-overdue WORKFLOW MAX_MINUTES` for the others. + +| runs in | starts | when its newest run on `main` is older than | +| --- | --- | --- | +| `sync.yml` job `keepalive` | `status.yml` | 120 minutes | +| `status.yml` job `kick` | `sync.yml` | 240 minutes | +| `status.yml` job `kick` | `watchdog.yml` | 1560 minutes | +| `watchdog.yml` job `schedules` | `sync.yml` | 240 minutes | +| `watchdog.yml` job `schedules` | `status.yml` | 120 minutes | + +A run that is queued or in progress counts as fresh, so nothing starts twice. A workflow with no run at all is started. Runs start on `main` only. + +`dispatch-overdue` starts only a workflow whose state is `active`. GitHub refuses to start a disabled workflow. The script skips a workflow that a person disabled, or that GitHub disabled for inactivity. It prints a notice and exits 0. In `sync.yml` and `watchdog.yml`, `keep-schedules-enabled` runs first, and enables again a workflow that GitHub disabled for inactivity. + +The restart steps in `sync.yml` and `watchdog.yml` run with `always()`. So a schedule check that fails earlier in the job does not stop the restart. A step with two `dispatch-overdue` calls runs both, then exits with the status of the last call that failed. + +`scripts/keep-schedules-enabled` runs in `sync.yml` and in `watchdog.yml`. GitHub disables the schedules of a repository after 60 days with no activity. The script enables each workflow of the tree again. ⚠ A workflow that a person disabled stays disabled, and the script names it. + +`scripts/check-schedules-fired` checks each scheduled workflow against its own cron. It fails when a schedule has not fired within its tolerance. It runs in `sync.yml` and in `watchdog.yml`, so each one sees the other stop. + +⚠ Neither check sees every schedule stop at once. Then no run is left to notice. Start `sync.yml` or `watchdog.yml` by hand to get the answer. + +## Staleness + +`scripts/check-staleness` reads the database that `latest` serves, through the wrapper's token flow (`PACMAN_OCI_NO_DUMMY=1`). So that path runs against GHCR on schedule too. It also reads upstream's current database from the target's `db_server` hosts. + +| verdict | when | +| --- | --- | +| `current` | the two databases are the same | +| `fresh` | they differ, and the promoted snapshot was fetched within the tolerance | +| `new` | there is no `latest`, and git added the target file within the tolerance | +| `stale` | they differ for longer than the tolerance, or there is no `latest` for longer | +| `unknown` | upstream could not be read | + +The tolerance comes from `scripts/cron-tolerance`: the largest gap in days of the cron, times 2, plus 1. The 3-hour cron of `sync.yml` gives a gap of 1 day and a tolerance of 3 days. ⚠ The tolerance is in days. A sub-daily cron gets the tolerance of a daily one, so a stop shows later than it could. + +The date a target file was added is the date of the commit that brought it to the checked-out branch (`git log --first-parent`). A commit on the branch it came from does not count. + +⚠ When git cannot tell when a target file was added (no git, a shallow clone, an untracked file), the target is stale, never new. The `staleness` job checks out the whole history for this. + +## Issues the automation raises and clears + +`scripts/track-issue` keeps one issue per problem. `raise` opens the issue the first time and only rewrites its body after that. `clear` comments, closes, and does nothing when no issue is open. The key is in the body as an HTML comment. The script finds an issue by listing the open issues, never by search. + +| issue | raised when | cleared when | by | +| --- | --- | --- | --- | +| `visibility-` | the package reads with the workflow's token and not without one | a run reads the target anonymously | `scripts/report`, on `main` | +| `replaced-` | a package tag holds other bytes than the database names | the target stages or is current again | `scripts/report`, on `main` | +| `stale-` | `check-staleness` says `stale` or `unknown` | it says anything else | `watchdog.yml` job `staleness` | +| `worker-down` | the Worker failed its probe in 2 status runs in a row | a status run records a probe that passes | `status.yml` job `worker` | + +A target that is held, behind or in error raises nothing in the report. A run that fails once is ordinary. `stale-` opens when the mirror has stopped following upstream. + +The `worker` job of `status.yml` reads the record that `build` pushed to branch `status` in the same run. The job fails when that record is more than 3 hours old. + +## What needs a person + +- **A private package.** GitHub may create a package private, and has no API to change that. Open the settings link in `visibility-`, and make the package public. +- **A replaced file.** Arch does not publish a filename again with new bytes. So `replaced-` means something unusual upstream or on the registry. The issue names the file and both digests. Wait for upstream to publish a new filename, or accept the new bytes under a new tag. +- **Renovate.** Merge the keyring image updates and the major updates. +- **Pages.** Enable Pages once (below). +- **The Worker.** Set `CLOUDFLARE_API_TOKEN` to deploy it. +- **A disabled workflow.** Enable a workflow that a person disabled, when that decision no longer holds. + +Start a sync by hand: + +```bash +gh workflow run sync.yml --ref main +gh workflow run sync.yml --ref main -f targets='x86_64-core x86_64-extra' -f max_targets=2 +``` + +## Renovate + +`.github/renovate.json` sets the update policy. + +- Actions are pinned by commit, with the version in a comment. Images are pinned by digest, under a `# tag: ` line. +- Actions move once a week (`* 0-5 * * 1`), in one pull request, only to a release at least 3 days old. +- Digest, pin, pinDigest, patch and minor updates of actions and images merge on their own, as a squash, when every check is green. +- `platformAutomerge` is `false`. `main` has no required checks, so GitHub's own auto-merge would merge a pull request with red checks. Renovate reads the checks itself. +- ⚠ A major update waits for a person. +- ⚠ The keyring images (`ghcr.io/pkgforge-dev/archlinux`) wait for a person. They are the trust root of the aarch64, armv7h, loong64 and riscv64 targets, and no CI job runs them. +- The 15 `verify_image` lines and the Arch image of the tests move in one pull request. So CI runs pacman in the image the targets will verify with, before it merges. +- The node image of `worker.yml` and `ci.yml` moves in one pull request. +- `ignorePaths` is `["**/node_modules/**"]`. It replaces the preset's list, which ignores `**/tests/**`, so the test images are updated too. + +`tests/static/75-renovate.sh` checks these rules and checks that the regex manager matches every image pin. + +## Repository settings + +| setting | value | +| --- | --- | +| default workflow permissions | read | +| Actions can approve pull requests | no | +| `sha_pinning_required` | `true`: only actions pinned by commit run | + +`tests/static/50-workflow-policy.sh` checks the shape of every workflow. It checks `permissions: {}` at the top, the permissions of each job, and `timeout-minutes` on every job. It also checks that each action is pinned by commit, and that no untrusted value is inside a `run:` block. + +## Secrets + +| secret | needed by | without it | +| --- | --- | --- | +| `CLOUDFLARE_API_TOKEN` | `worker.yml` job `deploy` | the job shows a notice and ends with success, with no deploy | + +Every other job uses the workflow's own `GITHUB_TOKEN`. + +## Pages + +The site is published with a workflow build: `actions/upload-pages-artifact` in job `build`, then `actions/deploy-pages` in job `deploy`. Only `deploy` has `pages: write` and `id-token: write`. + +⚠ A person must enable Pages once, with an admin token. `GITHUB_TOKEN` cannot do it: + +```bash +gh api -X POST repos/pkgforge-dev/mirror-archlinux/pages -f build_type=workflow +``` + +Until then, `deploy` shows a notice and deploys nothing. + +The site at holds: + +| path | contents | +| --- | --- | +| `index.html` | the dashboard: one static page, no JavaScript, light and dark | +| `badges/.svg` | coverage and snapshot age of one target | +| `badges/.svg` | the same over the targets of one architecture | +| `badges/all.svg` | the same over every target | +| `badges/speed.svg` | pacman's time through the wrapper and through the official mirror, from the bench | +| `charts/coverage.svg` | share of upstream files in the mirror, per target | +| `charts/freshness.svg` | snapshot age per target, against the sync interval | +| `charts/size.svg` | GiB per target, in the mirror and upstream | +| `charts/history-coverage.svg` | coverage per architecture, last 30 days | +| `charts/history-age.svg` | oldest snapshot age per architecture, last 30 days | +| `charts/speed.svg` | the bench: seconds per path and operation | +| `charts/narrow/.svg` | each chart at 420 px, for a phone | +| `status.json`, `history.json`, `bench.json` | the data behind the page | + +`status.json` has schema 1. The header of `scripts/status-collect` describes each field. + +## The status branch + +On `main`, job `build` pushes branch `status` as one orphan commit, with force, on each run. The branch holds the site, plus: + +| file | contents | +| --- | --- | +| `history.jsonl` | one line per status run; the last 2000 lines are kept | +| `worker.md` | the body of the `worker-down` issue: every probe, and how to reproduce the first failure | +| `bench.json` | the newest bench | +| `bench/.json` | each daily bench | + +The next status run reads `history.jsonl` and the bench from the branch. The `worker` job of `status.yml` reads `status.json` and `worker.md` from it. ⚠ The branch is the only copy of the history. A run that fails to push leaves the previous branch in place, so no history is lost. + +The pushes to `status` start no CI run: the push trigger of `ci.yml` covers `main` only, and its pull request trigger leaves out `status`. diff --git a/docs/tests.md b/docs/tests.md new file mode 100644 index 0000000..a3a650e --- /dev/null +++ b/docs/tests.md @@ -0,0 +1,334 @@ +# Tests + +## The suites + +| suite | files | needs | +| --- | --- | --- | +| `static` | `tests/static/*.sh`: shellcheck, actionlint, py_compile and the policy checks | `shellcheck`, `actionlint`, `jq`; `python3` when there is python to compile; `git` for the file modes | +| `unit` | `tests/unit/*.sh`: the wrapper and the scripts on fixed inputs, with no network | `curl`, `jq`; GNU tar for the database cases; `python3` and `git` for parts of `60-status.sh` | +| `integration` | `tests/integration/*.sh`: a local registry and pacman in containers, oras on the host | docker or podman (`CONTAINER_RUNTIME`), curl, jq, tar, sha256sum, and network to pull the pinned images and oras | +| `worker` | `tests/worker/*.sh`: the Worker under `node --test` | node 22 or newer, and `sh` | + +| file | checks | +| --- | --- | +| `static/10-shellcheck.sh` | every shell file outside `tests/fixtures/` passes shellcheck in the dialect of its first line | +| `static/20-actionlint.sh` | every workflow passes actionlint | +| `static/30-failures-stay-visible.sh` | no shipped script or workflow hides a failure; every curl call has a deadline, and so does every oras call in `scripts/` | +| `static/35-pipeline-traps.sh` | no pipeline in a shipped bash script ends in a reader that stops early; `client/pacman-oci` runs with no `pipefail` and is left out | +| `static/40-executable-bits.sh` | a shipped file with a `#!` line is 100755 in the git index; one without is 100644 | +| `static/50-workflow-policy.sh` | the shape of every workflow: permissions, timeouts, pins, no untrusted value in `run:` | +| `static/60-targets.sh` | every target file parses and pins its images; one verify image; a plan slot for each target | +| `static/70-python.sh` | every python file compiles; every file in `client/` and `scripts/` is shell or python | +| `static/72-test-runner.sh` | `tests/run.sh` finds the files, runs one file, gates a live test, and fails a failed run | +| `static/75-renovate.sh` | Renovate can move every image pin, and merges only what CI can judge | +| `unit/10-tag-encoding.sh` | every package file maps to one tag, and no two files share one | +| `unit/20-json-parser.sh` | the JSON parser reads every legal shape and refuses anything else | +| `unit/30-wrapper-urls.sh` | one manifest request per URL; a URL it cannot read is refused before the network | +| `unit/31-run-store.sh` | the run store: key, ownership, sweep, remover, host blacklist | +| `unit/32-prefetch-plan.sh` | the argument rewrite, the print lines, the signatures written from `%g` | +| `unit/40-lib.sh` | the decisions of the sync on fixed inputs | +| `unit/45-dispatch-overdue.sh` | `scripts/dispatch-overdue` starts a workflow only when it is active and overdue | +| `unit/60-status.sh` | `status-collect` and `status-render` on fixtures | +| `integration/10-pipeline.sh` | the whole mirror against its own registry and upstream | +| `integration/20-client.sh` | the wrapper against its own registry, through faulty proxies | +| `worker/10-node.sh` | `node --test 'worker/test/*.test.js'` | +| `worker/live.sh` | a smoke test of the deployed Worker; only with `WORKER_LIVE=1` | + +## Run the suites + +Install the pinned tools first. Each one is a static binary for x86_64 or aarch64 Linux, checked by sha256. + +```bash +scripts/install-test-tools /tmp/mirror-archlinux-tools +export PATH="/tmp/mirror-archlinux-tools:$PATH" +``` + +⚠ `install-test-tools` needs `curl`, `sha256sum` and `tar` on the host, `xz` to unpack shellcheck and curl, and `zstd` and `dd` to unpack bash. + +Then run a suite, one file of a suite, or all four: + +```bash +tests/run.sh static +tests/run.sh unit +CONTAINER_RUNTIME=podman tests/run.sh integration +tests/run.sh worker +tests/run.sh integration 20-client +tests/run.sh all +``` + +| form | does | +| --- | --- | +| `tests/run.sh` | the unit suite | +| `tests/run.sh SUITE` | every file of SUITE, in name order | +| `tests/run.sh SUITE FILE` | one file, by name (`20-client.sh` or `20-client`) or by path | +| `tests/run.sh all` | the four suites in turn; takes no FILE | +| `tests/run.sh --list SUITE` | prints the name of each file SUITE runs, one per line | + +The exit status is 0 when every file passed, 1 when a file failed or no file ran, and 2 for usage. The summary names each failed file with the command that runs it again. + +`ci.yml` makes its integration matrix from `tests/run.sh --list integration`. A new file gets its own job with no edit. + +### Live tests + +⚠ A file named `live.sh` talks to a deployed service. It runs only when `_LIVE=1`, also when the file is named. For `tests/worker/live.sh` the switch is `WORKER_LIVE=1`: + +```bash +WORKER_LIVE=1 tests/run.sh worker live +``` + +### Tools that may be absent + +A test that needs a tool can call `need_tool NAME DESC` (`tests/lib/harness.sh`). When the tool is absent, the test is skipped with a TAP `# SKIP` line. When `TESTS_REQUIRE_TOOLS` names the tool, the test fails instead. + +```bash +TESTS_REQUIRE_TOOLS='shellcheck actionlint jq python3' tests/run.sh static +``` + +`50-workflow-policy.sh`, `70-python.sh`, `75-renovate.sh` and `45-dispatch-overdue.sh` use `need_tool`. The static, unit and worker jobs of CI set `TESTS_REQUIRE_TOOLS`. The integration job does not. + +⚠ Other tests handle an absent tool in their own way: + +- `10-shellcheck.sh`, `20-actionlint.sh` and `worker/10-node.sh` fail when their tool is absent. +- Parts of `30-wrapper-urls.sh` and `31-run-store.sh` (dash), `40-lib.sh` and `60-status.sh` (GNU tar, git, python3) and `40-executable-bits.sh` (git) print a plain `ok` line that says they skipped. `TESTS_REQUIRE_TOOLS` does not turn these into failures. +- So the `bash-gawk` and `busybox` unit legs pass with parts of `60-status.sh` skipped. + +### Integration options + +| variable | effect | +| --- | --- | +| `CONTAINER_RUNTIME` | `docker` or `podman`; else `docker` when it is on PATH, then `podman` | +| `IT_ORAS_CACHE` | a directory that keeps the oras download between runs of `10-pipeline.sh` | +| `IT_KEEP` | `1` keeps the work directory, and prints its path | + +Each run picks its ports at run time, and each container name carries the shell's pid and a random number. So several runs can share one host. + +### In a container + +The unit and static suites run in the pinned `archlinux:base`. Mount the tools last on PATH, so the image's own tools stay the ones under test: + +```bash +docker run --rm -v "$PWD:/work:ro" -v /tmp/mirror-archlinux-tools:/tools:ro -w /work \ + docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 \ + sh -c 'PATH="$PATH:/tools" exec bash tests/run.sh unit' +``` + +⚠ `40-executable-bits.sh` reads the git index. In a copy with no git index, it passes with one line that says it skipped. + +## CI + +| job | runs | +| --- | --- | +| `static` | `tests/static/72-test-runner.sh` with bash first, then `tests/run.sh static`; `TESTS_REQUIRE_TOOLS='shellcheck actionlint jq python3'` | +| `unit (dash-mawk)` | `tests/run.sh unit` on the runner, with dash and mawk; requires `jq curl python3` | +| `unit (bash-gawk)` | the same in the pinned `archlinux:base`; requires `jq curl` | +| `unit (busybox)` | the same in the pinned `alpine:3.22`; requires `jq curl` | +| `integration` | one job per file of `tests/run.sh --list integration` | +| `worker` | `tests/run.sh worker` in the pinned `node:24-alpine` | +| `debloat` | `scripts/debloat-runner` on a real runner; a warning when it frees less than 1 GiB | + +⛔ No CI job installs a package with a package manager. `scripts/install-test-tools` downloads pinned static binaries and checks each sha256. Its bash comes from Ubuntu's `bash-static` package, checked by sha256 and unpacked. Each unit leg checks that it runs the shell and the awk it is named for. + +`72-test-runner.sh` runs first, on its own: a `run.sh` that hides failures would also hide this file's failure. + +## Output + +The output is TAP. Each file is one plan. A failed assertion prints what was measured and the command that reproduces it. A file with no assertion fails. + +## Seen to fail + +⛔ A test that never failed is not shown to test anything. So each test is seen to fail once: + +1. Copy the repository. +2. Change one exact piece of text in one file of the copy. +3. Run the one test file, with `REPO_ROOT` set to the copy. +4. Record the assertion that failed. + +The tree itself is never changed. An integration fault runs with its own registry and upstream ports. + +### client/pacman-oci + +| test | fault | observed | +| --- | --- | --- | +| unit/10-tag-encoding | a literal `_` before `p`, `c`, `a` or `u` is no longer escaped | not ok: `tag of a_pb-1-1-any.pkg.tar.zst` | +| unit/10-tag-encoding | upper case kept in tags, which GHCR folds together | not ok: `a name outside the mapping hashes: lshw-B.02.20-3-x86_64.pkg.tar.zst` | +| unit/10-tag-encoding | the 128-character tag limit off by one | not ok: `a name of 129 characters is hashed` | +| unit/20-json-parser | an escaped newline or tab accepted in a JSON string | not ok: `refused: an escaped newline, which could forge an output line` | +| unit/20-json-parser | data after the JSON document ignored | not ok: `refused: trailing data` | +| unit/30-wrapper-urls | an uncompressed `.pkg.tar` read as a database | not ok: `an uncompressed package` | +| unit/30-wrapper-urls | a database signature read from the moving tag | not ok: `core.db.sig is read by the content of the core.db beside it` | +| unit/30-wrapper-urls | a failed attempt leaves its `.part` behind | not ok: `a failed attempt removes the .part it was given` | +| integration/10-pipeline | the blob's sha256 no longer checked | not ok: `the wrapper refuses a blob whose sha256 is not its descriptor's` | +| integration/20-client | no host is marked down; no transfer resumes; the background remover removes nothing | 12 not ok, among them `dead: a registry that answers 503: the second file makes no request`, `resume: the cut package was resumed with a Range request` and `main-root: no store file is left once pacman has ended` | +| integration/20-client | the prefetch keeps a whole file with no size and sha256 check | 7 not ok, among them `flip: one byte changed in each first answer: the prefetch keeps neither database` | +| integration/20-client | the batch check reads the `sha256sum` output one column off | 16 not ok, among them `main-du: the serial pass makes no network request` | +| integration/20-client | a transfer cut by its time limit never goes on in the next pass | not ok: `stall: the prefetch cut the stopped transfer and resumed it from 64 KiB` | +| integration/20-client | the prefetch makes one pass only | not ok: `resume: every blob cut in half: the prefetch resumed it, the serial pass made no request` | + +### scripts + +| test | fault | observed | +| --- | --- | --- | +| unit/40-lib | `decide_push` pushes over a tag that holds other bytes | not ok: `write-once: other bytes under this name are never overwritten` | +| integration/10-pipeline | `mirror-file` pushes over a tag that holds other bytes | not ok: `a file whose bytes changed under its own name holds the target` | +| unit/40-lib | older and newer swapped in the behind rule | not ok: `core of 10-03 is not behind 10-02` | +| integration/10-pipeline | plan takes a database behind the promoted one | not ok: `an older upstream database is refused as behind` | +| unit/40-lib | a `/` allowed in a filename from a database | not ok: `db field check refuses a / alone, after a valid prefix` | +| unit/40-lib | staleness waits ten times too long | not ok: `stale: different, fetched longer ago than the tolerance` | +| unit/40-lib | an earlier snapshot may be promoted over a later one | not ok: `a later snapshot may be promoted` | +| integration/10-pipeline | promote no longer refuses an older snapshot | not ok: `promoting an older run's snapshot again is refused` | +| unit/40-lib | an item that left no result counts as pushed | not ok: `an item that left no result is counted as failed` | +| unit/40-lib | an unknown key in a target file ignored | not ok: `refused: an unknown key` | +| integration/10-pipeline | a lagging db_server that only dropped a package wins | not ok: `a lagging db_server that only drops a package does not move latest` | +| integration/10-pipeline | with no `%PGPSIG%`, the detached `.sig` is never fetched | not ok: `a database without %PGPSIG% stages` | +| integration/10-pipeline | the signature layer is not the `%PGPSIG%` (the decode loses its last base64 group) | not ok: `alpha-1:1.0+r2-1-any.pkg.tar.zst signature layer is its %PGPSIG%, decoded` | +| unit/40-lib | the keyring name check accepts anything | not ok: `refused: a keyring name with a leading -`, `refused: a keyring name with a /`, `refused: a keyring name that is a glob` | +| unit/40-lib | `keyring_image` accepted with no `keyring_platform` | not ok: `refused: keyring_image with no keyring_platform` | +| unit/40-lib | `keyring_from` may name another arch | not ok: `refused: keyring_from naming a target of another arch` | +| unit/40-lib | `db_check` accepts an unreadable member that is not NUL bytes | not ok: `db_check refuses an unreadable member that holds other bytes than NUL, and names it` | +| unit/40-lib | `db_check` refuses every unreadable member | not ok: `db_check accepts package directories whose desc is NUL bytes or empty, and names them` | +| unit/40-lib | `db_check` no longer compares records and members | not ok: `db_check refuses a desc that holds two records` | +| unit/40-lib | the `MAX_SHARDS` clamp removed | not ok: `shards: x86_64 extra's seed wants 216 by bytes and gets MAX_SHARDS` | +| unit/40-lib | batches do not grow to fit `VERIFY_MAX_BATCHES` | not ok: `batches: x86_64 extra's seed wants 54, both limits grow, and it gets 36 to 40` | +| unit/40-lib | the build date written with no `Z` | not ok: `a build date as ISO 8601 UTC` | +| unit/40-lib | the "required key missing from keyring" pattern removed | not ok: `refusal: a primary key that expired is a signature reason` | +| unit/40-lib | the checksum pattern removed | not ok: `refusal: a checksum is not a signature reason` | +| unit/40-lib | the `QQ==` read turned off | not ok: the three `registry:` assertions on the dummy token's 200 and 404 | +| unit/40-lib | `registry_token_file` returns nothing | not ok: `registry: a second process reads the token the first one cached`, and the two after it | +| unit/40-lib | a refused cached token is not asked for again | not ok: `registry: a cached token the registry refuses is asked for again, once`, `registry: the cache then holds the new token` | +| integration/10-pipeline | the `MAX_SHARDS` clamp removed | not ok: `MAX_SHARDS caps a target's shards`, `the plan logs that the shard cap binds` | +| integration/10-pipeline | `shard_count` always 1 | not ok: each target gets shards of its own | +| integration/10-pipeline | the `dev.pkgforge.mirror.upstream` annotation added again | not ok: the package manifest names no upstream server; two writers push byte-identical manifests | +| integration/10-pipeline | `Architecture = auto` | not ok: `verify writes Architecture from the target`; the aarch64 batch fails in the x86_64 image | +| integration/10-pipeline | `db_check` refuses unreadable members | not ok: `a database with a NUL-only package directory is planned`, `the NUL-only package directory is named`, `the plan summary names it` | +| integration/10-pipeline | `upstream_refused` always says no | not ok: upstream-refused packages listed, verdict passed, promoted, in the report | +| integration/10-pipeline | the report's refused list turned off | not ok: the report lists the upstream-refused packages | +| integration/10-pipeline | the trust guard turned off | not ok: `a trust root that accepts nothing excuses nothing` | +| integration/10-pipeline | `keyring_repo` emptied after `keyring_from`'s `latest` is read | not ok: `verify installs the keyring package from keyring_from's repository` | +| integration/10-pipeline | plan's `%PGPSIG%` condition reverted | not ok: a file signed again with the same bytes is planned, its layer repaired, promoted | +| integration/10-pipeline | the signature comparison turned off | not ok: `a signature layer that is not the %PGPSIG% fails verify, and is named`, `a batch with a bad signature layer is never promoted` | +| integration/10-pipeline | every `sig-refused` package excused | not ok: with no `%PGPSIG%`, a layer that is not upstream's `.sig` is not excused | +| integration/10-pipeline | `build-repo` signs on every build | not ok: an update is planned as new files only; `latest` does not move backwards | +| static/60-targets | five edits in a copy: a `docker.io` keyring image, a removed `# tag: aarch64`, another verify digest, renamed plan slots, a `keyring_from` that does not exist | one not ok for each edit, among them `every target verifies in the one image, on linux/amd64` | + +### status + +| test | fault | observed | +| --- | --- | --- | +| unit/60-status | `cron-tolerance --minutes` leaves out the gap from the last firing to the first | not ok: `interval: only on 29 February is 1461 days, 2103840 minutes` | +| unit/60-status | coverage keyed by file name only | 17 not ok, among them `x86_64-extra: a file counts only with the same name and sha256: 3 of 5` and the badge texts | +| unit/60-status | `failed_runs` always 1 | not ok: `worker: two failed runs in a row, failing since the first` | +| unit/60-status | the Location check accepts `http://` | not ok: `worker: a 302 to http is not a redirect to https://*.githubusercontent.com` | +| unit/60-status | the history keeps every line | not ok: `history keeps the last 2000 lines`, `the oldest lines go first and the new one is last` | +| unit/60-status | a background `` left unclosed | 18 not ok, among them `site: all 22 SVG files parse as XML, each with a title and its own dark style` | +| unit/60-status | the badge message left empty | 30 not ok: each `: badge carries its text: ` | +| unit/60-status | the narrow `` left out of the page | not ok: `index.html references every chart and badge`, in each of the three sites | +| unit/60-status | `bucket()` keeps `None` points with no bucketing | not ok: `status-render site3 exits 0` | +| unit/60-status | the grace limit 30 days, not the tolerance | not ok: `check-staleness: no latest, added 13 days ago, tolerance 3: stale`, with the exit status and the added date | +| unit/60-status | the shallow-clone check never true | not ok: `check-staleness: in a shallow clone the age is unknown, so stale, never new`, `check-staleness: says that the clone is shallow` | +| unit/60-status | `read_json` returns a stub status | not ok: `status-render with no status.json exits 1`, `and names the file it looked for` | + +### worker + +| test | fault | observed | +| --- | --- | --- | +| worker/test (cache) | cache keys on `https://cache.invalid` | not ok: `every cache key is on the request host, under /__cache/v1` | +| worker/test (cache) | 404 answers not stored under the `loc` key | not ok: `a repeat request is one cache lookup and no subrequest` | +| worker/test (cache) | no 600 s cap on the signed URL time to live | not ok: TTL 880 where 600 is expected | +| worker/test (cache) | a URL with no `se` stored for 600 s | not ok: `a signed URL with no se is not stored` | +| worker/test (cache) | the package-with-sig time to live 86400 | not ok: manifest TTLs | +| worker/test (cache) | the layer count check uses `>= 1` | not ok: `a package manifest with a second layer of the file name is not kept 6 h` | +| worker/test (cache) | the `loc` entry ignores the manifest entry | not ok: `a latest redirect never outlives the latest manifest entry, so a new latest shows within 30 s` | +| worker/test (cache) | no `ctx.waitUntil(prewarmSig)` | not ok: `a package answer fills the .sig entry: the .sig is then one lookup and no subrequest` | +| worker/test (cache) | the prewarm starts after the package answer | not ok: `a package request asks for the .sig blob at the same time as the package blob` | +| worker/test (cache) | `cache.match` exceptions not caught | not ok: `a cache that fails is logged, and the request still gets its answer from GHCR` | +| worker/test (ghcr) | no `signal` on fetch | not ok: `every subrequest has a 10 s deadline, the anonymous token and no redirect followed` | +| worker/test (ghcr) | a blob 200 returned as is | not ok: `a blob that GHCR answers with 200 gets 502: the Worker never sends the bytes` | +| worker/test (ghcr) | the Location host check removed | not ok: `the Location must be https and its host must end in .githubusercontent.com` | +| worker/test (ghcr) | the `@sha256` pin not compared | not ok: `an @sha256 pin whose manifest hashes to another value gets 502 and is not stored` | +| worker/test (ghcr) | the 64 KiB byte cap removed | not ok: `a manifest over 64 KiB with no or a false content-length is cut at 64 KiB` | +| worker/test (paths) | `TAG_RE` accepts every tag | not ok: `any other tag gets 404 with no subrequest, staging included` | +| worker/test (paths) | no `decodeURIComponent` | not ok: `a percent-encoded file name is decoded before its tag is made` | +| worker/test (paths) | HEAD answered with the GET body | not ok: `HEAD gets the same answer as GET, with no body` | +| worker/test (tags) | the `_u` escape only for `p`, `c`, `a` | not ok: `tagOf matches client/pacman-oci --tags for every name in tests/fixtures/filenames.txt` | +| worker/test (config) | `head_sampling_rate = 0.01` | not ok: `wrangler.toml sets the compatibility date, the flags, the CPU limit and the logs` | +| worker/10-node | a node shim that reports v20.18.0 | not ok: `node is 22 or newer` | +| worker/live | run against the deployed version 1 | 3 of 28 not ok: `:staging` gets 302, `%2B` gets 404 | + +63 faults, each applied alone to a copy of `worker/`, were each caught. Every one of the 51 node tests was seen to fail at least once. + +### policy and tools + +| test | fault | observed | +| --- | --- | --- | +| static/10-shellcheck | an unquoted expansion in `scripts/plan` | not ok: `shellcheck -s bash scripts/plan` | +| static/10-shellcheck | `script_kind` returns `other` for bash | not ok: `the walk finds tests/run.sh` | +| static/20-actionlint | a job reads an output that does not exist | not ok: `actionlint .github/workflows/sync.yml` | +| static/30-failures-stay-visible | a failure swallowed with `\|\| true` | not ok: `scripts/stage hides no failure` | +| static/30-failures-stay-visible | a curl call with no deadline | not ok: `scripts/install-oras: every curl has a connect deadline and a transfer or stall deadline` | +| static/30-failures-stay-visible | an oras call with no deadline | not ok: `scripts/promote: every oras call has a deadline` | +| static/35-pipeline-traps | a pipeline that ends in a reader that stops early | not ok: `scripts/install-oras: no pipeline ends in a reader that stops early` | +| static/40-executable-bits | a script committed without its executable bit | not ok: `scripts/plan is mode 100755 in the index` | +| static/40-executable-bits | a `#!` line added to `scripts/lib.sh` | not ok: `scripts/lib.sh has no #! line, so it stays 100644` | +| static/50-workflow-policy | a job with no `timeout-minutes` | not ok: `.github/workflows/watchdog.yml: every job has timeout-minutes` | +| static/50-workflow-policy | a dispatch input expanded inside a `run:` block | not ok: `.github/workflows/sync.yml: no untrusted value is expanded inside a run: block` | +| static/50-workflow-policy | an action pinned by tag | not ok: `.github/workflows/ci.yml: every action is pinned by commit` | +| static/50-workflow-policy | a call to the packages DELETE API | not ok: `no script or workflow deletes a package or a version` | +| static/50-workflow-policy | `pages: write` in `ci.yml` job `static`; `contents: write` in `sync.yml` job `plan`; `id-token: write` in `watchdog.yml` job `staleness` | not ok: `pages, id-token and contents write only where the status site needs them`, once for each workflow | +| static/50-workflow-policy | `keep-schedules-enabled` skips `watchdog.yml` | not ok: `.github/workflows/watchdog.yml has a schedule, and keep-schedules-enabled enables it` | +| static/60-targets | a verify image pinned without a tag | not ok: `targets/x86_64-core.conf pins its verify_image as name:tag@sha256` | +| static/70-python | a `#!python3` file with a syntax error | not ok: `python3 -I -m py_compile scripts/status-render` | +| static/70-python | a data file `scripts/notes.txt` | not ok: `scripts/notes.txt is a shell or a python script` | +| static/70-python | no python3, with `TESTS_REQUIRE_TOOLS=python3` | not ok: `python3 is on PATH, to compile the python in the tree`; without the variable, `# SKIP python3 is not on PATH` | +| static/72-test-runner | `gated()` true for every file | not ok: every `--list` and run assertion; the list is empty and nothing runs | +| static/72-test-runner | the live gate removed | not ok: `--list worker leaves out live.sh while WORKER_LIVE is not 1`, `worker does not run live.sh while WORKER_LIVE is not 1`, `a named live.sh does not run either` | +| static/72-test-runner | the failed-file count never grows | not ok: `a failing file fails the run` | +| static/75-renovate | the `# tag: base` line removed from a target | not ok: `targets/x86_64-core.conf: the regex manager matches each of its 1 pin(s)` | +| static/75-renovate | a `# tag: 3.21` marker over an `alpine:3.22` pin | not ok: `.github/workflows/ci.yml: each '# tag:' marker names the tag of its pin` | +| static/75-renovate | `tests/` removed from `managerFilePatterns` | not ok: `tests/integration/10-pipeline.sh: the regex manager reads it (3 pin(s))` | +| static/75-renovate | `major` added to the automerge rule | not ok: `no rule in .github/renovate.json automerges a major update` | +| static/75-renovate | `platformAutomerge: true` | not ok: `.github/renovate.json sets platformAutomerge to false` | +| static/75-renovate | `ignorePaths` removed | not ok: `.github/renovate.json sets ignorePaths, and no entry is a tests directory` | +| static/75-renovate | the keyring images set to automerge | not ok: `the keyring images (ghcr.io/pkgforge-dev/archlinux) never merge on their own` | + +## What the integration suite drives on every run + +These are not code faults. They are hostile or broken inputs, built by `tests/fixtures/build-repo`, or served by a web server that stands in for a registry that lies. The suite fails when the mirror accepts any of them. + +| input | must end as | +| --- | --- | +| more shards wanted than `MAX_SHARDS` | capped, and the plan says so | +| two writers of the same files | manifests of the same digests | +| a database with a NUL-only package directory | planned, with the directory named | +| a database with an unreadable member that is not NUL bytes | refused, with the member named | +| another arch | verified in the x86_64 image with `Architecture` from the target | +| a package that upstream signs nowhere, and one signed by an expired key | upstream-refused: listed, and the snapshot promoted | +| the same batch with no trust root | failed: nothing is excused | +| a keyring package that only `keyring_from` holds | installed from that target's `latest`, pinned by digest | +| an upstream database older than the promoted one | `behind`; `latest` does not move | +| a file signed again with the same bytes | planned; its signature layer repaired under the same tag | +| a signature layer that is not the `%PGPSIG%` | verify fails and names the file; never promoted | +| a file whose bytes change under its own name | `replaced`; the tag does not move | +| a new file whose tag already holds other bytes | `replaced`; the tag does not move | +| two db_servers, each ahead of the other on one package | `held` | +| a package signed by a key nobody trusts | upstream-refused: listed, and the snapshot promoted | +| a database with no `%PGPSIG%` | each new signature layer is upstream's detached `.sig` | +| a db_server, listed first, that lags by one dropped package | `latest` does not move | +| with no `%PGPSIG%`, a signature layer that is not upstream's `.sig` | verify fails: the mirror is at fault | +| a manifest whose bytes are not the pinned digest | refused by the wrapper | +| a manifest with two layers of one name | refused by the wrapper | +| a package manifest with no signature layer | refused by the wrapper, with a message | +| a blob whose size or sha256 is not its descriptor's | refused by the wrapper | +| a manifest over 64 KiB | given up within 20 s | +| a database signature whose snapshot moved on | not paired; the wrapper misses | + +`20-client.sh` adds these. The first four go through proxies of its own: + +| input | must end as | +| --- | --- | +| a registry that answers 503 | the second file misses in under 1 s, with no request; a `.sig` is still asked for | +| every blob cut in half | the install succeeds; the cut package resumes with a Range request | +| a blob that stops after 64 KiB for 40 s | the install takes less than 20 s; the transfer resumes from 64 KiB | +| one byte changed in the first answer for each blob | the prefetch keeps nothing; each file is read again alone; the install succeeds | +| a provider other than the default | the prefetch takes the first provider; the chosen one is read alone | +| two repositories with no database signature, under the sandbox | both signatures answered with an empty file; `-Sy` takes less than 1.9 s | +| the same with no sandbox | both stay misses; `-Sy` takes 4 s or more | diff --git a/docs/worker.md b/docs/worker.md new file mode 100644 index 0000000..c7e7d5b --- /dev/null +++ b/docs/worker.md @@ -0,0 +1,155 @@ +# The Worker + +The Worker is an HTTPS front for the GHCR mirror on Cloudflare. Its name is `ghcr-mirror-archlinux-pkgforge`, and its custom domain is `ghcr.mirror.archlinux.pkgforge.dev`. The code is `worker/src/index.js`, and the configuration is `worker/wrangler.toml`. + +```ini +[core] +Server = https://ghcr.mirror.archlinux.pkgforge.dev/$arch/$repo +Include = /etc/pacman.d/mirrorlist +``` + +pacman uses its own downloader with this Server. `ParallelDownloads`, resume and the server error limit apply, and no `XferCommand` is necessary. + +⭐ The Worker sends no package bytes. It finds the blob digest of the file in the manifest, and asks GHCR for that blob. It then answers 302 with the Location of GHCR's redirect, a signed blob URL. pacman checks the sha256 of each package against the database, and each signature against the keyring. + +## Paths + +| path | answers | +| --- | --- | +| `/` | 200: the usage text, with the Server line for the request host | +| `/ARCH/REPO/FILE` | FILE from tag `latest` | +| `/ARCH/REPO:latest/FILE` | the same | +| `/ARCH/REPO:YYYY.MM.DD/FILE` | the first promoted snapshot that was fetched on that UTC date | +| `/ARCH/REPO:db-HEX/FILE` | the snapshot whose `REPO.db` has sha256 HEX | +| `/ARCH/REPO:files-HEX/FILE` | the snapshot whose `REPO.files` has sha256 HEX | +| `/ARCH/REPO@sha256:HEX/FILE` | one manifest; its body must hash to HEX | + +FILE is `REPO.db`, `REPO.files`, a package file, or one of these with `.sig`. FILE can be percent-encoded. The Worker decodes it, then checks its character set again. ARCH has at most 32 characters. The decoded FILE has at most 255. The query string is ignored. + +A package file and its `.sig` always come from the package's own tag, whatever reference the URL names. Any other tag, `staging` included, gets 404 with no subrequest. A pin or a `db-` or `files-` tag with upper-case hex gets 404 too. Any other path gets 404 with no subrequest. + +The Worker checks the format of a `YYYY.MM.DD`, `db-HEX` or `files-HEX` tag only. It does not check that the database of a `db-HEX` manifest has the sha256 HEX. It checks the hash of an `@sha256:` manifest. + +## Answers + +| status | when | +| --- | --- | +| 200 | `/` only, with `cache-control: max-age=3600` | +| 302 | the redirect to the signed blob URL, with `cache-control: no-store` and no body | +| 404 | the mirror does not hold the file, or the path or tag is not served | +| 405 | a method other than GET and HEAD, with `allow: GET, HEAD` | +| 502 | GHCR failed, GHCR answered something not expected, or the Worker failed; with `retry-after: 30` | + +A 404, 405 or 502 answer is plain text, with `cache-control: no-store`. HEAD gets the same answer as GET, with no body. HEAD makes the same subrequests as GET. + +The Worker answers 502 to these, among others: + +- a blob answered with 200: the Worker never sends the bytes +- a blob status that is not 3xx and not 404, or a redirect with no valid Location +- a redirect whose Location is not https, or whose host does not end in `.githubusercontent.com` +- a manifest status that is not 200 and not 404 +- a manifest over 64 KiB, by `content-length` and then by the bytes read +- a manifest that is not UTF-8 JSON, or has no `layers` array +- a layer whose digest is not `sha256:` and 64 hex digits +- an `@sha256:` manifest whose body hashes to another value +- a subrequest that runs past 10 s, or that fails on the network +- an exception in the Worker + +A file with no layer or two layers of its name gets 404. A 502 answer and a blob 404 from GHCR are not stored in the cache. + +## Caching + +The Worker uses the Cache API. There is one cache per data centre, and entries do not replicate. Every key is on the request host under `/__cache/v1`. A change of entry format needs only a new version in that path. + +| entry | key | holds | time to live | +| --- | --- | --- | --- | +| `loc` | repository, reference, file | the final answer; a hit is the whole request | a redirect: until the `blob` entry ends, never past its manifest entry; a 404: 60 s, never past its manifest entry | +| `man` | repository, reference | the layers of the manifest | see below | +| `blob` | repository, digest | the signed URL | until the signed URL's `se` minus 120 s, at most 600 s | + +A signed URL with no `se` is not stored. Neither is one with less than 30 s left before `se` minus 120 s. The redirect to such a URL is not stored either. + +| manifest | time to live | +| --- | --- | +| `@sha256:` pin, `YYYY.MM.DD`, `db-HEX`, `files-HEX` | 1 year | +| package manifest with exactly one layer `FILE` and one layer `FILE.sig` | 6 hours | +| any other package manifest | 60 s | +| `latest` | 30 s | +| a 404 | 60 s | + +A new `latest` shows within 30 s. A package manifest that is pushed again to repair its `.sig` shows within 6 hours, or within 60 s when its `.sig` layer was missing. + +⚠ The 1-year time to live holds only while dated tags stay write-once and `db-`/`files-` tags stay content-addressed. Change `/__cache/v1` in the code to clear every entry with one deploy. + +A package request that misses the `loc` cache also fills the `loc` entry of its `.sig`, with `ctx.waitUntil`. The `.sig` blob is asked for at the same time as the package blob. pacman asks for the `.sig` next, from the same Server. When the entry is ready first, that request is one cache lookup in the same data centre. ⚠ A `.sig` asked for very soon after its package can miss the entry (measured in [measurements.md](measurements.md#cloudflare-workers)). + +## Logs + +The Worker calls `console.error` on each 5xx answer and on each cache failure. It also logs a `.sig` entry that it could not fill, and a body that it could not cancel. It logs nothing else. `wrangler.toml` sets `[observability] enabled = true`, `head_sampling_rate = 1` and `[observability.logs] invocation_logs = false`. + +## Limits + +| setting | value | +| --- | --- | +| `compatibility_date` | `2026-10-13` | +| `compatibility_flags` | `no_nodejs_compat`, `no_nodejs_compat_v2`: the Worker uses only Web APIs | +| `[limits] cpu_ms` | 50 | +| `workers_dev`, `preview_urls` | `false` | +| subrequest deadline | 10 s | +| manifest size | 64 KiB | + +The answer to one request takes at most three cache lookups and two GHCR subrequests. A package request also fills the entry of its `.sig` in the same invocation: at most two more cache lookups and one more subrequest. The Worker hashes a manifest only for an `@sha256:` pin. + +## Cost + +Cloudflare bills Workers by request and by CPU time. Subrequests and egress are not billed. Each file pacman asks for is one billed request, and so is each `.db.sig` that answers 404. The prices are in [measurements.md](measurements.md#cloudflare-workers). + +## Deploy + +`.github/workflows/worker.yml` runs on a push to `main` that changes `worker/**` or the workflow, and on dispatch. + +| job | does | +| --- | --- | +| `test` | `node --test 'worker/test/*.test.js'` in the pinned `node:24-alpine` image | +| `deploy` | on `main`, after `test` passed: `cloudflare/wrangler-action` with wrangler `4.150.0` in `worker/`, then `tests/worker/live.sh` up to 3 times, 20 s apart | + +⚠ The deploy job needs the `CLOUDFLARE_API_TOKEN` secret. When the secret is empty, the job shows a notice and ends with success, with no deploy. Make the token from the "Edit Cloudflare Workers" template. + +⚠ The live check runs after the deploy. A GHCR or blob host outage then fails the job, and the new version stays deployed. There is no rollback. + +Deploy by hand: + +```bash +cd worker && npx wrangler@4.150.0 deploy +``` + +Check the configuration with no deploy: + +```bash +cd worker && npx wrangler@4.150.0 deploy --dry-run --outdir "$(mktemp -d)" +``` + +## Tests + +```bash +REPO_ROOT=$PWD bash tests/worker/10-node.sh +WORKER_LIVE=1 REPO_ROOT=$PWD bash tests/worker/live.sh +``` + +`tests/worker/10-node.sh` runs `worker/test/*.test.js` with node 22 or newer. The tests use a mock fetch that stands in for GHCR, and a Cache API stub with a clock. They need no network. One test compares `tagOf` with `client/pacman-oci --tags` for every name in `tests/fixtures/filenames.txt`. + +⚠ Give `node --test` the quoted glob `'worker/test/*.test.js'`, not the directory. node 26 runs a directory argument as a file and fails. + +`tests/worker/live.sh` checks the deployed Worker. It runs only when `WORKER_LIVE=1`. It reads from the Worker, from GHCR with the anonymous token, and from the blob host. It needs curl, tar with gzip, awk, sha256sum and the usual coreutils. + +| variable | default | +| --- | --- | +| `WORKER_URL` | `https://ghcr.mirror.archlinux.pkgforge.dev` | +| `WORKER_ARCH` | `x86_64` | +| `WORKER_REPO` | `core` | + +## Trust + +The Worker holds no secret. The API token reaches only the `deploy` job: one step checks that it is set, and `wrangler-action` uses it. The account id is in `worker/wrangler.toml`. + +The databases of the mirror are unsigned. A consumer that reads through the Worker trusts the Worker for the database it gets. So the Worker's code, and every account that can deploy it, join the trust root of that consumer. pacman still checks each package's sha256 against the database and each signature against the keyring. A consumer that must not trust Cloudflare reads the registry directly with the wrapper. diff --git a/scripts/check-schedules-fired b/scripts/check-schedules-fired index cb5d450..679a315 100755 --- a/scripts/check-schedules-fired +++ b/scripts/check-schedules-fired @@ -35,7 +35,7 @@ # Environment: # GITHUB_REPOSITORY owner/name, default read from the origin remote # SCHEDULES_TODAY the date to measure against, YYYY-MM-DD, default today in -# UTC. ⚠ For fault injection only. HISTORY/tests-seen-to-fail.md +# UTC. ⚠ For fault injection only. docs/tests.md # # Needs a token with actions:read. GH_TOKEN in CI, gh's own login locally. # diff --git a/scripts/cron-tolerance b/scripts/cron-tolerance index c6d9d5c..09deca3 100755 --- a/scripts/cron-tolerance +++ b/scripts/cron-tolerance @@ -45,7 +45,7 @@ # 1, so a schedule expected four times a day is given three days of grace. It # would still catch a stop, later than it could have. sync.yml fires every # three hours and so gets three days; that is the accepted cost of reusing a -# measured, day-resolution routine (HISTORY/design.md, the watchdog). +# measured, day-resolution routine (docs/operations.md, staleness). # # Exit status: # 0 a tolerance was derived and printed diff --git a/scripts/install-oras b/scripts/install-oras index ef7d48f..8310128 100755 --- a/scripts/install-oras +++ b/scripts/install-oras @@ -9,7 +9,7 @@ # ⛔ The version and both checksums are written here, not fetched. A checksum # read from the same release it checks proves only that the release is # self-consistent. These were read from the v1.3.4 release's checksums.txt on -# 2026-10-13 and match the measurement in HISTORY/design.md. +# 2026-10-13, as docs/measurements.md records. # # ⭐ oras is stable and nothing here follows its releases. A newer version is a # deliberate edit of the three lines below and nothing else. diff --git a/scripts/mirror-file b/scripts/mirror-file index 35c3da7..47e6fe5 100755 --- a/scripts/mirror-file +++ b/scripts/mirror-file @@ -41,7 +41,8 @@ # REGISTRY_USER, REGISTRY_TOKEN, REGISTRY_TOKEN_CACHE (lib.sh), MIRROR_MIN_RATE # (bytes per second a push may not fall below; default 2097152). A push # attempt gets 300 s plus the time at that rate, and never more than 600 s, -# GHCR's limit for one upload. +# GHCR's limit for one upload. MIRROR_RATE_WAIT (seconds; default 1800) bounds +# the pauses one file takes after a 429 from the registry (below). # # Exit status: 0 pushed or present, 1 anything else. set -euo pipefail diff --git a/scripts/report b/scripts/report index b46c703..867c9f6 100755 --- a/scripts/report +++ b/scripts/report @@ -220,7 +220,7 @@ while IFS=$'\t' read -r target state why; do sed -n 1,50p "$d/replaced.tsv" printf '```\n\n' fi - printf 'A person decides: wait for upstream to publish a new filename, or accept the new bytes under a new tag (HISTORY/design.md, write-once tags).\n\nRun: %s\n' "$run_url" + printf 'A person decides: wait for upstream to publish a new filename, or accept the new bytes under a new tag (docs/design.md, rules).\n\nRun: %s\n' "$run_url" } > "$work/body.md" track "replaced-$target" raise "$target: a package tag would have to change" "$work/body.md" fi diff --git a/scripts/status-collect b/scripts/status-collect index e87aee9..8295df2 100755 --- a/scripts/status-collect +++ b/scripts/status-collect @@ -1,7 +1,7 @@ #!/usr/bin/env bash # # Measure what the mirror serves against what upstream serves, for the status -# page and the watchdog. +# page and the worker-down check of status.yml. # # scripts/status-collect OUT [TARGET.conf ...] # From ee21cc51ef2f2e4cfedf8f5ea0d6a0fd5e74dddb Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 21:22:39 +0545 Subject: [PATCH 16/22] lib: the registry credential reaches curl on stdin; staging slug cut before its dashes; guard against overlapping target names --- .github/workflows/sync-target.yml | 2 +- scripts/lib.sh | 12 +++++++----- tests/static/60-targets.sh | 17 +++++++++++++++++ worker/wrangler.toml | 5 +++-- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sync-target.yml b/.github/workflows/sync-target.yml index ec414e2..3a1f14c 100644 --- a/.github/workflows/sync-target.yml +++ b/.github/workflows/sync-target.yml @@ -126,7 +126,7 @@ jobs: if [ "$GITHUB_REF" = refs/heads/main ]; then export STAGING_TAG=staging else - slug="$(printf '%s' "$GITHUB_REF_NAME" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9\n' '-' | sed 's/--*/-/g; s/^-//; s/-$//' | cut -c1-40)" + slug="$(printf '%s' "$GITHUB_REF_NAME" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9\n' '-' | sed 's/--*/-/g' | cut -c1-40 | sed 's/^-//; s/-$//')" export STAGING_TAG="staging-${slug:-branch}" fi scripts/stage "$RUNNER_TEMP/plan" "$RUNNER_TEMP/results" "$RUNNER_TEMP/stage" "$TARGET" diff --git a/scripts/lib.sh b/scripts/lib.sh index c459fa7..b736d67 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -623,11 +623,13 @@ registry_token() { fi out="$(mktemp)" if [ -n "${REGISTRY_TOKEN:-}" ]; then - curl -q --silent --show-error --fail --connect-timeout 20 --max-time 60 \ - --retry 3 --retry-connrefused --retry-delay 2 \ - --user "${REGISTRY_USER:-token}:$REGISTRY_TOKEN" \ - --get --data-urlencode "service=$MIRROR_REGISTRY" --data-urlencode "scope=repository:$repo:$scope" \ - --output "$out" "https://$MIRROR_REGISTRY/token" || rc=$? + # The credential goes to curl on stdin, as a config line, never on its + # command line, where every process of the runner could read it. + printf 'user = "%s:%s"\n' "${REGISTRY_USER:-token}" "$REGISTRY_TOKEN" | + curl -q --silent --show-error --fail --connect-timeout 20 --max-time 60 \ + --retry 3 --retry-connrefused --retry-delay 2 --config - \ + --get --data-urlencode "service=$MIRROR_REGISTRY" --data-urlencode "scope=repository:$repo:$scope" \ + --output "$out" "https://$MIRROR_REGISTRY/token" || rc=$? else curl -q --silent --show-error --fail --connect-timeout 20 --max-time 60 \ --retry 3 --retry-connrefused --retry-delay 2 \ diff --git a/tests/static/60-targets.sh b/tests/static/60-targets.sh index afbda9b..9c73741 100644 --- a/tests/static/60-targets.sh +++ b/tests/static/60-targets.sh @@ -119,4 +119,21 @@ else "add slot steps to the plan job" "reproduce: grep -c 'outputs.slot' .github/workflows/sync.yml" fi +# sync-target.yml downloads a chain's artifacts by --[0-9]*. A +# target named like another plus "-" and a digit (x86_64-core-2 beside +# x86_64-core) would match that pattern and read the other chain's files. +overlap="$(for c in targets/*.conf; do basename "$c" .conf; done | awk ' + { name[NR] = $0 } + END { + for (i = 1; i <= NR; i++) for (j = 1; j <= NR; j++) + if (i != j && index(name[j], name[i] "-") == 1 && substr(name[j], length(name[i]) + 2, 1) ~ /[0-9]/) + print name[j] " extends " name[i] + }')" +if [ -z "$overlap" ]; then + ok "no target name is another's name, a dash and a digit" +else + fail "no target name is another's name, a dash and a digit" "$overlap" \ + "the artifact patterns of sync-target.yml would mix the two chains" "reproduce: ls targets" +fi + summary diff --git a/worker/wrangler.toml b/worker/wrangler.toml index ece3109..a196143 100644 --- a/worker/wrangler.toml +++ b/worker/wrangler.toml @@ -18,8 +18,9 @@ routes = [ { pattern = "ghcr.mirror.archlinux.pkgforge.dev", custom_domain = true }, ] -# Each request is at most three cache lookups, two GHCR subrequests and one -# SHA-256 of a manifest of at most 64 KiB. +# Each answer is at most three cache lookups, two GHCR subrequests and one +# SHA-256 of a manifest of at most 64 KiB. A package request also fills its +# .sig entry: at most two more lookups and one more subrequest. [limits] cpu_ms = 50 From 68a4edb3f3732a31d94b98e91241b8e43aeabc74 Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 21:23:20 +0545 Subject: [PATCH 17/22] ci: alpine 3.24 for the busybox unit leg --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b55ee9f..3b68504 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,8 +80,8 @@ jobs: image: docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 require: jq curl - leg: busybox - # tag: 3.22 - image: docker.io/library/alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8 + # tag: 3.24 + image: docker.io/library/alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 require: jq curl steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 From 233876a79cb705d5484fdcab3dc425c88d45f55d Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 21:23:55 +0545 Subject: [PATCH 18/22] docs: no image version or digest that a Renovate update would leave stale --- docs/tests.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/tests.md b/docs/tests.md index a3a650e..e6ee5f2 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -107,11 +107,13 @@ Each run picks its ports at run time, and each container name carries the shell' The unit and static suites run in the pinned `archlinux:base`. Mount the tools last on PATH, so the image's own tools stay the ones under test: ```bash -docker run --rm -v "$PWD:/work:ro" -v /tmp/mirror-archlinux-tools:/tools:ro -w /work \ - docker.io/library/archlinux:base@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 \ +image="$(sed -n 's/^verify_image = //p' targets/x86_64-core.conf)" +docker run --rm -v "$PWD:/work:ro" -v /tmp/mirror-archlinux-tools:/tools:ro -w /work "$image" \ sh -c 'PATH="$PATH:/tools" exec bash tests/run.sh unit' ``` +The image is the targets' verify image, so this command follows each Renovate update. + ⚠ `40-executable-bits.sh` reads the git index. In a copy with no git index, it passes with one line that says it skipped. ## CI @@ -121,7 +123,7 @@ docker run --rm -v "$PWD:/work:ro" -v /tmp/mirror-archlinux-tools:/tools:ro -w / | `static` | `tests/static/72-test-runner.sh` with bash first, then `tests/run.sh static`; `TESTS_REQUIRE_TOOLS='shellcheck actionlint jq python3'` | | `unit (dash-mawk)` | `tests/run.sh unit` on the runner, with dash and mawk; requires `jq curl python3` | | `unit (bash-gawk)` | the same in the pinned `archlinux:base`; requires `jq curl` | -| `unit (busybox)` | the same in the pinned `alpine:3.22`; requires `jq curl` | +| `unit (busybox)` | the same in the pinned alpine image; requires `jq curl` | | `integration` | one job per file of `tests/run.sh --list integration` | | `worker` | `tests/run.sh worker` in the pinned `node:24-alpine` | | `debloat` | `scripts/debloat-runner` on a real runner; a warning when it frees less than 1 GiB | From 97be6cb254b9beaafd2e24ee9a64119bd61feb6f Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 22:11:00 +0545 Subject: [PATCH 19/22] client: fix review findings A signature request never marks its host down, so a failed optional .sig no longer makes the next file from that host miss. The size of a held .part comes from its metadata (ls -ln), so a .part that the download user cannot read no longer stops the call with an arithmetic error. After a refused range (curl 33), resume and the per-attempt time limit are off, and the 30 s stop starts again, so the fetch from zero runs. The remover touches ,open once a second, and the sweep keeps the entries of a run whose ,open is less than 2 minutes old, so a run in another pid namespace keeps its store. --- client/pacman-oci | 130 ++++++++++++++++++++------- docs/client.md | 26 ++++-- docs/tests.md | 8 +- tests/fixtures/client/consumer | 10 +++ tests/fixtures/client/fault-proxy.py | 40 ++++++++- tests/integration/20-client.sh | 37 ++++++-- tests/unit/31-run-store.sh | 51 ++++++++++- tests/unit/32-prefetch-plan.sh | 89 ++++++++++++++++++ 8 files changed, 338 insertions(+), 53 deletions(-) diff --git a/client/pacman-oci b/client/pacman-oci index 4d04846..e7fa59c 100755 --- a/client/pacman-oci +++ b/client/pacman-oci @@ -47,13 +47,15 @@ # flat in OUTPUT's directory as .pacman-oci.-.. pacman # removes them with its download-XXXXXX directory, a background process removes # them when pacman exits, and each call removes those of runs that are gone. A -# call whose parent is not pacman keeps no store. +# run is gone when its pid is gone and its ,open has had no heartbeat for 2 +# minutes. A call whose parent is not pacman keeps no store. # # pacman runs this with fork and execvp, never through a shell. With # DownloadUser set (stock Arch) it runs as that user, from a download-XXXXXX # directory inside the sync or cache directory; without it, as root. # -# Needs curl, awk, sed, grep, sha256sum, mktemp, base64 (coreutils or busybox). +# Needs curl, awk, sed, grep, sha256sum, mktemp, base64, ls, and find with +# -mmin (coreutils, findutils or busybox). # Uses pacman, pacman-conf and timeout when they are there. # # Environment: @@ -106,7 +108,8 @@ debug() { # entry is used only when it is a regular file this user owns, never through a # symbolic link, and each is made under a mktemp name, then renamed. # FILE a file ready for pacman, its size and sha256 checked -# ,open the store of this run is open in this directory +# ,open the store of this run is open in this directory; the +# background remover touches it once a second # ,prefetch the package prefetch of this run ran here # ,dbprefetch the database prefetch of this run ran here # ,down.HOST HOST failed once its retries were spent @@ -515,6 +518,22 @@ size_of() { printf '%s\n' $((_n)) } +# meta_size FILE: the size of FILE from its metadata into $_ms, as pacman's +# own stat() reads it. Unlike size_of, it needs no read permission. Under +# DownloadUser, a .part that root left with mode 0600 still has a size here. +meta_size() { + _ms='' + _ls=$(ls -ln -- "$1") || return 1 + set -f + # shellcheck disable=SC2086 # the fields of ls -l are words + set -- $_ls + set +f + case ${5:-} in + '' | *[!0-9]*) return 1 ;; + esac + _ms=$5 +} + # A clock in nanoseconds. busybox date has no %N, so there it counts seconds. now_ns() { _ns=$(date +%s%N) @@ -557,31 +576,54 @@ put() { mv -f -- "$_t" "$store$1" } -# Remove the entries of runs that are gone. Once this run's store is open in -# this directory, that was done, and the call goes straight on. +# beating KEY: run KEY touched its ,open in this directory in the last 2 +# minutes. Its remover touches that entry once a second while the run is live. +beating() { + _o=$out_dir/.pacman-oci.$1.,open + owned "$_o" || return 1 + case $_o in + /*) ;; + *) _o=./$_o ;; + esac + _m=$(find "$_o" -prune -mmin -2) || return 1 + [ -n "$_m" ] +} + +# Remove the entries of runs that are gone. A run is gone when its pid is +# missing or has another start time, and its ,open here has no heartbeat. A +# run in another pid namespace has a pid that this /proc cannot show. Its +# heartbeat keeps its entries. Once this run's store is open in this +# directory, that was done, and the call goes straight on. sweep() { [ -d "$PROC" ] || return 0 if entry_ok ,open; then return 0 fi set -- - _live='' + # The entries of one key are next to each other, so each key is decided once. + _seen='' + _keep=1 for _e in "$out_dir"/.pacman-oci.*; do owned "$_e" || continue _k=${_e##*/.pacman-oci.} _k=${_k%%.*} - [ "$_k" != "$key" ] && [ "$_k" != "$_live" ] || continue - _pid=${_k%%-*} - _start=${_k#*-} - case $_pid:$_start in - *[!0-9:]* | :* | *:) continue ;; - esac - start_of "$_pid" - if [ "$_st" = "$_start" ]; then - _live=$_k - continue + [ "$_k" != "$key" ] || continue + if [ "$_k" != "$_seen" ]; then + _seen=$_k + _keep=1 + _pid=${_k%%-*} + _start=${_k#*-} + case $_pid:$_start in + *[!0-9:]* | :* | *:) continue ;; + esac + start_of "$_pid" + if [ "$_st" != "$_start" ] && ! beating "$_k"; then + _keep=0 + fi + fi + if [ "$_keep" = 0 ]; then + set -- "$@" "$_e" fi - set -- "$@" "$_e" done if [ "$#" -gt 0 ]; then debug "removing $# file(s) of pacman runs that are gone" @@ -590,11 +632,16 @@ sweep() { } # reap PID START PREFIX: wait for the pacman run to end, then remove its -# entries in one directory. pacman waits for its child alone, never for this. +# entries in one directory. While the run is live, it touches PREFIX,open +# once a second: that is the heartbeat sweep reads. pacman waits for its +# child alone, never for this. # shellcheck disable=SC2329 # run in the background by store_open reap() { trap '' HUP INT while start_of "$1" && [ "$_st" = "$2" ]; do + if owned "$3,open"; then + : > "$3,open" + fi sleep 1 done rm -f -- "$3"* @@ -688,8 +735,15 @@ host_of() { } # mark_down HOST HTTP: later requests of this run to HOST miss at once. Only no -# answer at all, 429 and 5xx count. +# answer at all, 429 and 5xx count. A signature request never marks a host: +# pacman ignores a failed optional signature, then asks the same host for the +# next file. A failed required signature stops the transaction anyway. pacman's +# own downloader also never counts an optional request against a server +# (libalpm dload.c, request_errors_ok). mark_down() { + case $kind in + pkgsig | dbsig) return 0 ;; + esac [ -n "$1" ] || return 0 case $2 in 000 | 429 | 5??) ;; @@ -1190,7 +1244,8 @@ say_summary() { # line per file it would download: %l (the first Server, or file:// when the # file is cached), %h (sha256), %s (bytes to download) and %g (base64 # %PGPSIG%). Print mode picks the first provider and never prompts. %s is the -# size less any .part pacman holds in the cache, so that .part is added back. +# size less any .part pacman holds in the cache, so the size of that .part, +# from its metadata, is added back. # # PLAN_AWK prints, per file to fetch: # TYPE FILE SHA256 SIZE PGPSIG BASE URL (TYPE oci or http; "-" when absent) @@ -1298,8 +1353,8 @@ plan_pkgs() { _n=0 while IFS="$TAB" read -r _type _f _h _s _g _b _u; do _n=$((_n + 1)) - if [ -f "$held_dir/$_f.part" ] && [ ! -L "$held_dir/$_f.part" ]; then - _s=$((_s + $(size_of "$held_dir/$_f.part"))) + if [ -f "$held_dir/$_f.part" ] && [ ! -L "$held_dir/$_f.part" ] && meta_size "$held_dir/$_f.part"; then + _s=$((_s + _ms)) fi if [ "$_h" != - ]; then printf '%s\t%s\t%s\n' "$_f" "$_h" "$_s" >> "$work/index" @@ -1584,19 +1639,26 @@ parse_oci() { # fetch_one URL SIZE AUTH(0|1) PROTO RESUMABLE(0|1): OUTPUT from URL in up to # three attempts. A resumable file keeps what it got and the next attempt # resumes with --continue-at -; anything else starts from zero. When SIZE is -# known, each attempt has its time limit (see deadline). No attempt -# starts after 30 s, unless the one before it added bytes. Sets one_rc (curl's -# exit), one_code (the last HTTP status) and one_resumed. +# known and the file can resume, each attempt has its time limit (see +# deadline). An attempt that cannot resume gets no time limit: a cut would +# lose all its bytes. A server that refuses a range (curl 33) turns resume off +# for the attempts after it. No attempt starts after 30 s, unless the one +# before it added bytes. A refused range starts these 30 s again. Sets one_rc +# (curl's exit), one_code (the last HTTP status) and one_resumed. #---------------------------------------------------------------------------# fetch_one() { _url=$1 _max=$2 _auth=$3 _proto=$4 _resumable=$5 _try=1 _stop=$(($(date +%s) + 30)) while :; do + _resume=0 + if [ "$_resumable" = 1 ] && [ "${no_resume:-0}" != 1 ]; then + _resume=1 + fi one_resumed=0 _had=0 set -- - if [ "$_resumable" = 1 ] && [ "${no_resume:-0}" != 1 ] && [ -s "$out" ]; then + if [ "$_resume" = 1 ] && [ -s "$out" ]; then _had=$(size_of "$out") fi if [ "$_had" -gt 0 ] && { [ -z "$_max" ] || [ "$_had" -lt "$_max" ]; }; then @@ -1608,9 +1670,11 @@ fetch_one() { fi if [ -n "$_max" ]; then set -- "$@" --max-filesize "$_max" - deadline "$_max" "$_try" - if [ -n "$_dl" ]; then - set -- "$@" --max-time "$_dl" + if [ "$_resume" = 1 ]; then + deadline "$_max" "$_try" + if [ -n "$_dl" ]; then + set -- "$@" --max-time "$_dl" + fi fi fi if [ -n "$_proto" ]; then @@ -1632,13 +1696,17 @@ fetch_one() { debug "GET $_url -> $one_code (curl $one_rc, resumed $one_resumed)" [ "$one_rc" -ne 0 ] || return 0 case $one_rc:$one_code in - 33:*) rm -f -- "$out" ;; + 33:*) + rm -f -- "$out" + no_resume=1 + _stop=$(($(date +%s) + 30)) + ;; *:408 | *:429 | *:5?? | 6:* | 16:* | 18:* | 28:* | 35:* | 52:* | 55:* | 56:* | 92:*) ;; *) return 1 ;; esac [ "$_try" -lt 3 ] || return 1 if [ "$(date +%s)" -ge "$_stop" ]; then - if [ "$_resumable" != 1 ] || [ ! -s "$out" ] || [ "$(size_of "$out")" -le "$_had" ]; then + if [ "$_resume" != 1 ] || [ ! -s "$out" ] || [ "$(size_of "$out")" -le "$_had" ]; then return 1 fi fi diff --git a/docs/client.md b/docs/client.md index 8a0d838..0469cb9 100644 --- a/docs/client.md +++ b/docs/client.md @@ -52,7 +52,7 @@ The wrapper sorts each request into one kind: ## Requirements -The wrapper needs `curl`, `awk`, `sed`, `grep`, `sha256sum`, `mktemp` and `base64`, and the other usual tools of coreutils or busybox. It uses `pacman`, `pacman-conf` and `timeout` when they are there. The test suite runs it under bash with gawk, dash with mawk, and busybox. +The wrapper needs `curl`, `awk`, `sed`, `grep`, `sha256sum`, `mktemp` and `base64`, and the other usual tools of coreutils or busybox. Its `find` must know `-mmin`, as GNU findutils and busybox do. It uses `pacman`, `pacman-conf` and `timeout` when they are there. The test suite runs it under bash with gawk, dash with mawk, and busybox. pacman runs it with fork and execvp, never through a shell. With `DownloadUser` set, it runs as that user, from a `download-XXXXXX` directory in the sync or cache directory. Without `DownloadUser`, it runs as root, in the sync or cache directory. @@ -69,7 +69,7 @@ A run of pacman makes many calls of the wrapper, one per file. The run store let | entry | means | | --- | --- | | `` | a file ready for pacman | -| `,open` | the store of this run is open in this directory | +| `,open` | the store of this run is open in this directory; the background remover touches it once a second | | `,prefetch` | the package prefetch of this run ran here | | `,dbprefetch` | the database prefetch of this run ran here | | `,down.` | the host failed once its retries were spent | @@ -83,7 +83,9 @@ Three things remove the entries: 1. pacman unlinks every regular file of its `download-XXXXXX` directory when it removes that directory. 2. The call that opens the store starts a background remover. It waits until the pacman run ends, then removes the run's entries. -3. Until the store of its run is open in a directory, each call removes the entries of runs that are gone there. A run is gone when its pid is missing or its start time differs. +3. Until the store of its run is open in a directory, each call removes the entries of runs that are gone there. A run is gone when two things are true. Its pid is missing or has another start time. It has no `,open` there that is less than 2 minutes old. + +While a run is live, its background remover touches its `,open` once a second. A run in another pid namespace has a pid that this `/proc` cannot show. Its fresh `,open` keeps its entries. When a pid namespace ends, its remover ends with it. The first call there after 2 minutes removes the entries that the run left. ⚠ Entries stay flat in OUTPUT's directory, never in a subdirectory. pacman cannot remove a `download-XXXXXX` directory that holds a subdirectory, and leaves it behind. The private work directory goes there only when `TMPDIR` is not writable, and the wrapper removes it on exit. @@ -112,7 +114,11 @@ The prefetch runs once per run when all of these are true: 3. It finds the pacman binary from `/proc//exe`, else from `argv[0]`. The download user cannot read `/proc//exe`. 4. It runs pacman with `--print --print-format '%l%h%s%g' --noconfirm --noprogressbar`, under `timeout 120` when `timeout` exists. `pacman -S --print` takes no lock and needs no root. -Each line names the first Server URL (`%l`), the sha256 (`%h`), the download size (`%s`) and the base64 `%PGPSIG%` (`%g`). A `file://` line is a cached file and is skipped. A line is work when all of these are true: +Each line names the first Server URL (`%l`), the sha256 (`%h`), the download size (`%s`) and the base64 `%PGPSIG%` (`%g`). A `file://` line is a cached file and is skipped. + +`%s` leaves out a `.part` that pacman holds in the cache, so the wrapper adds the size of that `.part` back. It reads the size from the file's metadata (`ls -ln`), as pacman's own `stat()` does. So a `.part` that root left with mode 0600 still counts under `DownloadUser`. + +A line is work when all of these are true: - it starts with `oci://`, `oci+http://`, `https://` or `http://` - it names a package file, not a `.sig`, and `%s` is a number @@ -176,9 +182,9 @@ A file the store does not hold is read alone. A blob download keeps OUTPUT between attempts and resumes with `--continue-at -`. A resumed file that fails the sha256 check is removed and fetched once more from zero. -An attempt is tried again after HTTP 408, 429 or 5xx. It is also tried again after curl exits 6, 16, 18, 28, 35, 52, 55, 56 or 92. These are a name, TLS, HTTP/2 or time failure, an empty answer, or a cut transfer. After curl exit 33 (a range the server refuses), OUTPUT is removed and the attempt is tried again. Any other failure misses at once. The waits between attempts are 1 s, then 2 s. +An attempt is tried again after HTTP 408, 429 or 5xx. It is also tried again after curl exits 6, 16, 18, 28, 35, 52, 55, 56 or 92. These are a name, TLS, HTTP/2 or time failure, an empty answer, or a cut transfer. After curl exit 33 (a range the server refuses), OUTPUT is removed, and the attempts after it start from zero. Any other failure misses at once. The waits between attempts are 1 s, then 2 s. -When the size of the file is known, each attempt has a time limit: +When the size of the file is known and the file can resume, each attempt has a time limit: | attempt | time limit | | --- | --- | @@ -186,7 +192,9 @@ When the size of the file is known, each attempt has a time limit: | 2 | 30 s plus 1 s per 128 KiB | | 3 | none | -No attempt starts after 30 s, unless the one before it added bytes. +An attempt that cannot resume has no time limit, because a cut would lose all its bytes. That is an attempt after a refused range, and the fetch from zero after a resumed file failed its sha256 check. + +No attempt starts after 30 s, unless the one before it added bytes. A refused range starts these 30 s again. ### Plain URLs @@ -194,7 +202,9 @@ pacman's own downloader is off when an `XferCommand` is set. So the wrapper also ### The host blacklist -A request that ends with no answer (HTTP 000), 429 or 5xx, once its own retries are spent, writes `,down.`. Later `pkg` and `db` requests of the same run, in the same directory, to that host miss at once. pacman then tries the next Server with no wait for timeouts. +A `pkg` or `db` request that ends with no answer (HTTP 000), 429 or 5xx, once its own retries are spent, writes `,down.`. Later `pkg` and `db` requests of the same run, in the same directory, to that host miss at once. pacman then tries the next Server with no wait for timeouts. + +A signature request never writes `,down.`. pacman ignores a failed optional signature, then asks the same host for the next file. A failed required signature stops the transaction anyway. pacman's own downloader also never counts an optional request against a server. ### An absent database signature diff --git a/docs/tests.md b/docs/tests.md index e6ee5f2..396a3d2 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -25,7 +25,7 @@ | `unit/20-json-parser.sh` | the JSON parser reads every legal shape and refuses anything else | | `unit/30-wrapper-urls.sh` | one manifest request per URL; a URL it cannot read is refused before the network | | `unit/31-run-store.sh` | the run store: key, ownership, sweep, remover, host blacklist | -| `unit/32-prefetch-plan.sh` | the argument rewrite, the print lines, the signatures written from `%g` | +| `unit/32-prefetch-plan.sh` | the argument rewrite, the print lines, the signatures written from `%g`, the size of a held `.part` | | `unit/40-lib.sh` | the decisions of the sync on fixed inputs | | `unit/45-dispatch-overdue.sh` | `scripts/dispatch-overdue` starts a workflow only when it is active and overdue | | `unit/60-status.sh` | `status-collect` and `status-render` on fixtures | @@ -159,12 +159,17 @@ The tree itself is never changed. An integration fault runs with its own registr | unit/30-wrapper-urls | an uncompressed `.pkg.tar` read as a database | not ok: `an uncompressed package` | | unit/30-wrapper-urls | a database signature read from the moving tag | not ok: `core.db.sig is read by the content of the core.db beside it` | | unit/30-wrapper-urls | a failed attempt leaves its `.part` behind | not ok: `a failed attempt removes the .part it was given` | +| unit/31-run-store | a failed signature request marks its host down | 3 not ok, among them `a database signature: no host is marked down, so the next file is still asked for` | +| unit/31-run-store | the sweep ignores the age of `,open` | not ok: `the sweep keeps a run whose ,open is fresh, and removes one whose ,open is old` | +| unit/31-run-store | the remover never touches `,open` | not ok: `the remover touches ,open within 3 s while the run is live` | +| unit/32-prefetch-plan | the size of a held `.part` read with `wc -c` | 2 not ok, among them `a .part the wrapper cannot read is sized from its metadata` | | integration/10-pipeline | the blob's sha256 no longer checked | not ok: `the wrapper refuses a blob whose sha256 is not its descriptor's` | | integration/20-client | no host is marked down; no transfer resumes; the background remover removes nothing | 12 not ok, among them `dead: a registry that answers 503: the second file makes no request`, `resume: the cut package was resumed with a Range request` and `main-root: no store file is left once pacman has ended` | | integration/20-client | the prefetch keeps a whole file with no size and sha256 check | 7 not ok, among them `flip: one byte changed in each first answer: the prefetch keeps neither database` | | integration/20-client | the batch check reads the `sha256sum` output one column off | 16 not ok, among them `main-du: the serial pass makes no network request` | | integration/20-client | a transfer cut by its time limit never goes on in the next pass | not ok: `stall: the prefetch cut the stopped transfer and resumed it from 64 KiB` | | integration/20-client | the prefetch makes one pass only | not ok: `resume: every blob cut in half: the prefetch resumed it, the serial pass made no request` | +| integration/20-client | a refused range keeps the 30 s stop of the attempts | 3 not ok, among them `norange: a plain Server that ignores Range: pacman -S big succeeds after its range is refused past 30 s` | ### scripts @@ -331,6 +336,7 @@ These are not code faults. They are hostile or broken inputs, built by `tests/fi | every blob cut in half | the install succeeds; the cut package resumes with a Range request | | a blob that stops after 64 KiB for 40 s | the install takes less than 20 s; the transfer resumes from 64 KiB | | one byte changed in the first answer for each blob | the prefetch keeps nothing; each file is read again alone; the install succeeds | +| a plain Server of its own that ignores Range, and stops a large file for 35 s | the install succeeds; the refused range is followed by one fetch from zero | | a provider other than the default | the prefetch takes the first provider; the chosen one is read alone | | two repositories with no database signature, under the sandbox | both signatures answered with an empty file; `-Sy` takes less than 1.9 s | | the same with no sandbox | both stay misses; `-Sy` takes 4 s or more | diff --git a/tests/fixtures/client/consumer b/tests/fixtures/client/consumer index 6e0f6fc..b5aec48 100644 --- a/tests/fixtures/client/consumer +++ b/tests/fixtures/client/consumer @@ -24,6 +24,9 @@ # 40 s after its first 64 KiB # flip -Sy and -S alpha gamma through the proxy that changes the # last byte of the first answer for each blob +# norange -Sy and -S big with PACMAN_OCI_PREFETCH=0 from a plain http +# Server that ignores Range and stops sending big for 35 s +# after its first 64 KiB # # Exit status: 0 the case ran (its results say how it went), 1 it could not. set -euo pipefail @@ -190,6 +193,13 @@ EOS result s_ms "$((($(date +%s%N) - t0) / 1000000))" installed big ;; + norange) + conf yes 'Required DatabaseNever' + export PACMAN_OCI_PREFETCH=0 + sync sy -Sy + sync s -S big + installed big + ;; flip) conf yes 'Required DatabaseNever' sync sy -Sy diff --git a/tests/fixtures/client/fault-proxy.py b/tests/fixtures/client/fault-proxy.py index 22e4722..556d63e 100644 --- a/tests/fixtures/client/fault-proxy.py +++ b/tests/fixtures/client/fault-proxy.py @@ -15,16 +15,23 @@ headers, with the full Content-Length, and 64 KiB of its body; then nothing for 40 s, and the connection closes. Every other GET is passed through. + norange + not a proxy: a plain http server for the files under UPSTREAM, a + directory. Each GET gets the whole file with 200, whatever its Range + header says. The first GET of each file over 1 MiB gets its + headers, with the full Content-Length, and 64 KiB of its body; then + nothing for 35 s, and the connection closes. -UPSTREAM is the registry, host:port. The proxy listens on 127.0.0.1 on a port -the system picks, and prints "PORT " first. Then each request is one line: -" range=". +UPSTREAM is the registry, host:port, or the directory for norange. The proxy +listens on 127.0.0.1 on a port the system picks, and prints "PORT " first. +Then each request is one line: " range=". Standard library only. """ import http.client import http.server +import os import sys import threading import time @@ -60,9 +67,36 @@ def answer(self, code, headers, body): if body: self.wfile.write(body) + def serve_file(self): + """MODE norange: the file under UPSTREAM, whole, whatever Range says.""" + rel = os.path.normpath("/" + self.path.split("?", 1)[0]).lstrip("/") + full = os.path.join(UPSTREAM, rel) + if not rel or not os.path.isfile(full): + self.answer(404, [("Content-Length", "0")], b"") + return + with open(full, "rb") as f: + body = f.read() + keep = [("Content-Type", "application/octet-stream"), ("Content-Length", str(len(body)))] + if len(body) > 1048576 and first_time(self.path): + self.send_response(200) + for name, value in keep: + self.send_header(name, value) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(body[:65536]) + self.wfile.flush() + print(" stalled after 65536 of %d bytes" % len(body), flush=True) + time.sleep(35) + self.close_connection = True + return + self.answer(200, keep, body) + def handle_one(self): rng = self.headers.get("Range") print("%s %s range=%s" % (self.command, self.path, rng or "-"), flush=True) + if MODE == "norange": + self.serve_file() + return if MODE == "503": self.answer(503, [("Content-Length", "0")], b"") return diff --git a/tests/integration/20-client.sh b/tests/integration/20-client.sh index 6807df3..588eccf 100644 --- a/tests/integration/20-client.sh +++ b/tests/integration/20-client.sh @@ -9,8 +9,8 @@ # without DownloadUser, with the signatures from the registry, with a provider # other than the default, through a proxy that answers 503, through a proxy # that cuts every blob in half, through a proxy that stops sending a large -# blob, and through a proxy that changes one byte of the first answer for each -# blob. +# blob, through a proxy that changes one byte of the first answer for each +# blob, and from a plain http Server that ignores Range. # # Needs docker or podman (CONTAINER_RUNTIME), curl, jq, sha256sum, and network # to pull the pinned images and oras. Ports are picked at run time and every @@ -36,9 +36,10 @@ cut_name="client-it-cut-$id" dead_name="client-it-dead-$id" flip_name="client-it-flip-$id" stall_name="client-it-stall-$id" +norange_name="client-it-norange-$id" cleanup() { local out - out="$("$RUNTIME" rm -f "$reg_name" "$cut_name" "$dead_name" "$flip_name" "$stall_name" 2>&1)" || diag "cleanup: $out" + out="$("$RUNTIME" rm -f "$reg_name" "$cut_name" "$dead_name" "$flip_name" "$stall_name" "$norange_name" 2>&1)" || diag "cleanup: $out" if [ "${IT_KEEP:-0}" = 1 ]; then echo "# kept $work" >&2 else @@ -112,13 +113,17 @@ check "a package is pushed as the mirror lays it out" "alpha-1.0-1-any.pkg.tar.z "curl http://$REG/v2/it/client/x86_64/one/manifests/" #---------------------------------------------------------------------------# -# The four proxies, on ports they pick themselves +# The four proxies and the plain server, on ports they pick themselves #---------------------------------------------------------------------------# -# proxy NAME MODE: start one and print its port +# proxy NAME MODE: start one and print its port. The norange mode serves the +# files of the repositories, mounted at /up, and no registry. proxy() { - local p='' + local p='' up="$REG" + if [ "$2" = norange ]; then + up=/up + fi "$RUNTIME" run -d --rm --name "$1" --network host -v "$FIX/fault-proxy.py:/proxy.py:ro" \ - "$PYTHON_IMAGE" python3 -I -u /proxy.py "$2" "$REG" > /dev/null + -v "$work/up:/up:ro" "$PYTHON_IMAGE" python3 -I -u /proxy.py "$2" "$up" > /dev/null for _ in $(seq 1 30); do p="$("$RUNTIME" logs "$1" 2>&1 | awk '$1 == "PORT" { print $2 }')" [ -z "$p" ] || break @@ -130,8 +135,11 @@ cut_port="$(proxy "$cut_name" cut)" dead_port="$(proxy "$dead_name" 503)" flip_port="$(proxy "$flip_name" flip)" stall_port="$(proxy "$stall_name" stall)" -[ -n "$cut_port" ] && [ -n "$dead_port" ] && [ -n "$flip_port" ] && [ -n "$stall_port" ] || stop "the fault proxies start" \ - "cut '$cut_port' 503 '$dead_port' flip '$flip_port' stall '$stall_port'" "reproduce: $RUNTIME logs $cut_name" +norange_port="$(proxy "$norange_name" norange)" +[ -n "$cut_port" ] && [ -n "$dead_port" ] && [ -n "$flip_port" ] && [ -n "$stall_port" ] && [ -n "$norange_port" ] || + stop "the fault proxies start" \ + "cut '$cut_port' 503 '$dead_port' flip '$flip_port' stall '$stall_port' norange '$norange_port'" \ + "reproduce: $RUNTIME logs $cut_name" #---------------------------------------------------------------------------# # The cases @@ -254,6 +262,17 @@ want_cmp stall s_ms lt 20000 "a blob that stops for 40 s: pacman -S big takes le check "stall: the prefetch cut the stopped transfer and resumed it from 64 KiB" 1 "$ranges" \ "$RUNTIME logs $stall_name | grep $big_hex" +# The plain Server ignores Range. The first GET of big stops after 64 KiB for +# 35 s, so the attempt that resumes it starts after 30 s. That attempt gets +# curl exit 33, and the next one reads big from zero. +consumer norange "http://127.0.0.1:$norange_port" +big_gets="$("$RUNTIME" logs "$norange_name" 2>&1 | awk '$1 == "GET" && $2 == "/one/big-1-1-any.pkg.tar.zst" { print $3 }' | paste -sd ' ' -)" +want norange sy_rc 0 "a plain Server that ignores Range: pacman -Sy succeeds" +want norange s_rc 0 "a plain Server that ignores Range: pacman -S big succeeds after its range is refused past 30 s" +want norange installed_big 0 "a plain Server that ignores Range: big is installed" +check "norange: the refused range is followed by one GET from zero" "range=- range=bytes=65536- range=-" "$big_gets" \ + "$RUNTIME logs $norange_name | grep big" + consumer flip "oci+http://127.0.0.1:$flip_port/it/client/x86_64" want flip sy_rc 0 "one byte changed in each first answer: pacman -Sy succeeds" want flip sy_refused 2 "one byte changed in each first answer: the prefetch keeps neither database" diff --git a/tests/unit/31-run-store.sh b/tests/unit/31-run-store.sh index 252112a..f83b63e 100644 --- a/tests/unit/31-run-store.sh +++ b/tests/unit/31-run-store.sh @@ -120,6 +120,7 @@ check "the file a link points to is left as it was" planted "$(cat "$work/target #---------------------------------------------------------------------------# # Markers: a .sig the database manifest does not have misses with no request, # and a host that failed is skipped for the rest of the run, never for a .sig. +# A .sig that fails marks no host. #---------------------------------------------------------------------------# parent 810 K="$$-810" @@ -138,9 +139,29 @@ check "the next package from that host misses with no request" "1:0" "$rc:$(gets call "$d/two-1-1-any.pkg.tar.zst.sig.part" "$U/two-1-1-any.pkg.tar.zst.sig" check "a .sig is still asked of that host" "1:1" "$rc:$(gets)" "client/pacman-oci DIR/two...sig.part URL" +# A signature that gets no answer marks no host down. pacman ignores a failed +# optional signature, then asks the same host for the next file. +start=811 +while IFS='|' read -r label sig next; do + start=$((start + 1)) + parent "$start" + d="$work/sig.$start" + mkdir -p "$d" + call "$d/${sig##*/}.part" "$sig" + marks="$rc:$(gets):$(find "$d" -name ".pacman-oci.$$-$start.,down.*" | wc -l | tr -d ' ')" + call "$d/${next##*/}.part" "$next" + check "$label: no host is marked down, so the next file is still asked for" "1:1:0 1:1" \ + "$marks $rc:$(gets)" \ + "client/pacman-oci DIR/SIG.part '$sig'; ls -a DIR; client/pacman-oci DIR/NEXT.part '$next'" +done << EOF +a database signature|$U/core.db.sig|$U/two-1-1-any.pkg.tar.zst +a package signature|$U/one-1-1-any.pkg.tar.zst.sig|$U/two-1-1-any.pkg.tar.zst +a database signature from a plain Server|http://127.0.0.1:9/core/os/x86_64/core.db.sig|http://127.0.0.1:9/core/os/x86_64/core.db +EOF + #---------------------------------------------------------------------------# # The sweep: entries of runs that are gone go; those of live runs, links, and -# names that are no key stay. +# names that are no key stay. A run with a fresh ,open is live. #---------------------------------------------------------------------------# parent 820 d="$work/sweep" @@ -156,6 +177,22 @@ check "the sweep keeps a live run, a link and a name that is no key" \ ".pacman-oci.4242-5.live-1-1-any.pkg.tar.zst .pacman-oci.999999999-1.link .pacman-oci.notakey " \ "$(entries "$d")" "ls -a DIR after one call" +# A run in another pid namespace has a pid that this /proc cannot show. Its +# ,open had a heartbeat less than 2 minutes ago, so its entries stay. A run +# whose ,open is older than that is gone. +parent 821 +d="$work/heartbeat" +mkdir -p "$d" +: > "$d/.pacman-oci.999999998-2.,open" +: > "$d/.pacman-oci.999999998-2.tmp.AbCdEf" +: > "$d/.pacman-oci.999999997-3.,open" +: > "$d/.pacman-oci.999999997-3.tmp.GhIjKl" +touch -t 200001010000 "$d/.pacman-oci.999999997-3.,open" +call "$d/x.part" "file://$work/no-such-file" +check "the sweep keeps a run whose ,open is fresh, and removes one whose ,open is old" \ + ".pacman-oci.999999998-2.,open .pacman-oci.999999998-2.tmp.AbCdEf " "$(entries "$d")" \ + "ls -a DIR after one call, with the ,open of 999999997-3 at 2000-01-01" + #---------------------------------------------------------------------------# # The background remover: when the run ends, its entries go. #---------------------------------------------------------------------------# @@ -165,6 +202,18 @@ d="$work/reap" mkdir -p "$d" call "$d/foo-1-1-any.pkg.tar.zst.part" "$U/foo-1-1-any.pkg.tar.zst" before="$(find "$d" -name ".pacman-oci.$K.*" | wc -l | tr -d ' ')" +# While the run is live, the remover touches its ,open once a second. +beat='' +if [ -f "$d/.pacman-oci.$K.,open" ]; then + touch -t 200001010000 "$d/.pacman-oci.$K.,open" + for _ in 1 2 3 4 5 6; do + sleep 0.5 + beat="$(find "$d/.pacman-oci.$K.,open" -prune -mmin -2)" + [ -z "$beat" ] || break + done +fi +check "the remover touches ,open within 3 s while the run is live" "$d/.pacman-oci.$K.,open" "$beat" \ + "touch -t 200001010000 DIR/.pacman-oci.KEY.,open; sleep 3; find it -prune -mmin -2" parent 831 left=1 for _ in 1 2 3 4 5 6; do diff --git a/tests/unit/32-prefetch-plan.sh b/tests/unit/32-prefetch-plan.sh index e42d8ab..2ab4465 100644 --- a/tests/unit/32-prefetch-plan.sh +++ b/tests/unit/32-prefetch-plan.sh @@ -165,6 +165,95 @@ check "PACMAN_OCI_SIG=registry: no .sig from %g" "" \ "$(find "$dl" -name ".pacman-oci.$$-$start.*.sig")" "PACMAN_OCI_SIG=registry ...; ls -a DIR" : > "$work/lines" +#---------------------------------------------------------------------------# +# pacman can hold a .part in the cache. %s leaves it out, so the index adds +# its size back, from the file's metadata, as pacman's own stat() does. Under +# DownloadUser, a .part that root left with mode 0600 is not readable. It +# must still have its size, and the prefetch must go on. +# +# The wrapper runs as a user who cannot read locked...part. A user that is +# not root runs it with the file at mode 000. Root runs it as uid 65534, +# through setpriv or busybox su. Each must keep this shell as the parent, and +# a probe checks that first. busybox su also reads options after the user, so +# no argument after it starts with -. +#---------------------------------------------------------------------------# +cache="$work/cache" +dlp="$cache/download-PaRt01" +locked="$cache/locked-1-1-any.pkg.tar.zst.part" +mkdir -p "$dlp" +head -c 1000 /dev/zero > "$cache/held-1-1-any.pkg.tar.zst.part" +head -c 3000 /dev/zero > "$locked" +as_other=() +how='' +why='' +if [ "$(id -u)" -ne 0 ]; then + chmod 000 "$locked" + how="this user, with the .part at mode 000" +else + chmod 755 "$work" + mkdir -p "$work/probe" + chmod 777 "$work/probe" + cat > "$work/probe/probe" << 'EOF' +#!/bin/sh +printf '%s %s\n' "$PPID" "$(id -u)" > "$1" +EOF + chmod 755 "$work/probe/probe" + for try in setpriv su; do + command -v "$try" > /dev/null 2>&1 || continue + case $try in + setpriv) cand=(setpriv --reuid=65534 --regid=65534 --clear-groups --) ;; + *) + # shellcheck disable=SC2016 # the shell su starts expands it, this one must not + cand=(su -s /bin/sh nobody -c 'exec "$0" "$@"') + ;; + esac + rm -f "$work/probe/out" + rc=0 + "${cand[@]}" "$work/probe/probe" "$work/probe/out" > "$work/probe/log" 2>&1 || rc=$? + if [ "$rc" -eq 0 ] && [ -f "$work/probe/out" ] && [ "$(cat "$work/probe/out")" = "$$ 65534" ]; then + as_other=("${cand[@]}") + how="uid 65534 through $try, with the .part of root at mode 0600" + break + fi + printed='' + if [ -f "$work/probe/out" ]; then + printed="$(cat "$work/probe/out")" + fi + why="$why; $try: exit $rc, printed '$printed', said '$(cat "$work/probe/log")'" + done + if [ -n "$how" ]; then + chmod 600 "$locked" + chown 65534:65534 "$dlp" + chmod 700 "$dlp" + fi +fi +start=$((start + 1)) +printf '%s (pacman) S 1 1 1 0 -1 4194560 837 0 0 0 0 0 0 0 20 0 1 0 %s 62676992 3236\n' "$$" "$start" > "$fake/$$/stat" +printf '%s\0' pacman -S held locked > "$fake/$$/cmdline" +{ + printf '%s/held-1-1-any.pkg.tar.zst\t%s\t24\t%s\n' "$U" "$H1" "$SIG" + printf '%s/locked-1-1-any.pkg.tar.zst\t%s\t7\t%s\n' "$U" "$H2" "$SIG" +} > "$work/lines" +rc=0 +FAKE_ARGS=/dev/null PACMAN_OCI_DEBUG=1 ${as_other[@]+"${as_other[@]}"} "$W" "$dlp/held-1-1-any.pkg.tar.zst.part" "$U/held-1-1-any.pkg.tar.zst" > /dev/null 2> "$work/err" || rc=$? +index="$dlp/.pacman-oci.$$-$start.,index" +if [ -f "$index" ]; then + got="$(tr '\t' ' ' < "$index" | paste -sd '|' -)" +else + got="no index; the wrapper ended $rc: $(awk '!/^pacman-oci: /' "$work/err" | paste -sd ' ' -)" +fi +check "a .part pacman holds is added back to the planned size" \ + "held-1-1-any.pkg.tar.zst $H1 1024" "${got%%|*}" "head -c 1000 > CACHE/held...part; a print line with %s 24; cat DIR/.pacman-oci.KEY.,index" +if [ -n "$how" ]; then + check "a .part the wrapper cannot read is sized from its metadata ($how)" \ + "held-1-1-any.pkg.tar.zst $H1 1024|locked-1-1-any.pkg.tar.zst $H2 3007" "$got" \ + "an unreadable CACHE/locked...part of 3000 bytes; a print line with %s 7; cat DIR/.pacman-oci.KEY.,index" +else + diag "no command here runs the wrapper as uid 65534 with this shell ($$) as its parent${why:-; no setpriv and no su}" + ok "skipped: no way here to run the wrapper as a user who cannot read a .part" +fi +: > "$work/lines" + #---------------------------------------------------------------------------# # The database prefetch reads pacman's config through pacman-conf, never # with --sysroot. From b48cdb8452c210af4d607f55a418ff61ea7fe531 Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 22:17:26 +0545 Subject: [PATCH 20/22] workflows: fix review findings Pages artifact named per run attempt; report starts status.yml only when active; ci.yml group per run outside pull requests; sync.yml queue: max; status sums, badges and page say over how many targets they count, a registry that did not answer is unknown, the history carries a target's last measured files; worker-down needs 50 minutes as well as 2 runs; check-staleness grace from the enabled date; a stale bench shows its age. --- .github/workflows/ci.yml | 10 +- .github/workflows/status.yml | 27 +- .github/workflows/sync.yml | 25 +- docs/operations.md | 46 ++- docs/tests.md | 20 +- scripts/check-staleness | 32 ++- scripts/status-collect | 113 ++++++-- scripts/status-render | 198 ++++++++++--- tests/static/20-actionlint.sh | 11 +- tests/static/50-workflow-policy.sh | 448 +++++++++++++++++++++++++++++ tests/unit/60-status.sh | 187 +++++++++++- 11 files changed, 1005 insertions(+), 112 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b68504..f64ae37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,11 +28,13 @@ on: permissions: {} -# One group per pull request, or per ref. A newer push to a pull request -# cancels the run in progress; a run on main always finishes, so each commit -# on main gets a verdict. +# One group per pull request: a newer push to it cancels the run in progress. +# Any other run has a group of its own (its run id). A group holds one pending +# run, and a newer run cancels it, so a shared group for main would leave a +# commit between two quick pushes with no verdict. No job writes shared state, +# so runs on main need not wait for each other. concurrency: - group: ci-${{ github.event.pull_request.number || github.ref }} + group: ci-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} defaults: diff --git a/.github/workflows/status.yml b/.github/workflows/status.yml index 20e8bd1..3a5612d 100644 --- a/.github/workflows/status.yml +++ b/.github/workflows/status.yml @@ -155,6 +155,12 @@ jobs: timeout-minutes: 30 permissions: contents: write + # One Pages artifact name per run attempt. A re-run that runs build again + # uploads a second artifact into the same run, and deploy-pages refuses a + # name that two artifacts share. deploy reads the name from here, so a + # re-run of deploy alone takes the name that build used. + outputs: + pages_artifact: github-pages-${{ github.run_attempt }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -242,6 +248,7 @@ jobs: - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 if: ${{ github.ref == 'refs/heads/main' }} with: + name: github-pages-${{ github.run_attempt }} path: ${{ runner.temp }}/site # Each of these names its needs' results: with no status function an if gets @@ -289,6 +296,8 @@ jobs: id: deploy if: ${{ steps.pages.outputs.deploy == 'true' }} uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 + with: + artifact_name: ${{ needs.build.outputs.pages_artifact }} # The Worker check reads the record build has just pushed to the status # branch. Here, not in watchdog.yml: an hourly cron there would cancel a @@ -305,13 +314,15 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Raise worker-down after 2 failed status runs in a row + - name: Raise worker-down after failed probes in 2 runs in a row, over 50 minutes env: GH_TOKEN: ${{ github.token }} run: | - # status.yml probes the Worker each hour and counts the runs in a - # row with a failed probe (scripts/status-collect). This reads its - # last record from the status branch. + # status.yml probes the Worker each hour, and once more after each + # sync, so two runs can be minutes apart. scripts/status-collect + # counts the runs in a row with a failed probe and the time since + # the first; it sets worker.down when both reach its limits. This + # reads its last record from the status branch. rc=0 git ls-remote --exit-code --heads origin status > "$RUNNER_TEMP/heads.txt" || rc=$? case "$rc" in @@ -328,8 +339,8 @@ jobs: git fetch --quiet --depth 1 origin status git show FETCH_HEAD:status.json > "$RUNNER_TEMP/status.json" git show FETCH_HEAD:worker.md > "$RUNNER_TEMP/worker.md" - read -r generated age failed ok <<< "$(jq -r '[.generated, (now - .generated_epoch | floor), .worker.failed_runs, .worker.ok] | map(tostring) | join(" ")' "$RUNNER_TEMP/status.json")" - echo "status.json from $generated, $age s ago: Worker ok $ok, failed $failed run(s) in a row" + read -r generated age failed ok down span <<< "$(jq -r '[.generated, (now - .generated_epoch | floor), .worker.failed_runs, .worker.ok, .worker.down, .worker.failing_seconds] | map(tostring) | join(" ")' "$RUNNER_TEMP/status.json")" + echo "status.json from $generated, $age s ago: Worker ok $ok, failed $failed run(s) in a row over $span s, down $down" # Three status runs missed: the record says nothing about now. if [ "$age" -gt 10800 ]; then echo "::error::the last Worker probe is from $generated, $age s ago; status.yml has stopped recording" @@ -338,8 +349,8 @@ jobs: fi if [ "$ok" = true ]; then scripts/track-issue clear worker-down "The Worker passes its probes again, measured $generated: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/status/worker.md" - elif [ "$ok" = false ] && [ "$failed" -ge 2 ]; then + elif [ "$down" = true ]; then scripts/track-issue raise worker-down "The Worker fails its probe" "$RUNNER_TEMP/worker.md" else - echo "nothing to raise or clear: worker-down opens at 2 failed runs in a row" + echo "nothing to raise or clear: worker-down opens at failed probes in 2 runs in a row over 50 minutes or more (status.json .worker.down_after)" fi diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 0dcbcb9..9088890 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -51,9 +51,16 @@ on: permissions: {} +# With the default queue (single), a group holds one pending run, and a newer +# run cancels it: a dispatch with allow_older could be replaced by the next +# scheduled run. queue: max keeps up to 100 pending runs, each in its turn +# (docs.github.com, "Control the concurrency of workflows and jobs"). +# actionlint 1.7.12 does not know the key: tests/static/20-actionlint.sh +# ignores that one message. concurrency: group: sync cancel-in-progress: false + queue: max defaults: run: @@ -316,11 +323,27 @@ jobs: run: | mkdir -p "$RUNNER_TEMP/dl-plan" "$RUNNER_TEMP/summaries" scripts/report "$RUNNER_TEMP/dl-plan" "$RUNNER_TEMP/summaries" + # GitHub refuses to start a workflow that is not active, and a person + # may disable status.yml on purpose: a notice, as in + # scripts/dispatch-overdue, not a red sync. A state that does not read + # still fails the step. - name: Start the status workflow if: ${{ always() && github.ref == 'refs/heads/main' }} env: GH_TOKEN: ${{ github.token }} - run: gh workflow run status.yml --ref main --repo "$GITHUB_REPOSITORY" + run: | + rc=0 + state="$(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/status.yml" --jq .state)" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "::error::could not read the state of status.yml (gh ended $rc)" + echo " reproduce: gh api repos/$GITHUB_REPOSITORY/actions/workflows/status.yml --jq .state" + exit "$rc" + fi + if [ "$state" = active ]; then + gh workflow run status.yml --ref main --repo "$GITHUB_REPOSITORY" + else + echo "::notice::status.yml is $state; it is not started" + fi # No needs, in either direction: this runs whatever the sync does, and the # watchdog workflow runs the same two checks, so each witnesses the other. diff --git a/docs/operations.md b/docs/operations.md index b109de3..263fe96 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -33,8 +33,24 @@ This document tells what runs by itself, which issues the automation raises and `build` runs when `decide` passed, also when `bench` was skipped or failed. `deploy` and `worker` name the result of `build` in their `if`. An `if` with no status function gets an implicit `success()` over every job before it. `bench` is skipped in most runs, so that implicit check would skip `deploy` and `worker` too. +The Pages artifact of `build` is named `github-pages-`. `deploy` reads that name from the outputs of `build`. A re-run that runs `build` again uploads a second artifact into the same run, and `deploy-pages` refuses a name that two artifacts share. Each `upload-artifact` step sets `overwrite: true` for the same reason. `tests/static/50-workflow-policy.sh` checks both. + The Worker check is a job of `status.yml`, not of `watchdog.yml`. An hourly cron in `watchdog.yml` would cancel a pending daily run, because both share one concurrency group. +### Concurrency + +⚠ By default a concurrency group holds one running run and one pending run. A newer run cancels the pending one. `cancel-in-progress: false` keeps only the running run. + +| workflow | group | queue | effect | +| --- | --- | --- | --- | +| `sync.yml` | `sync`, for every ref | `max` | up to 100 runs wait, each in its turn; a dispatch with `allow_older` is never replaced by the next scheduled run | +| `ci.yml` | `ci-pr-` for a pull request, else `ci-` | default | a newer push to a pull request cancels its run in progress; every other run has a group of its own, so each push to `main` gets a verdict | +| `status.yml` | `status` | default | one run waits; a newer one replaces it, and each run measures everything again | +| `watchdog.yml` | `watchdog` | default | one run waits | +| `worker.yml` | `worker-` | default | one run waits | + +GitHub refuses `queue: max` with `cancel-in-progress: true`. actionlint 1.7.12 does not know the key `queue`, so `tests/static/20-actionlint.sh` ignores that one message, and `tests/static/50-workflow-policy.sh` checks the value instead. When a long seed holds the group `sync`, the scheduled runs queue behind it and then run one after another. + ## Each scheduled workflow starts the others ⛔ GitHub can delay or drop a scheduled run. A dropped run leaves nothing behind that could notice. So each scheduled workflow runs `scripts/dispatch-overdue WORKFLOW MAX_MINUTES` for the others. @@ -51,6 +67,8 @@ A run that is queued or in progress counts as fresh, so nothing starts twice. A `dispatch-overdue` starts only a workflow whose state is `active`. GitHub refuses to start a disabled workflow. The script skips a workflow that a person disabled, or that GitHub disabled for inactivity. It prints a notice and exits 0. In `sync.yml` and `watchdog.yml`, `keep-schedules-enabled` runs first, and enables again a workflow that GitHub disabled for inactivity. +The job `report` of `sync.yml` starts `status.yml` after each sync on `main`, whether or not it is overdue. It follows the same rule: when `status.yml` is not `active`, it prints a notice and the sync stays green. When the state does not read, the step fails. `tests/static/50-workflow-policy.sh` runs each workflow step that calls `gh workflow run` against a stub `gh`, with the workflow active, disabled and unreadable. + The restart steps in `sync.yml` and `watchdog.yml` run with `always()`. So a schedule check that fails earlier in the job does not stop the restart. A step with two `dispatch-overdue` calls runs both, then exits with the status of the last call that failed. `scripts/keep-schedules-enabled` runs in `sync.yml` and in `watchdog.yml`. GitHub disables the schedules of a repository after 60 days with no activity. The script enables each workflow of the tree again. ⚠ A workflow that a person disabled stays disabled, and the script names it. @@ -67,13 +85,13 @@ The restart steps in `sync.yml` and `watchdog.yml` run with `always()`. So a sch | --- | --- | | `current` | the two databases are the same | | `fresh` | they differ, and the promoted snapshot was fetched within the tolerance | -| `new` | there is no `latest`, and git added the target file within the tolerance | +| `new` | there is no `latest`, and git added or enabled the target file within the tolerance | | `stale` | they differ for longer than the tolerance, or there is no `latest` for longer | | `unknown` | upstream could not be read | The tolerance comes from `scripts/cron-tolerance`: the largest gap in days of the cron, times 2, plus 1. The 3-hour cron of `sync.yml` gives a gap of 1 day and a tolerance of 3 days. ⚠ The tolerance is in days. A sub-daily cron gets the tolerance of a daily one, so a stop shows later than it could. -The date a target file was added is the date of the commit that brought it to the checked-out branch (`git log --first-parent`). A commit on the branch it came from does not count. +The date a target file was added is the date of the commit that brought it to the checked-out branch (`git log --first-parent`). A commit on the branch it came from does not count. The date it was enabled is the date of the last such commit that changed its `enabled` line. The later of the two dates counts. So a target file added with `enabled = no` gets its grace from the day that line goes or turns to `yes`. ⚠ When git cannot tell when a target file was added (no git, a shallow clone, an untracked file), the target is stale, never new. The `staleness` job checks out the whole history for this. @@ -86,12 +104,14 @@ The date a target file was added is the date of the commit that brought it to th | `visibility-` | the package reads with the workflow's token and not without one | a run reads the target anonymously | `scripts/report`, on `main` | | `replaced-` | a package tag holds other bytes than the database names | the target stages or is current again | `scripts/report`, on `main` | | `stale-` | `check-staleness` says `stale` or `unknown` | it says anything else | `watchdog.yml` job `staleness` | -| `worker-down` | the Worker failed its probe in 2 status runs in a row | a status run records a probe that passes | `status.yml` job `worker` | +| `worker-down` | the Worker failed its probe in 2 status runs in a row, over 50 minutes or more | a status run records a probe that passes | `status.yml` job `worker` | A target that is held, behind or in error raises nothing in the report. A run that fails once is ordinary. `stale-` opens when the mirror has stopped following upstream. The `worker` job of `status.yml` reads the record that `build` pushed to branch `status` in the same run. The job fails when that record is more than 3 hours old. +Status runs are not an hour apart. The cron runs `status.yml` at minute 47 of each hour, and `sync.yml` starts it after each sync, so two runs can be minutes apart. So `scripts/status-collect` sets `worker.down` in `status.json` only when two conditions are true. The probes failed in 2 runs in a row. The first of these failures is 3000 s (50 minutes) or more before this run. The `worker` job raises `worker-down` on `worker.down`. A Worker or GHCR fault of a few minutes opens no issue. + ## What needs a person - **A private package.** GitHub may create a package private, and has no API to change that. Open the settings link in `visibility-`, and make the package public. @@ -132,7 +152,7 @@ gh workflow run sync.yml --ref main -f targets='x86_64-core x86_64-extra' -f max | Actions can approve pull requests | no | | `sha_pinning_required` | `true`: only actions pinned by commit run | -`tests/static/50-workflow-policy.sh` checks the shape of every workflow. It checks `permissions: {}` at the top, the permissions of each job, and `timeout-minutes` on every job. It also checks that each action is pinned by commit, and that no untrusted value is inside a `run:` block. +`tests/static/50-workflow-policy.sh` checks the shape of every workflow. It checks `permissions: {}` at the top, the permissions of each job, and `timeout-minutes` on every job. It also checks that each action is pinned by commit, and that no untrusted value is inside a `run:` block. It also checks the artifact names against re-runs, the steps that start another workflow, and the concurrency groups (above). ## Secrets @@ -162,7 +182,7 @@ The site at holds: | `badges/.svg` | coverage and snapshot age of one target | | `badges/.svg` | the same over the targets of one architecture | | `badges/all.svg` | the same over every target | -| `badges/speed.svg` | pacman's time through the wrapper and through the official mirror, from the bench | +| `badges/speed.svg` | pacman's time through the wrapper and through the official mirror, from the bench; a bench older than 2 days shows its age, in the warning colour | | `charts/coverage.svg` | share of upstream files in the mirror, per target | | `charts/freshness.svg` | snapshot age per target, against the sync interval | | `charts/size.svg` | GiB per target, in the mirror and upstream | @@ -174,13 +194,27 @@ The site at holds: `status.json` has schema 1. The header of `scripts/status-collect` describes each field. +A target has a coverage only when a run reads both upstream and the registry. A sum over the targets counts only those, and the page says so: + +| where | every target read | some targets not read | +| --- | --- | --- | +| `badges/.svg` | `77.7% · 3.0 h` | `77.7% · 3.0 h · 2 of 3 measured` | +| `badges/all.svg` | `53.8% · 2/5 repos` | `53.8% · 2/5 repos · 4 of 5 measured` | +| the headline | over 5 repositories | over the 4 of 5 repositories measured in this run | +| the tiles `Files` and `Size in the mirror` | of the upstream total | of the upstream total, in 4 of 5 repositories | +| the tile `Repositories with a snapshot` | `2 of 5` | `1 of 4 read`, and how many the registry did not answer | + +A group badge with no snapshot says `unknown` when its registry did not answer. It says `not mirrored` only when the registry says there is no `latest`. When no target is read on both sides, the headline and the file and size tiles say `unknown`. + +In `status.json`, `summary.measured` counts the targets read on both sides, and `summary.mirror_unread` the targets whose registry did not answer. The file and byte sums are `null` when no target was read. + ## The status branch On `main`, job `build` pushes branch `status` as one orphan commit, with force, on each run. The branch holds the site, plus: | file | contents | | --- | --- | -| `history.jsonl` | one line per status run; the last 2000 lines are kept | +| `history.jsonl` | one line per status run; the last 2000 lines are kept. A target that a run could not read keeps the files and bytes of its last measured line, marked `"k": 1`. So a read that fails does not move the coverage history | | `worker.md` | the body of the `worker-down` issue: every probe, and how to reproduce the first failure | | `bench.json` | the newest bench | | `bench/.json` | each daily bench | diff --git a/docs/tests.md b/docs/tests.md index e6ee5f2..0c07575 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -12,11 +12,11 @@ | file | checks | | --- | --- | | `static/10-shellcheck.sh` | every shell file outside `tests/fixtures/` passes shellcheck in the dialect of its first line | -| `static/20-actionlint.sh` | every workflow passes actionlint | +| `static/20-actionlint.sh` | every workflow passes actionlint; only the message on the concurrency key `queue`, which actionlint 1.7.12 does not know, is ignored | | `static/30-failures-stay-visible.sh` | no shipped script or workflow hides a failure; every curl call has a deadline, and so does every oras call in `scripts/` | | `static/35-pipeline-traps.sh` | no pipeline in a shipped bash script ends in a reader that stops early; `client/pacman-oci` runs with no `pipefail` and is left out | | `static/40-executable-bits.sh` | a shipped file with a `#!` line is 100755 in the git index; one without is 100644 | -| `static/50-workflow-policy.sh` | the shape of every workflow: permissions, timeouts, pins, no untrusted value in `run:` | +| `static/50-workflow-policy.sh` | the shape of every workflow: permissions, timeouts, pins, no untrusted value in `run:`; artifact names that survive a re-run; each step that calls `gh workflow run`, run against a stub `gh`; the concurrency group of each kind of `ci.yml` run, evaluated; the concurrency queues | | `static/60-targets.sh` | every target file parses and pins its images; one verify image; a plan slot for each target | | `static/70-python.sh` | every python file compiles; every file in `client/` and `scripts/` is shell or python | | `static/72-test-runner.sh` | `tests/run.sh` finds the files, runs one file, gates a live test, and fails a failed run | @@ -28,7 +28,7 @@ | `unit/32-prefetch-plan.sh` | the argument rewrite, the print lines, the signatures written from `%g` | | `unit/40-lib.sh` | the decisions of the sync on fixed inputs | | `unit/45-dispatch-overdue.sh` | `scripts/dispatch-overdue` starts a workflow only when it is active and overdue | -| `unit/60-status.sh` | `status-collect` and `status-render` on fixtures | +| `unit/60-status.sh` | `status-collect`, `status-render` and `check-staleness` on fixtures, also with one registry read that fails and with no registry read at all | | `integration/10-pipeline.sh` | the whole mirror against its own registry and upstream | | `integration/20-client.sh` | the wrapper against its own registry, through faulty proxies | | `worker/10-node.sh` | `node --test 'worker/test/*.test.js'` | @@ -228,6 +228,13 @@ The tree itself is never changed. An integration fault runs with its own registr | unit/60-status | the grace limit 30 days, not the tolerance | not ok: `check-staleness: no latest, added 13 days ago, tolerance 3: stale`, with the exit status and the added date | | unit/60-status | the shallow-clone check never true | not ok: `check-staleness: in a shallow clone the age is unknown, so stale, never new`, `check-staleness: says that the clone is shallow` | | unit/60-status | `read_json` returns a stub status | not ok: `status-render with no status.json exits 1`, `and names the file it looked for` | +| unit/60-status | `status-render` of before the partial-measurement rule | 52 not ok, among them `site5: badge x86_64 carries its text: x86_64 \| 100% \| 4.0 h \| 1 of 3 measured` (it said `100% · 4.0 h`), `site6: badge x86_64 does not say not mirrored` and `site7: badge speed carries its text: ... 73 d ago` | +| unit/60-status | `status-collect` of before `measured`, `worker.down` and the carried history | 10 not ok, among them `summary: no registry read: 0 promoted, 5 not read, 0 measured, every sum unknown` (got `0 null null 0 0 0 0 null`) and `history: x86_64-extra keeps its last measured files, marked carried` | +| unit/60-status | `WORKER_DOWN_SECONDS=0` | not ok: `worker: two failed runs 7 minutes apart are not down; down needs 50 minutes` | +| unit/60-status | no history carried for a target that a run did not read | not ok: `history: x86_64-extra keeps its last measured files, marked carried`, `site5: index.html says: x86_64 2026-10-14 03:00 77.7% 77.7%` | +| unit/60-status | `BENCH_MAX_AGE` 100 days | 4 not ok, among them `site7: a bench older than 2 days gives the speed badge the warning colour` | +| unit/60-status | a group with no snapshot is `not mirrored` also when its registry did not answer | 6 not ok, among them `site6: badge all does not say not mirrored` | +| unit/60-status | `check-staleness` of before the enabled date | not ok: `check-staleness: no latest, added disabled 43 days ago, enabled 1 day ago: new` | ### worker @@ -277,6 +284,13 @@ The tree itself is never changed. An integration fault runs with its own registr | static/50-workflow-policy | a call to the packages DELETE API | not ok: `no script or workflow deletes a package or a version` | | static/50-workflow-policy | `pages: write` in `ci.yml` job `static`; `contents: write` in `sync.yml` job `plan`; `id-token: write` in `watchdog.yml` job `staleness` | not ok: `pages, id-token and contents write only where the status site needs them`, once for each workflow | | static/50-workflow-policy | `keep-schedules-enabled` skips `watchdog.yml` | not ok: `.github/workflows/watchdog.yml has a schedule, and keep-schedules-enabled enables it` | +| static/50-workflow-policy | `status.yml` of before the per-attempt Pages artifact; then the output of `build` renamed `pages-...` | not ok: `.github/workflows/status.yml: a re-run does not stop on an artifact of the same name`, each time | +| static/50-workflow-policy | `overwrite: true` removed from one `upload-artifact` of `sync-target.yml` | not ok: `.github/workflows/sync-target.yml: a re-run does not stop on an artifact of the same name` | +| static/50-workflow-policy | `sync.yml` of before the state check in `report` | not ok: `.github/workflows/sync.yml, the run: block at line 323: skips a workflow that is not active, and names its state` (exit 1 on the stub's HTTP 422) | +| static/50-workflow-policy | `exit 0` where the state does not read | not ok: `... fails when the workflow's state does not read` | +| static/50-workflow-policy | `ci.yml` of before the run-id group | not ok: `.github/workflows/ci.yml: two pushes to main and a dispatch get three concurrency groups, so none cancels another's pending run` (all three `ci-refs/heads/main`) | +| static/50-workflow-policy | `sync.yml` with no `queue: max`; then with `cancel-in-progress: true` | not ok: `.github/workflows/sync.yml: the group sync queues every pending run, so a dispatch waits its turn`; then also `every concurrency queue is single or max, and max never cancels in progress` | +| static/20-actionlint | the `queue` ignore removed | not ok: `actionlint .github/workflows/sync.yml` (`unexpected key "queue" for "concurrency" section`) | | static/60-targets | a verify image pinned without a tag | not ok: `targets/x86_64-core.conf pins its verify_image as name:tag@sha256` | | static/70-python | a `#!python3` file with a syntax error | not ok: `python3 -I -m py_compile scripts/status-render` | | static/70-python | a data file `scripts/notes.txt` | not ok: `scripts/notes.txt is a shell or a python script` | diff --git a/scripts/check-staleness b/scripts/check-staleness index ed494a5..ae454d8 100755 --- a/scripts/check-staleness +++ b/scripts/check-staleness @@ -16,11 +16,13 @@ # run that is green and promotes nothing, a target held run after run by one # bad file: each ends here, as a database that has stopped following upstream. # -# A target with no latest at all, whose file git added no more than the -# tolerance ago, is new: the sync has not seeded it yet. Older, it is stale. ⚠ -# When git cannot say when the file was added (no git, a shallow clone, a file -# git does not track), it is stale: an unknown age is never a grace period. -# watchdog.yml checks out the whole history for this. +# A target with no latest at all, whose file git added or enabled no more than +# the tolerance ago, is new: the sync has not seeded it yet. Older, it is +# stale. Enabled is the last change to the file's enabled line: a file added +# with "enabled = no" starts its grace when that line goes. ⚠ When git cannot +# say when the file was added (no git, a shallow clone, a file git does not +# track), it is stale: an unknown age is never a grace period. watchdog.yml +# checks out the whole history for this. # # OUT/.state current fresh new stale unknown # OUT/.md the issue body for a stale or unknown target @@ -59,10 +61,11 @@ run_url="${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:-pkgforge- work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT -# added_date CONF -> YYYY-MM-DD of the last commit that added CONF, or nothing -# when git cannot say; what git said then goes to stderr. +# added_date CONF -> YYYY-MM-DD of the later of two commits: the last that +# added CONF, and the last that changed its enabled line. Nothing when git +# cannot say; what git said then goes to stderr. added_date() { - local dir base shallow rc=0 + local dir base shallow added enabled rc=0 if ! command -v git > /dev/null; then log "git is not on PATH, so the date $1 was added is unknown" return 0 @@ -80,8 +83,13 @@ added_date() { return 0 fi # --first-parent: the date CONF reached this branch (the merge or squash - # commit), not the date of a commit on the branch it came from. - git -C "$dir" log --first-parent --diff-filter=A --format=%cs -- "$base" | awk 'NR == 1' + # commit), not the date of a commit on the branch it came from. git diffs a + # merge against its first parent here, so -G sees an enabled line that a + # merge changed. + added="$(git -C "$dir" log --first-parent --diff-filter=A --format=%cs -- "$base" | awk 'NR == 1')" + enabled="$(git -C "$dir" log -1 --first-parent --format=%cs -G '^[[:space:]]*enabled[[:space:]]*=' -- "$base")" + # YYYY-MM-DD sorts as text. + printf '%s\n' "$added" "$enabled" | awk 'NF' | sort | tail -n 1 } worst=0 @@ -132,7 +140,7 @@ for conf in "$@"; do printf '%s\n' "$verdict" > "$out/$name.state" printf '%-24s %-8s promoted %s, fetched %s, upstream %s, tolerance %s day(s)%s\n' \ "$name" "$verdict" "${promoted:-none}" "${fetched:-never}" "${upstream:-unreadable}" "$tolerance" \ - "${added:+, target added $added}" + "${added:+, target added or enabled $added}" case "$verdict" in current | fresh | new) continue ;; @@ -148,7 +156,7 @@ for conf in "$@"; do printf '| fetched | %s |\n' "${fetched:-never}" printf '| upstream now | `%s` |\n' "${upstream:-no db_server answered}" if [ "$latest_absent" -eq 1 ]; then - printf '| target file added | %s |\n' "${added:-unknown to git}" + printf '| target file added or enabled | %s |\n' "${added:-unknown to git}" fi printf '| tolerance | %s day(s), from sync.yml'"'"'s cron |\n\n' "$tolerance" printf 'The latest sync runs and their summaries: %s\n\n' "$run_url" diff --git a/scripts/status-collect b/scripts/status-collect index 8295df2..738264a 100755 --- a/scripts/status-collect +++ b/scripts/status-collect @@ -33,12 +33,23 @@ # schema 1, generated, generated_epoch, source, site_url, registry, # namespace, worker_url # sync {workflow, interval_minutes, tolerance_days}, from sync.yml's cron -# summary {targets, promoted, current, upstream_files, upstream_bytes, -# mirrored_files, mirrored_bytes, coverage_files, coverage_bytes, -# oldest_age_seconds, newest_age_seconds} +# summary {targets, promoted, current, measured, mirror_unread, +# upstream_files, upstream_bytes, mirrored_files, mirrored_bytes, +# coverage_files, coverage_bytes, oldest_age_seconds, +# newest_age_seconds} +# measured counts the targets with a coverage, read on both +# sides. The files, bytes and coverage sums are over those +# targets only, and null when there is none. mirror_unread counts +# the targets whose registry did not answer (mirror state error): +# promoted does not count them, whatever they hold. # worker {probed, ok, probes, failures, failed_runs, failing_since, +# failing_seconds, down, down_after {runs, seconds}, # median_seconds}; failed_runs counts the runs in a row, this one -# included, in which a probe failed +# included, in which a probe failed. failing_seconds is the time +# from failing_since to this run. down is true when failed_runs +# and failing_seconds both reach down_after: status.yml raises +# worker-down then. Runs are not an hour apart: sync.yml also +# starts a status run when it ends. # ghcr {median_manifest_seconds} # targets[] {target, arch, repo, repository, # upstream {ok, server, sha256, files, bytes, newest_builddate, tried[] @@ -48,7 +59,8 @@ # newest_builddate}, # current true when the mirror database is upstream's database, # coverage {files, of_files, ratio_files, bytes, of_bytes, ratio_bytes, -# missing, missing_names[]} or null when upstream is unread, +# missing, missing_names[]} or null when upstream or the +# registry is unread (mirror state error), # age_seconds, lag_seconds, # worker {db, package}, each {url, code, expect, seconds, location, ok, # error} or null} @@ -56,8 +68,11 @@ # history.jsonl, one object per line: # {"t": generated, "w": {"ok", "f": failed_runs, "s": failing_since, "ms"}, # "g": {"ms"}, "x": {"": {"n": upstream files, "m": mirrored -# files, "b": upstream bytes, "mb": mirrored bytes, "a": age, "l": lag, -# "c": 1 current, 0 not, null unknown}}} +# files, "b": upstream bytes, "mb": mirrored bytes, "k": 1 carried, "a": +# age, "l": lag, "c": 1 current, 0 not, null unknown}}} +# A target with no coverage in this run keeps n, m, b and mb from the last +# line that has them, with "k": 1. So a read that fails once does not move +# the coverage history of its architecture. With no such line they are null. # # Environment: # MIRROR_NAMESPACE owner/name, default from GITHUB_REPOSITORY @@ -93,6 +108,11 @@ WORKER_URL="${STATUS_WORKER_URL-https://ghcr.mirror.archlinux.pkgforge.dev}" WORKER_URL="${WORKER_URL%/}" HISTORY_LINES="${STATUS_HISTORY_LINES:-2000}" [[ "$HISTORY_LINES" =~ ^[1-9][0-9]*$ ]] || die "STATUS_HISTORY_LINES must be a positive integer, not '$HISTORY_LINES'" +# worker-down: failed probes in this many runs in a row, over at least this +# many seconds. The cron runs status.yml at :47 each hour, and sync.yml starts +# it after each sync, so two runs can be minutes apart. +WORKER_DOWN_RUNS=2 +WORKER_DOWN_SECONDS=3000 owner="${MIRROR_NAMESPACE%%/*}" name_part="${MIRROR_NAMESPACE#*/}" SITE_URL="${STATUS_SITE_URL:-https://$owner.github.io/${name_part%%/*}/}" @@ -424,6 +444,11 @@ if [ -s "$out/history.jsonl" ]; then if [ "$rc" -eq 0 ] && [[ "$parsed" =~ ^([0-9]+)\ ?(.*)$ ]]; then prev_f="${BASH_REMATCH[1]}" prev_s="${BASH_REMATCH[2]}" + if [ -n "$prev_s" ] && ! [[ "$prev_s" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then + warn "the last line of $out/history.jsonl names a failing time '$prev_s' that is not ISO 8601 UTC; the Worker failure time starts again" \ + "reproduce: tail -n 1 $out/history.jsonl | jq .w" + prev_s="" + fi else warn "the last line of $out/history.jsonl does not read; the Worker failure count starts again" \ "$(cat "$work/jq.err")" "reproduce: tail -n 1 $out/history.jsonl | jq ." @@ -435,10 +460,12 @@ jq -s \ --arg site "$SITE_URL" --arg registry "$MIRROR_REGISTRY" --arg ns "$MIRROR_NAMESPACE" \ --arg worker "$WORKER_URL" --argjson interval "$(num "$interval")" \ --argjson tolerance "$(num "${tolerance:-}")" \ - --argjson prev_f "$prev_f" --arg prev_s "$prev_s" ' + --argjson prev_f "$prev_f" --arg prev_s "$prev_s" \ + --argjson down_runs "$WORKER_DOWN_RUNS" --argjson down_secs "$WORKER_DOWN_SECONDS" ' def median: sort | if length == 0 then null elif length % 2 == 1 then .[length / 2 | floor] else (.[length / 2 - 1] + .[length / 2]) / 2 end; - def sum_of(f): map(f // 0) | add // 0; + # A sum over the measured targets; null when none was measured, not 0. + def total(f): if length == 0 then null else map(f // 0) | add end; . as $t | ([$t[] | .worker.db, .worker.package | select(. != null)]) as $probes | ([$probes[] | select(.ok == false)] | length) as $fail @@ -446,7 +473,14 @@ jq -s \ elif $fail > 0 then false elif ([$probes[] | select(.ok == true)] | length) > 0 then true else null end) as $wok + | (if $wok == false then $prev_f + 1 else 0 end) as $f_runs + | (if $wok == false then (if $prev_f > 0 and $prev_s != "" then $prev_s else $generated end) else null end) as $f_since + | (if $f_since == null then null else $now - ($f_since | fromdateiso8601) end) as $f_secs | ([$t[] | select(.coverage != null)]) as $cov + | ($cov | total(.coverage.of_files)) as $nf + | ($cov | total(.coverage.files)) as $mf + | ($cov | total(.coverage.of_bytes)) as $nb + | ($cov | total(.coverage.bytes)) as $mb | ([$t[] | .age_seconds | select(. != null)]) as $ages | { schema: 1, @@ -462,14 +496,14 @@ jq -s \ targets: ($t | length), promoted: ([$t[] | select(.mirror.state == "promoted")] | length), current: ([$t[] | select(.current == true)] | length), - upstream_files: ($cov | sum_of(.coverage.of_files)), - upstream_bytes: ($cov | sum_of(.coverage.of_bytes)), - mirrored_files: ($cov | sum_of(.coverage.files)), - mirrored_bytes: ($cov | sum_of(.coverage.bytes)), - coverage_files: (($cov | sum_of(.coverage.of_files)) as $n - | if $n > 0 then ($cov | sum_of(.coverage.files)) / $n else null end), - coverage_bytes: (($cov | sum_of(.coverage.of_bytes)) as $n - | if $n > 0 then ($cov | sum_of(.coverage.bytes)) / $n else null end), + measured: ($cov | length), + mirror_unread: ([$t[] | select(.mirror.state == "error")] | length), + upstream_files: $nf, + upstream_bytes: $nb, + mirrored_files: $mf, + mirrored_bytes: $mb, + coverage_files: (if ($nf // 0) > 0 then $mf / $nf else null end), + coverage_bytes: (if ($nb // 0) > 0 then $mb / $nb else null end), oldest_age_seconds: ($ages | max), newest_age_seconds: ($ages | min) }, @@ -478,8 +512,11 @@ jq -s \ ok: $wok, probes: ($probes | length), failures: $fail, - failed_runs: (if $wok == false then $prev_f + 1 else 0 end), - failing_since: (if $wok == false then (if $prev_f > 0 and $prev_s != "" then $prev_s else $generated end) else null end), + failed_runs: $f_runs, + failing_since: $f_since, + failing_seconds: $f_secs, + down: ($wok == false and $f_runs >= $down_runs and $f_secs >= $down_secs), + down_after: {runs: $down_runs, seconds: $down_secs}, median_seconds: ([$probes[] | .seconds | select(. != null)] | median) }, ghcr: {median_manifest_seconds: ([$t[] | .mirror.manifest_seconds | select(. != null)] | median)}, @@ -490,15 +527,39 @@ mv "$work/status.json" "$out/status.json" #---------------------------------------------------------------------------# # History: one compact line per run, the last HISTORY_LINES kept #---------------------------------------------------------------------------# -jq -c '{ +# The last measured files and bytes of each target, from the lines before. +# A line that does not read is left out here, and counted. +known='{"x":{},"bad":0}' +if [ -s "$out/history.jsonl" ]; then + rc=0 + known="$(jq -c -n -R ' + reduce (inputs | try fromjson catch "unreadable") as $l ({x: {}, bad: 0}; + if ($l | type) != "object" then .bad += 1 + else reduce (($l.x // {}) | if type == "object" then to_entries[] else empty end + | select((.value | type) == "object" and (.value.n | type) == "number" + and (.value.m | type) == "number")) as $e (.; + .x[$e.key] = ($e.value | {n, m, b, mb})) + end)' "$out/history.jsonl" 2> "$work/known.err")" || rc=$? + [ "$rc" -eq 0 ] || die "could not read $out/history.jsonl for the last measured files (jq ended $rc)" \ + "$(cat "$work/known.err")" "reproduce: jq -R 'fromjson? | .x' $out/history.jsonl" + bad="$(jq -r .bad <<< "$known")" + if [ "$bad" -gt 0 ]; then + warn "$bad line(s) of $out/history.jsonl do not read as JSON objects; they give no last measured files" \ + "reproduce: jq -c . $out/history.jsonl" + fi +fi +jq -c --argjson known "$known" '{ t: .generated, w: {ok: .worker.ok, f: .worker.failed_runs, s: .worker.failing_since, ms: (if .worker.median_seconds == null then null else (.worker.median_seconds * 1000 | round) end)}, g: {ms: (if .ghcr.median_manifest_seconds == null then null else (.ghcr.median_manifest_seconds * 1000 | round) end)}, - x: (.targets | map({key: .target, value: { - n: .coverage.of_files, m: .coverage.files, b: .coverage.of_bytes, mb: .coverage.bytes, - a: .age_seconds, l: .lag_seconds, - c: (if .current == true then 1 elif .current == false then 0 else null end)}}) | from_entries) + x: (.targets | map({key: .target, value: ( + (if .coverage != null then + {n: .coverage.of_files, m: .coverage.files, b: .coverage.of_bytes, mb: .coverage.bytes} + elif $known.x[.target] != null then $known.x[.target] + {k: 1} + else {n: null, m: null, b: null, mb: null} end) + + {a: .age_seconds, l: .lag_seconds, + c: (if .current == true then 1 elif .current == false then 0 else null end)})}) | from_entries) }' "$out/status.json" > "$work/line.json" if [ -f "$out/history.jsonl" ]; then cat "$out/history.jsonl" "$work/line.json" > "$work/history.all" @@ -516,7 +577,7 @@ jq -r ' def cell: if . == null then "-" else tostring end; "", (if .worker.ok == false then - "The Worker fails its probe in \(.worker.failed_runs) run(s) in a row, since \(.worker.failing_since)." + "The Worker fails its probe in \(.worker.failed_runs) run(s) in a row, since \(.worker.failing_since). worker-down opens at \(.worker.down_after.runs) runs in a row over \(.worker.down_after.seconds / 60 | floor) minutes or more." elif .worker.ok == true then "The Worker passes every probe." else "The Worker was not probed, or no probe could be judged." end), "", @@ -536,4 +597,4 @@ jq -r ' ' "$out/status.json" > "$work/worker.md" mv "$work/worker.md" "$out/worker.md" -jq -r '"status-collect: \(.summary.targets) target(s), \(.summary.promoted) promoted, coverage \(if .summary.coverage_files == null then "unknown" else (.summary.coverage_files * 1000 | floor / 10 | tostring) + "%" end), worker \(.worker.ok | if . == null then "not judged" else tostring end), failed runs \(.worker.failed_runs)"' "$out/status.json" >&2 +jq -r '"status-collect: \(.summary.targets) target(s), \(.summary.promoted) promoted, \(.summary.mirror_unread) not read from the registry, coverage \(if .summary.coverage_files == null then "unknown" else (.summary.coverage_files * 1000 | floor / 10 | tostring) + "%" end) over \(.summary.measured) measured, worker \(.worker.ok | if . == null then "not judged" else tostring end), failed runs \(.worker.failed_runs), down \(.worker.down)"' "$out/status.json" >&2 diff --git a/scripts/status-render b/scripts/status-render index 88baaf3..277dab9 100755 --- a/scripts/status-render +++ b/scripts/status-render @@ -26,6 +26,13 @@ # phone # status.json, history.json, bench.json the data behind the page # +# A figure over part of the targets says which part. A target that a run +# could not read on both sides has no coverage: the sums, the group badges +# and the page then name how many targets they count ("77.7% · 3.0 h · 2 of +# 3 measured"). A group whose registry did not answer is "unknown", never +# "not mirrored". A bench older than 2 days shows its age in the warning +# colour. +# # Every SVG has a title, axis labels, a legend and its own style with a dark # variant. Colours are the validated colour-blind safe categorical slots, and # an architecture keeps its colour in every chart. A state colour always comes @@ -355,6 +362,47 @@ def not_read(t): return "registry not read" +def measured(targets): + """The targets with a coverage in this run: read on both sides.""" + return [t for t in targets if t.get("coverage")] + + +def mirror_unread(targets): + """The targets whose registry did not answer: mirror state error.""" + return [t for t in targets if (t.get("mirror") or {}).get("state") == "error"] + + +def scope(targets, lead=", in"): + """Empty when every target was measured, else ', in K of N + repositories': a sum over part of the targets says so.""" + k, n = len(measured(targets)), len(targets) + return "" if k == n else "%s %d of %d repositories" % (lead, k, n) + + +# bench.json comes from a daily run. Older than this, it is shown with its age. +BENCH_MAX_AGE = 2 * 86400 + + +def bench_age(bench, now): + """Seconds from the bench to this measurement, or None when the bench + names no date that reads.""" + t = parse_time((bench or {}).get("date")) + if t is None or not now: + return None + return max(now - t, 0) + + +def bench_old(bench, now): + """None for a bench of the last BENCH_MAX_AGE, else the words that say + how old it is.""" + age = bench_age(bench, now) + if age is None: + return "date unknown" + if age <= BENCH_MAX_AGE: + return None + return "%d d ago" % (age // 86400) + + #---------------------------------------------------------------------------# # SVG #---------------------------------------------------------------------------# @@ -633,10 +681,14 @@ def chart_coverage(ctx, w): svg = Svg("Coverage by repository", "Share of each repository's upstream files that the mirror serves, " "by file name and sha256, grouped by architecture.", w) - cap = ("of %s upstream files are in the mirror, by name and sha256. %d of %d " - "repositories have a snapshot." % (fmt_int(s.get("upstream_files")), - s.get("promoted", 0), s.get("targets", 0))) - y = header(svg, "Coverage by repository", fmt_pct(s.get("coverage_files")), cap) + if not measured(ctx["targets"]): + cap = "No repository could be read on both sides in this run, so coverage is not known." + else: + cap = ("of %s upstream files%s are in the mirror, by name and sha256. %d of %d " + "repositories have a snapshot." % (fmt_int(s.get("upstream_files")), scope(ctx["targets"], " in"), + s.get("promoted", 0), s.get("targets", 0))) + y = header(svg, "Coverage by repository", fmt_pct(s.get("coverage_files")) if measured(ctx["targets"]) + else "unknown", cap) groups = grouped(ctx["targets"], ctx["order"]) items = [("box", "s%d" % ctx["slots"][a], a) for a, _ in groups] y = legend(svg, y, items + [("box", "track", "upstream only")]) @@ -712,9 +764,14 @@ def chart_size(ctx, w): svg = Svg("Size by repository", "Compressed package bytes per repository: in the mirror, and upstream " "only.", w) - cap = ("in the mirror, of %s GiB upstream (%s by bytes)." - % (fmt_gib(s.get("upstream_bytes")), fmt_pct(s.get("coverage_bytes")))) - y = header(svg, "Size by repository", "%s GiB" % fmt_gib(s.get("mirrored_bytes")), cap) + if s.get("mirrored_bytes") is None: + head = "unknown" + cap = "No repository could be read on both sides in this run, so the size is not known." + else: + head = "%s GiB" % fmt_gib(s.get("mirrored_bytes")) + cap = ("in the mirror, of %s GiB upstream (%s by bytes)%s." + % (fmt_gib(s.get("upstream_bytes")), fmt_pct(s.get("coverage_bytes")), scope(ctx["targets"]))) + y = header(svg, "Size by repository", head, cap) groups = grouped(ctx["targets"], ctx["order"]) items = [("box", "s%d" % ctx["slots"][a], a) for a, _ in groups] y = legend(svg, y, items + [("box", "track", "upstream only")]) @@ -904,10 +961,16 @@ def chart_history_coverage(ctx, w): s = ctx["status"]["summary"] svg = Svg("Coverage, last 30 days", "Share of upstream files in the mirror per architecture over time. " - "With more than 2 days of history, one point per 6 hours, the last " - "value of each.", w) - y = header(svg, "Coverage, last 30 days", fmt_pct(s.get("coverage_files")), - "of upstream files in the mirror now. One line per architecture, %s." % span) + "A repository that a run could not read counts with its last " + "measured files. With more than 2 days of history, one point per " + "6 hours, the last value of each.", w) + if measured(ctx["targets"]): + y = header(svg, "Coverage, last 30 days", fmt_pct(s.get("coverage_files")), + "of upstream files in the mirror now%s. One line per architecture, %s." + % (scope(ctx["targets"]), span)) + else: + y = header(svg, "Coverage, last 30 days", "unknown", + "now: no repository could be read on both sides. One line per architecture, %s." % span) series = [] for a in ctx["hist_order"]: pts = [(t, None if c is None else c * 100) for t, c, _ in ctx["series"][a]] @@ -1023,9 +1086,11 @@ def chart_speed(ctx, w): cases = bench_cases(bench) rounds = max([len(c.get("rounds") or []) for c in cases.values()] + [0]) oci = totals.get("oci") + old = bench_old(bench, ctx["now"]) cap = ("for -Sy and -Sw through the oci wrapper, median of %d rounds. Official " - "mirror: %s. Measured %s on %s." % (rounds, fmt_secs(totals.get("official")), - str(bench.get("date", "?"))[:10], bench.get("host", "?"))) + "mirror: %s. Measured %s%s on %s." % (rounds, fmt_secs(totals.get("official")), + str(bench.get("date", "?"))[:10], + " (%s)" % old if old else "", bench.get("host", "?"))) y = header(svg, "pacman speed by path", fmt_secs(oci) if oci is not None else "failed", cap) y = legend(svg, y, items) vals = [c["seconds"] for c in cases.values() if num(c.get("seconds"))] @@ -1094,30 +1159,44 @@ def target_badge(t, interval, tolerance): def group_badge(label, targets, interval, tolerance, count=False): + """Coverage and age over a group of targets. The ratio is over the + targets measured in this run. When that is not all of them, the badge + says how many were, so a part never reads as the whole.""" promoted = [t for t in targets if (t.get("mirror") or {}).get("state") == "promoted"] if not promoted: + if mirror_unread(targets): + # The registry did not answer: nothing says the group is not mirrored. + return badge(label, "unknown", "unknown") return badge(label, "not mirrored", "none") - n = sum(((t.get("coverage") or {}).get("of_files") or 0) for t in targets) - m = sum(((t.get("coverage") or {}).get("files") or 0) for t in targets) + done = measured(targets) + n = sum((t["coverage"].get("of_files") or 0) for t in done) + m = sum((t["coverage"].get("files") or 0) for t in done) worst = max((target_level(t, interval, tolerance) for t in targets), key=lambda lv: LEVEL_RANK[lv]) if worst in ("none", "unknown"): worst = "warning" ages = [t["age_seconds"] for t in promoted if num(t.get("age_seconds"))] + parts = [fmt_pct(m / n) if n else "coverage unknown"] if count: - msg = "%s · %d/%d repos" % (fmt_pct(m / n if n else None), len(promoted), len(targets)) + parts.append("%d/%d repos" % (len(promoted), len(targets))) else: - msg = "%s · %s" % (fmt_pct(m / n if n else None), fmt_age(max(ages) if ages else None)) - return badge(label, msg, worst) + parts.append(fmt_age(max(ages) if ages else None)) + if len(done) < len(targets): + parts.append("%d of %d measured" % (len(done), len(targets))) + return badge(label, " · ".join(parts), worst) -def speed_badge(bench): +def speed_badge(bench, now): if not bench: return badge("pacman speed", "no data", "none") tot = bench_totals(bench) if tot.get("oci") is None: return badge("pacman speed", "oci failed", "critical") - return badge("pacman speed", "oci %s · official %s" % (fmt_secs(tot["oci"]), fmt_secs(tot.get("official"))), - "info") + msg = "oci %s · official %s" % (fmt_secs(tot["oci"]), fmt_secs(tot.get("official"))) + old = bench_old(bench, now) + if old: + # The daily bench has not run since: the time is not current. + return badge("pacman speed", "%s · %s" % (msg, old), "warning") + return badge("pacman speed", msg, "info") #---------------------------------------------------------------------------# @@ -1234,17 +1313,58 @@ def page(ctx, charts, badges): wsub = "%d run(s) in a row; median %s" % (wk["failed_runs"], fmt_secs(wk.get("median_seconds"))) totals = bench_totals(bench) if bench else {} if totals.get("oci") is not None: - last = tile("pacman, oci wrapper", fmt_secs(totals["oci"]), - "official mirror %s" % fmt_secs(totals.get("official"))) + old = bench_old(bench, ctx["now"]) + when = "measured %s" % str(bench.get("date", "?"))[:10] + sub = "official mirror %s; %s" % (fmt_secs(totals.get("official")), when) + if old: + last = tile("pacman, oci wrapper", state_cell("warning", fmt_secs(totals["oci"])), + "%s, %s" % (sub, old), raw=True) + else: + last = tile("pacman, oci wrapper", fmt_secs(totals["oci"]), sub) else: last = tile("GHCR manifest", ghcr_ms, "median, read with no credential") + + # A count or a sum over part of the targets says which part. + n_all = s.get("targets", 0) + n_err = len(mirror_unread(targets)) + n_done = len(measured(targets)) + if n_err == 0: + snap = tile("Repositories with a snapshot", "%d of %d" % (s.get("promoted", 0), n_all), + "%d match upstream now" % s.get("current", 0)) + elif n_err == n_all: + snap = tile("Repositories with a snapshot", "unknown", + "the registry did not answer for any of the %d" % n_all) + else: + snap = tile("Repositories with a snapshot", "%d of %d read" % (s.get("promoted", 0), n_all - n_err), + "%d not read from the registry; %d match upstream now" % (n_err, s.get("current", 0))) + if n_done == 0: + files = tile("Files in the mirror", "unknown", "no repository read on both sides") + size = tile("Size in the mirror", "unknown", "no repository read on both sides") + else: + files = tile("Files in the mirror", fmt_int(s.get("mirrored_files")), + "of %s upstream%s" % (fmt_int(s.get("upstream_files")), scope(targets))) + size = tile("Size in the mirror", "%s GiB" % fmt_gib(s.get("mirrored_bytes")), + "of %s GiB, %s%s" % (fmt_gib(s.get("upstream_bytes")), fmt_pct(s.get("coverage_bytes")), + scope(targets))) + if n_done == 0: + hero_big = "unknown" + hero_cap = ("Coverage is not known: no repository could be read on both sides in this run. " + "The table of repositories names what failed.") + elif n_done == n_all: + hero_big = fmt_pct(s.get("coverage_files")) + hero_cap = ("of upstream files are in the mirror, counted by file name and sha256 over %d " + "repositories." % n_all) + else: + hero_big = fmt_pct(s.get("coverage_files")) + hero_cap = ("of upstream files are in the mirror, counted by file name and sha256 over the %d of %d " + "repositories measured in this run. %d could not be read." % (n_done, n_all, n_all - n_done)) + down = wk.get("down_after") or {} + down_text = ("Failed probes in %d runs in a row, over %d minutes or more, open the worker-down issue." + % (down.get("runs") or 2, (down.get("seconds") or 3000) // 60)) tiles = "".join([ - tile("Repositories with a snapshot", "%d of %d" % (s.get("promoted", 0), s.get("targets", 0)), - "%d match upstream now" % s.get("current", 0)), - tile("Files in the mirror", fmt_int(s.get("mirrored_files")), - "of %s upstream" % fmt_int(s.get("upstream_files"))), - tile("Size in the mirror", "%s GiB" % fmt_gib(s.get("mirrored_bytes")), - "of %s GiB, %s" % (fmt_gib(s.get("upstream_bytes")), fmt_pct(s.get("coverage_bytes")))), + snap, + files, + size, tile("Oldest snapshot", fmt_age(oldest) if num(oldest) else "none", "sync runs every %s" % fmt_interval(interval)), tile("Worker", state_cell(wl, wv), wsub, raw=True), @@ -1293,15 +1413,17 @@ def page(ctx, charts, badges): esc(fmt_int(c.get("files"))), esc("%.1f" % ((c.get("bytes") or 0) / 1048576))]) figs = "".join([ - fig("coverage", "Coverage by repository: %s of upstream files are in the mirror." - % fmt_pct(s.get("coverage_files")), + fig("coverage", ("Coverage by repository: %s of upstream files%s are in the mirror." + % (fmt_pct(s.get("coverage_files")), scope(targets, " in"))) if n_done + else "Coverage by repository: not known in this run.", table(["repository", "in the mirror", "upstream", "share", "missing"], cov_rows, (1, 2, 3, 4))), fig("freshness", "Snapshot age per repository, against the sync interval.", table(["repository", "state", "fetched, UTC", "age", "lag"], fresh_rows, (3, 4))), fig("history-coverage", "Coverage per architecture over the last 30 days.", hist_table), fig("history-age", "Oldest snapshot age per architecture over the last 30 days.", hist_table), - fig("size", "Size by repository: %s GiB of %s GiB are in the mirror." - % (fmt_gib(s.get("mirrored_bytes")), fmt_gib(s.get("upstream_bytes"))), + fig("size", ("Size by repository: %s GiB of %s GiB%s are in the mirror." + % (fmt_gib(s.get("mirrored_bytes")), fmt_gib(s.get("upstream_bytes")), scope(targets, " in"))) + if n_done else "Size by repository: not known in this run.", table(["repository", "GiB in the mirror", "GiB upstream", "share"], size_rows, (1, 2, 3))), fig("speed", "pacman speed through each path, from the daily benchmark.", table(["operation", "path", "median", "rounds", "files", "MiB"], speed_rows, (2, 4, 5)) @@ -1395,7 +1517,7 @@ def page(ctx, charts, badges):

%(cov)s

-

of upstream files are in the mirror, counted by file name and sha256 over %(n)d repositories.

+

%(cap)s

%(tiles)s
@@ -1405,7 +1527,7 @@ def page(ctx, charts, badges):

Repositories

%(repos)s

Worker probes

-

Each probe is one GET with no redirect followed. A repository with a snapshot must answer 302 to https://*.githubusercontent.com. A repository with no snapshot must answer 404. Two failed runs in a row open the worker-down issue.

+

Each probe is one GET with no redirect followed. A repository with a snapshot must answer 302 to https://*.githubusercontent.com. A repository with no snapshot must answer 404. %(down)s

%(probes)s

Badges

%(badges)s
@@ -1419,8 +1541,8 @@ def page(ctx, charts, badges): """ % { "css": PAGE_CSS, "gen": esc(fmt_time(status.get("generated")) + " UTC"), "source": esc(source), "source_text": esc(source.replace("https://", "")), - "narch": len(ctx["order"]), "cov": esc(fmt_pct(s.get("coverage_files"))), - "n": s.get("targets", 0), "tiles": tiles, "use": use, "figs": figs, "repos": repos, + "narch": len(ctx["order"]), "cov": esc(hero_big), "cap": esc(hero_cap), "down": down_text, + "tiles": tiles, "use": use, "figs": figs, "repos": repos, "probes": probes_html, "badges": table(["badge", "URL"], badge_rows), } @@ -1535,7 +1657,7 @@ def main(argv): write(os.path.join(site, "badges", "all.svg"), group_badge("mirror", ctx["targets"], interval, tolerance, count=True)) badges.append("all") - write(os.path.join(site, "badges", "speed.svg"), speed_badge(bench)) + write(os.path.join(site, "badges", "speed.svg"), speed_badge(bench, now)) badges.append("speed") write(os.path.join(site, "index.html"), page(ctx, charts, badges)) diff --git a/tests/static/20-actionlint.sh b/tests/static/20-actionlint.sh index 0b6ddc5..ca90d6e 100644 --- a/tests/static/20-actionlint.sh +++ b/tests/static/20-actionlint.sh @@ -13,16 +13,23 @@ if ! command -v actionlint > /dev/null 2>&1; then exit 1 fi +# ⚠ actionlint 1.7.12, the newest release, does not know the concurrency key +# queue. GitHub documents it for github.com ("Control the concurrency of +# workflows and jobs"; changelog 2026-05-07), and sync.yml uses queue: max. +# Only that one message is ignored. tests/static/50-workflow-policy.sh checks +# the key's value instead. Remove the ignore when actionlint knows the key. +IGNORE_QUEUE='unexpected key "queue" for "concurrency" section' + cd "$REPO_ROOT" found=0 for f in .github/workflows/*.yml .github/workflows/*.yaml; do [ -e "$f" ] || continue found=$((found + 1)) - out="$(actionlint -no-color "$f" 2>&1)" && rc=0 || rc=$? + out="$(actionlint -no-color -ignore "$IGNORE_QUEUE" "$f" 2>&1)" && rc=0 || rc=$? if [ "$rc" -eq 0 ]; then ok "actionlint $f" else - fail "actionlint $f" "$(printf '%s\n' "$out" | sed -n 1,30p)" "reproduce: actionlint $f" + fail "actionlint $f" "$(printf '%s\n' "$out" | sed -n 1,30p)" "reproduce: actionlint -ignore '$IGNORE_QUEUE' $f" fi done if [ "$found" -eq 0 ]; then diff --git a/tests/static/50-workflow-policy.sh b/tests/static/50-workflow-policy.sh index 5e62087..9f0dbc9 100644 --- a/tests/static/50-workflow-policy.sh +++ b/tests/static/50-workflow-policy.sh @@ -23,6 +23,13 @@ # - nothing needs the jobs that watch the others # - every workflow with a schedule: is one scripts/keep-schedules-enabled # enables +# - a re-run does not stop on an artifact of the same name: upload-artifact +# sets overwrite: true, and the Pages artifact is named per run attempt +# - a step that starts another workflow skips one that is not active +# - each run of ci.yml outside a pull request has a concurrency group of +# its own, so no commit on main loses its verdict to a newer push +# - a concurrency queue is single or max, max never with +# cancel-in-progress: true, and sync.yml queues every pending run set -euo pipefail # shellcheck source=../lib/harness.sh . "$REPO_ROOT/tests/lib/harness.sh" @@ -279,4 +286,445 @@ EOF fi fi +#---------------------------------------------------------------------------# +# step_lines WORKFLOW -> "jobwheretext" for each line of each job. +# where is the line number of the step that holds the line, "out" for a line +# of the job's outputs, or "-" for any other line. text has no indentation +# and no "- "; an input of a step's with: block reads "with.: value". +#---------------------------------------------------------------------------# +step_lines() { + awk ' + /^jobs:/ { injobs = 1; next } + injobs && /^[^[:space:]#]/ { injobs = 0 } + !injobs { next } + /^[[:space:]]*(#.*)?$/ { next } + /^ [A-Za-z0-9_-]+:[[:space:]]*(#.*)?$/ { job = $1; sub(/:$/, "", job); where = "-"; wi = -1; next } + /^ [^[:space:]]/ { where = ($0 ~ /^ outputs:[[:space:]]*$/) ? "out" : "-"; wi = -1 } + /^ - / && where != "out" { where = NR; wi = -1 } + { + match($0, /^[[:space:]]*/) + ind = RLENGTH + t = $0 + sub(/^[[:space:]]*(-[[:space:]]+)?/, "", t) + if (wi >= 0 && ind <= wi) wi = -1 + pre = (wi >= 0) ? "with." : "" + if (where != "-" && where != "out" && t ~ /^with:[[:space:]]*$/) wi = ind + print job "\t" where "\t" pre t + } + ' "$1" +} + +#---------------------------------------------------------------------------# +# A re-run uploads its artifacts again into the same run. upload-artifact +# refuses a name that is there already unless overwrite: true. deploy-pages +# refuses a name that two artifacts share, and upload-pages-artifact has no +# overwrite: so the Pages artifact is named per run attempt, and deploy-pages +# reads that name from an output of the job that uploads it. +#---------------------------------------------------------------------------# +for wf in "${workflows[@]}"; do + bad="$(step_lines "$wf" | awk -F '\t' ' + { key = $1 SUBSEP $2 } + $3 ~ /^uses: actions\/upload-artifact@/ { up[key] = 1 } + $3 ~ /^with\.overwrite:[[:space:]]*true[[:space:]]*$/ { ow[key] = 1 } + $3 ~ /^uses: actions\/upload-pages-artifact@/ { pg[key] = 1 } + $3 ~ /^with\.name:/ { v = $3; sub(/^with\.name:[[:space:]]*/, "", v); nm[key] = v } + $3 ~ /^uses: actions\/deploy-pages@/ { dp[key] = 1 } + $3 ~ /^with\.artifact_name:/ { v = $3; sub(/^with\.artifact_name:[[:space:]]*/, "", v); an[key] = v } + $2 == "out" { v = $3; k = v; sub(/:.*/, "", k); sub(/^[^:]*:[[:space:]]*/, "", v); out[$1 SUBSEP k] = v } + END { + for (k in up) if (!(k in ow)) { + split(k, p, SUBSEP) + printf "job %s, step at line %s: upload-artifact with no overwrite: true; ", p[1], p[2] + } + for (k in pg) { + split(k, p, SUBSEP) + if (nm[k] !~ /github\.run_attempt/) + printf "job %s, step at line %s: the Pages artifact name \"%s\" does not hold github.run_attempt; ", p[1], p[2], nm[k] + pname[p[1]] = nm[k] + } + for (k in dp) { + split(k, p, SUBSEP) + if (!match(an[k], /needs\.[A-Za-z0-9_-]+\.outputs\.[A-Za-z0-9_-]+/)) { + printf "job %s, step at line %s: deploy-pages artifact_name \"%s\" is not an output of the job that uploads; ", p[1], p[2], an[k] + continue + } + split(substr(an[k], RSTART, RLENGTH), r, ".") + if (!((r[2] SUBSEP r[4]) in out) || !(r[2] in pname) || out[r[2] SUBSEP r[4]] != pname[r[2]]) + printf "job %s, step at line %s: output %s of job %s is \"%s\", and its Pages upload is named \"%s\"; ", p[1], p[2], r[4], r[2], out[r[2] SUBSEP r[4]], pname[r[2]] + } + }')" + if [ -z "$bad" ]; then + ok "$wf: a re-run does not stop on an artifact of the same name" + else + fail "$wf: a re-run does not stop on an artifact of the same name" "$bad" \ + "a re-run uploads each artifact again into the same run: upload-artifact refuses the name with no overwrite: true, and deploy-pages refuses a name that two artifacts share" \ + "reproduce: grep -n -A6 -E 'upload-artifact|upload-pages-artifact|deploy-pages|outputs:' $wf" + fi +done + +#---------------------------------------------------------------------------# +# A step that starts another workflow skips one that is not active. GitHub +# refuses to start it (HTTP 422), and a person may disable a workflow on +# purpose: that is a notice, not a red run. A state that does not read still +# fails the step. Each run: block with "gh workflow run" runs here against a +# stub gh, under the shell GitHub uses for shell: bash. +#---------------------------------------------------------------------------# +# run_blocks WORKFLOW DIR: each run: block, its indentation removed, in +# DIR/..sh +run_blocks() { + awk -v dir="$2" -v base="$(basename "$1")" ' + function flush() { if (out != "") close(out); out = "" } + /^[[:space:]]*(-[[:space:]]+)?run:[[:space:]]*[|>][-+]?[[:space:]]*$/ { + flush(); out = dir "/" base "." NR ".sh"; printf "" > out; ind = -1; next + } + /^[[:space:]]*(-[[:space:]]+)?run:[[:space:]]*[^|>[:space:]]/ { + flush(); line = $0; sub(/^[[:space:]]*(-[[:space:]]+)?run:[[:space:]]*/, "", line) + f = dir "/" base "." NR ".sh"; print line > f; close(f); next + } + out != "" { + if ($0 ~ /^[[:space:]]*$/) { print "" > out; next } + match($0, /^[[:space:]]*/) + if (ind < 0) ind = RLENGTH + if (RLENGTH < ind) { flush(); next } + print substr($0, ind + 1) > out + } + END { flush() } + ' "$1" +} + +if need_tool jq "a step that starts another workflow skips one that is not active (the stub gh needs jq)"; then + mkdir -p "$work/blocks" "$work/dispatch-bin" + for wf in "${workflows[@]}"; do + run_blocks "$wf" "$work/blocks" + done + cat > "$work/dispatch-bin/gh" << 'EOF' +#!/usr/bin/env bash +# A stand-in for gh: a workflow in the state STUB_STATE ("unreadable": the +# read fails). It refuses to start a workflow that is not active, as GitHub +# does, and records each start it accepts. +set -euo pipefail +printf 'gh %s\n' "$*" >> "$STUB_DIR/calls.log" +args="$*" +jq_expr="" +prev="" +for a in "$@"; do + case "$prev" in + --jq | -q) jq_expr="$a" ;; + esac + prev="$a" +done +if [[ $args =~ ^api[[:space:]]+/?repos/[^[:space:]]+/actions/workflows/([^/?[:space:]]+)([[:space:]]|$) ]]; then + if [ "$STUB_STATE" = unreadable ]; then + echo "gh: HTTP 502: Bad Gateway" >&2 + exit 1 + fi + json="$(jq -n --arg p ".github/workflows/${BASH_REMATCH[1]}" --arg s "$STUB_STATE" '{id: 4242, path: $p, state: $s}')" + if [ -n "$jq_expr" ]; then + jq -r "$jq_expr" <<< "$json" + else + printf '%s\n' "$json" + fi +elif [[ $args =~ ^workflow[[:space:]]+run[[:space:]]+([^[:space:]]+) ]]; then + if [ "$STUB_STATE" != active ]; then + echo "could not create workflow dispatch event: HTTP 422: Cannot trigger a 'workflow_dispatch' on a disabled workflow" >&2 + exit 1 + fi + printf '%s\n' "${BASH_REMATCH[1]}" >> "$STUB_DIR/started.log" +else + echo "stub gh: no answer for: gh $args" >&2 + exit 1 +fi +EOF + chmod +x "$work/dispatch-bin/gh" + # dispatch BLOCK STATE -> the step's exit status; its output in $work/step.out + dispatch() { + : > "$work/calls.log" + : > "$work/started.log" + local rc=0 + STUB_DIR="$work" STUB_STATE="$2" GH_TOKEN=stub GITHUB_REPOSITORY=pkgforge-dev/mirror-archlinux \ + PATH="$work/dispatch-bin:$PATH" bash --noprofile --norc -eo pipefail "$1" > "$work/step.out" 2>&1 || rc=$? + return "$rc" + } + starters=0 + for block in "$work"/blocks/*.sh; do + [ -e "$block" ] || continue + if ! grep -qE '^[^#]*gh workflow run' "$block"; then + continue + fi + starters=$((starters + 1)) + name="$(basename "$block" .sh)" + where=".github/workflows/${name%.*}, the run: block at line ${name##*.}" + rc=0 + dispatch "$block" active || rc=$? + check "$where: starts the workflow when it is active" "0 1" \ + "$rc $(awk 'END { print NR }' "$work/started.log")" \ + "this file's stub gh with STUB_STATE=active; output: $(tr '\n' ' ' < "$work/step.out")" + rc=0 + dispatch "$block" disabled_manually || rc=$? + seen="$(awk '/disabled_manually/ { n++ } END { print n + 0 }' "$work/step.out")" + if [ "$rc" -eq 0 ] && [ ! -s "$work/started.log" ] && [ "$seen" -gt 0 ]; then + ok "$where: skips a workflow that is not active, and names its state" + else + fail "$where: skips a workflow that is not active, and names its state" \ + "exit $rc, started: $(tr '\n' ' ' < "$work/started.log"), state named $seen time(s)" \ + "output: $(tr '\n' ' ' < "$work/step.out")" \ + "GitHub refuses to start a disabled workflow (HTTP 422); a person may disable one on purpose" \ + "reproduce: this file's stub gh with STUB_STATE=disabled_manually" + fi + rc=0 + dispatch "$block" unreadable || rc=$? + if [ "$rc" -ne 0 ]; then + ok "$where: fails when the workflow's state does not read" + else + fail "$where: fails when the workflow's state does not read" "exit 0" \ + "output: $(tr '\n' ' ' < "$work/step.out")" "reproduce: this file's stub gh with STUB_STATE=unreadable" + fi + done + if [ "$starters" -eq 0 ]; then + ok "no run: block starts another workflow with gh workflow run" + fi +fi + +#---------------------------------------------------------------------------# +# ci.yml: a concurrency group holds one pending run, and a newer run cancels +# it. So each run outside a pull request needs a group of its own, or a +# commit on main between two quick pushes gets no verdict. The group and +# cancel-in-progress expressions are evaluated here for each kind of run. +#---------------------------------------------------------------------------# +# top_concurrency WORKFLOW KEY -> the value of KEY in the top-level +# concurrency block +top_concurrency() { + awk -v key="$2" ' + /^concurrency:/ { c = 1; next } + c && /^[^[:space:]#]/ { c = 0 } + c && $0 ~ "^ " key ":" { v = $0; sub("^ " key ":[[:space:]]*", "", v); sub(/[[:space:]]+#.*$/, "", v); print v } + ' "$1" +} + +ci=.github/workflows/ci.yml +if [ ! -f "$ci" ]; then + fail "$ci exists" "the concurrency rule of the test runs is checked there" "reproduce: ls .github/workflows" +elif need_tool python3 "$ci: each run outside a pull request has a concurrency group of its own (the evaluator needs python3)"; then + cat > "$work/expr.py" << 'EOF' +# Evaluate a GitHub Actions template ("text ${{ expr }} text") for each run +# in a JSON list: {"name", "ctx": {"github.x": value}}. Prints +# "namevalue" per run. Covers literals, contexts, ==, !=, !, &&, ||, +# parentheses and format(); any other token stops with a message. +import json +import re +import sys + +TOKEN = re.compile(r"\s*(?:('(?:[^']|'')*')|(\d+)|([A-Za-z_][A-Za-z0-9_.-]*)|(==|!=|&&|\|\||!|\(|\)|,))") + + +def text(v): + if v is None: + return "" + if v is True: + return "true" + if v is False: + return "false" + return str(v) + + +def truthy(v): + return v not in (None, False, 0, "") + + +def fmt(f, *args): + return re.sub(r"\{(\d+)\}", lambda m: text(args[int(m.group(1))]), f) + + +def expr(src, ctx): + src = src.strip() + toks = [] + pos = 0 + while pos < len(src): + m = TOKEN.match(src, pos) + if not m or m.end() == pos: + sys.exit("expr.py: cannot read the expression at: %s" % src[pos:]) + pos = m.end() + s, n, name, op = m.groups() + if s is not None: + toks.append(("v", s[1:-1].replace("''", "'"))) + elif n is not None: + toks.append(("v", int(n))) + elif name is not None: + if name in ("true", "false", "null"): + toks.append(("v", {"true": True, "false": False, "null": None}[name])) + elif name == "format": + toks.append(("f", name)) + elif not name.startswith("github."): + sys.exit("expr.py: unknown name %s" % name) + else: + toks.append(("v", ctx.get(name))) + else: + toks.append(("o", op)) + i = [0] + + def peek(): + return toks[i[0]] if i[0] < len(toks) else ("end", None) + + def take(want=None): + t = peek() + if want is not None and t != ("o", want): + sys.exit("expr.py: expected %s, found %r in %s" % (want, t, src)) + i[0] += 1 + return t + + def p_or(): + v = p_and() + while peek() == ("o", "||"): + take() + r = p_and() + v = v if truthy(v) else r + return v + + def p_and(): + v = p_eq() + while peek() == ("o", "&&"): + take() + r = p_eq() + v = r if truthy(v) else v + return v + + def p_eq(): + v = p_not() + while peek() in (("o", "=="), ("o", "!=")): + op = take()[1] + r = p_not() + same = (v.lower() == r.lower()) if isinstance(v, str) and isinstance(r, str) else v == r + v = same if op == "==" else not same + return v + + def p_not(): + if peek() == ("o", "!"): + take() + return not truthy(p_not()) + return p_primary() + + def p_primary(): + t = take() + if t == ("o", "("): + v = p_or() + take(")") + return v + if t[0] == "f": + take("(") + args = [p_or()] + while peek() == ("o", ","): + take() + args.append(p_or()) + take(")") + return fmt(*args) + if t[0] == "v": + return t[1] + sys.exit("expr.py: unexpected %r in %s" % (t, src)) + + v = p_or() + if peek() != ("end", None): + sys.exit("expr.py: unread tokens in %s" % src) + return v + + +def template(t, ctx): + out, pos = "", 0 + for m in re.finditer(r"\$\{\{(.*?)\}\}", t): + out += t[pos:m.start()] + text(expr(m.group(1), ctx)) + pos = m.end() + return out + t[pos:] + + +tpl = sys.argv[1] +for run in json.loads(sys.argv[2]): + print("%s\t%s" % (run["name"], template(tpl, run["ctx"]))) +EOF + runs='[ + {"name": "push1", "ctx": {"github.event_name": "push", "github.ref": "refs/heads/main", "github.run_id": 101, "github.sha": "a1"}}, + {"name": "push2", "ctx": {"github.event_name": "push", "github.ref": "refs/heads/main", "github.run_id": 102, "github.sha": "b2"}}, + {"name": "dispatch", "ctx": {"github.event_name": "workflow_dispatch", "github.ref": "refs/heads/main", "github.run_id": 103, "github.sha": "b2"}}, + {"name": "push7", "ctx": {"github.event_name": "push", "github.ref": "refs/heads/main", "github.run_id": 7, "github.sha": "c3"}}, + {"name": "pr7a", "ctx": {"github.event_name": "pull_request", "github.event.pull_request.number": 7, "github.ref": "refs/pull/7/merge", "github.run_id": 201, "github.sha": "d4"}}, + {"name": "pr7b", "ctx": {"github.event_name": "pull_request", "github.event.pull_request.number": 7, "github.ref": "refs/pull/7/merge", "github.run_id": 202, "github.sha": "e5"}}, + {"name": "pr8", "ctx": {"github.event_name": "pull_request", "github.event.pull_request.number": 8, "github.ref": "refs/pull/8/merge", "github.run_id": 203, "github.sha": "f6"}}]' + group_tpl="$(top_concurrency "$ci" group)" + cancel_tpl="$(top_concurrency "$ci" cancel-in-progress)" + rc=0 + python3 -I "$work/expr.py" "$group_tpl" "$runs" > "$work/groups.tsv" 2> "$work/expr.err" || rc=$? + python3 -I "$work/expr.py" "$cancel_tpl" "$runs" > "$work/cancels.tsv" 2>> "$work/expr.err" || rc=$? + if [ "$rc" -ne 0 ] || [ -z "$group_tpl" ]; then + fail "$ci: the concurrency group and cancel-in-progress evaluate" "group '$group_tpl', cancel-in-progress '$cancel_tpl'" \ + "$(cat "$work/expr.err")" "reproduce: grep -n -A3 '^concurrency' $ci" + else + g() { awk -F '\t' -v n="$1" '$1 == n { print $2 }' "$work/groups.tsv"; } + c() { awk -F '\t' -v n="$1" '$1 == n { print $2 }' "$work/cancels.tsv"; } + RC="python3 -I expr.py \"\$(grep -A2 '^concurrency' $ci)\" for each kind of run (this file)" + if [ "$(g push1)" != "$(g push2)" ] && [ "$(g push1)" != "$(g dispatch)" ] && [ "$(g push2)" != "$(g dispatch)" ]; then + ok "$ci: two pushes to main and a dispatch get three concurrency groups, so none cancels another's pending run" + else + fail "$ci: two pushes to main and a dispatch get three concurrency groups, so none cancels another's pending run" \ + "push 101: '$(g push1)', push 102: '$(g push2)', dispatch 103: '$(g dispatch)'" \ + "a group holds one pending run, and a newer run cancels it" "reproduce: $RC" + fi + check "$ci: a run on main never cancels a run in progress" "false false" "$(c push1) $(c dispatch)" "$RC" + if [ "$(g pr7a)" = "$(g pr7b)" ] && [ "$(c pr7a)" = true ]; then + ok "$ci: a newer push to a pull request cancels the run of the older one" + else + fail "$ci: a newer push to a pull request cancels the run of the older one" \ + "PR 7 run 201: '$(g pr7a)', run 202: '$(g pr7b)', cancel-in-progress '$(c pr7a)'" "reproduce: $RC" + fi + if [ "$(g pr7a)" != "$(g push7)" ] && [ "$(g pr7a)" != "$(g pr8)" ]; then + ok "$ci: the group of a pull request is never the group of another run" + else + fail "$ci: the group of a pull request is never the group of another run" \ + "PR 7: '$(g pr7a)', push with run id 7: '$(g push7)', PR 8: '$(g pr8)'" "reproduce: $RC" + fi + fi +fi + +#---------------------------------------------------------------------------# +# Concurrency queues. actionlint does not know the key queue yet +# (tests/static/20-actionlint.sh ignores it), so its values are checked here: +# single or max, and max never with cancel-in-progress: true, which GitHub +# refuses. sync.yml queues every pending run: with one pending run, a newer +# run cancels it, and a dispatch with allow_older could be replaced by the +# next scheduled run, which plans without it. +#---------------------------------------------------------------------------# +for wf in "${workflows[@]}"; do + bad="$(awk ' + function done_block() { + if (inc && q != "") { + if (q != "single" && q != "max") printf "line %d: queue: %s is not single or max; ", qline, q + if (q == "max" && cip != "" && cip != "false") printf "line %d: queue: max with cancel-in-progress: %s; ", qline, cip + } + inc = 0; q = ""; cip = "" + } + /^[[:space:]]*(#.*)?$/ { next } + { + match($0, /^[[:space:]]*/) + ind = RLENGTH + if (inc && ind <= cind) done_block() + } + /^[[:space:]]*concurrency:[[:space:]]*(#.*)?$/ { done_block(); inc = 1; cind = ind; next } + inc && /^[[:space:]]*queue:/ { q = $0; sub(/^[[:space:]]*queue:[[:space:]]*/, "", q); sub(/[[:space:]]*#.*$/, "", q); qline = NR } + inc && /^[[:space:]]*cancel-in-progress:/ { cip = $0; sub(/^[[:space:]]*cancel-in-progress:[[:space:]]*/, "", cip); sub(/[[:space:]]*#.*$/, "", cip) } + END { done_block() } + ' "$wf")" + if [ -z "$bad" ]; then + ok "$wf: every concurrency queue is single or max, and max never cancels in progress" + else + fail "$wf: every concurrency queue is single or max, and max never cancels in progress" "$bad" \ + "GitHub refuses queue: max with cancel-in-progress: true" "reproduce: grep -n -A4 'concurrency:' $wf" + fi +done +sync_wf=.github/workflows/sync.yml +if [ -f "$sync_wf" ]; then + check "$sync_wf: the group sync queues every pending run, so a dispatch waits its turn" \ + "sync false max" \ + "$(top_concurrency "$sync_wf" group) $(top_concurrency "$sync_wf" cancel-in-progress) $(top_concurrency "$sync_wf" queue)" \ + "grep -n -A4 '^concurrency' $sync_wf" +else + fail "$sync_wf exists" "the sync is the workflow whose pending runs must not be replaced" "reproduce: ls .github/workflows" +fi + summary diff --git a/tests/unit/60-status.sh b/tests/unit/60-status.sh index 5485170..639e76b 100644 --- a/tests/unit/60-status.sh +++ b/tests/unit/60-status.sh @@ -7,7 +7,9 @@ # first on PATH as curl. scripts/status-render then runs on what was # collected, with a fixture history and bench file. Every SVG must parse as # XML, every badge must carry its text, and the page must reference every -# chart and badge. +# chart and badge. With one registry read that fails, and with none that +# answers, a sum over part of the targets must say which part, and nothing +# may say "not mirrored" for a registry that did not answer. # # ⚠ db_records needs GNU tar, and status-render needs python3. A shell that # lacks one skips that part and says why: busybox has neither, Arch has no @@ -244,8 +246,12 @@ check "x86_64-multilib: no db_server answers: upstream not read, coverage unknow check "summary: 5 targets, 2 promoted, 1 current, 7 of 13 files" "5 2 1 7 13" \ "$(q '.summary | "\(.targets) \(.promoted) \(.current) \(.mirrored_files) \(.upstream_files)"')" "jq .summary OUT/status.json" +check "summary: the sums count 4 measured targets; x86_64-multilib is not read upstream" "4 0" \ + "$(q '.summary | "\(.measured) \(.mirror_unread)"')" "jq .summary OUT/status.json" check "worker: one failed probe fails the run, the first in a row" "false 1 1 2026-10-14T03:00:00Z" \ "$(q '.worker | "\(.ok) \(.failures) \(.failed_runs) \(.failing_since)"')" "jq .worker OUT/status.json" +check "worker: one failed run is not down" "0 false" \ + "$(q '.worker | "\(.failing_seconds) \(.down)"')" "jq .worker OUT/status.json" check "history.jsonl has one line for one run" 1 "$(awk 'END { print NR }' "$out/history.jsonl")" "wc -l OUT/history.jsonl" check "history keeps files, mirrored files and age per target" "5 3 10800" \ "$(jq -r '.x["x86_64-extra"] | "\(.n) \(.m) \(.a)"' "$out/history.jsonl")" "jq -c . OUT/history.jsonl" @@ -269,6 +275,8 @@ collect "$out" "$((NOW + 3600))" || rc=$? check "a second run measures and exits 0" 0 "$rc" "$RP" check "worker: two failed runs in a row, failing since the first" "false 2 2026-10-14T03:00:00Z" \ "$(q '.worker | "\(.ok) \(.failed_runs) \(.failing_since)"')" "jq .worker OUT/status.json" +check "worker: two failed runs an hour apart are down" "3600 true" \ + "$(q '.worker | "\(.failing_seconds) \(.down)"')" "jq .worker OUT/status.json" check "worker: a 302 to http is not a redirect to https://*.githubusercontent.com" \ "302 to 'pkg-containers.githubusercontent.com', not to https://*.githubusercontent.com" \ "$(q '.targets[] | select(.target == "x86_64-extra") | .worker.package.error')" "jq .targets OUT/status.json" @@ -278,8 +286,20 @@ check "history.jsonl has two lines after two runs" 2 "$(awk 'END { print NR }' " redirect x86_64/extra/alpha-1-1-x86_64.pkg.tar.zst rc=0 collect "$out" "$((NOW + 7200))" || rc=$? -check "worker: a run that passes clears the count" "true 0 null" \ - "$(q '.worker | "\(.ok) \(.failed_runs) \(.failing_since)"')" "jq .worker OUT/status.json" +check "worker: a run that passes clears the count" "true 0 null false" \ + "$(q '.worker | "\(.ok) \(.failed_runs) \(.failing_since) \(.down)"')" "jq .worker OUT/status.json" + +# sync.yml starts a status run when it ends, so the :47 run can follow it +# within minutes. Two failed runs 7 minutes apart are not yet down. +out="$work/run2b" +mkdir -p "$out" +cp "$out1/history.jsonl" "$out/history.jsonl" +redirect x86_64/extra/alpha-1-1-x86_64.pkg.tar.zst 503 '' +rc=0 +collect "$out" "$((NOW + 420))" || rc=$? +check "worker: two failed runs 7 minutes apart are not down; down needs 50 minutes" "false 2 420 false" \ + "$(q '.worker | "\(.ok) \(.failed_runs) \(.failing_seconds) \(.down)"')" "jq .worker OUT/status.json" +redirect x86_64/extra/alpha-1-1-x86_64.pkg.tar.zst # History is cut to the last 2000 lines, the new line last. out="$work/run3" @@ -305,6 +325,43 @@ if grep -q 'history.jsonl does not read' "$out.err"; then else fail "the line that does not read is named on stderr" "$(sed -n 1,20p "$out.err")" "$RP" fi +redirect x86_64/extra/alpha-1-1-x86_64.pkg.tar.zst + +# The registry answers 503 for x86_64-extra's latest. Its coverage is not +# measured: the sums say over which targets they are, and the history keeps +# its last measured files. +out="$work/run5" +mkdir -p "$out" +cp "$out1/history.jsonl" "$out/history.jsonl" +printf '503\n' > "$net/ghcr.io/v2/test/mirror/x86_64/extra/manifests/latest.code" +rc=0 +collect "$out" "$((NOW + 3600))" || rc=$? +rm "$net/ghcr.io/v2/test/mirror/x86_64/extra/manifests/latest.code" +check "a registry that answers 503 for one target: the collect measures and exits 0" 0 "$rc" "$RP" +check "x86_64-extra: latest answers 503: mirror state error, coverage unknown" "error null" \ + "$(q '.targets[] | select(.target == "x86_64-extra") | "\(.mirror.state) \(.coverage)"')" "jq .targets OUT/status.json" +check "summary: 1 promoted, 3 measured, 1 not read from the registry, 4 of 8 files" "1 3 1 4 8" \ + "$(q '.summary | "\(.promoted) \(.measured) \(.mirror_unread) \(.mirrored_files) \(.upstream_files)"')" "jq .summary OUT/status.json" +check "history: x86_64-extra keeps its last measured files, marked carried" "5 3 1500 600 1" \ + "$(jq -rs '.[-1].x["x86_64-extra"] | "\(.n) \(.m) \(.b) \(.mb) \(.k)"' "$out/history.jsonl")" "tail -n 1 OUT/history.jsonl | jq ." +check "history: a measured target is not marked carried" "4 4 null" \ + "$(jq -rs '.[-1].x["x86_64-core"] | "\(.n) \(.m) \(.k)"' "$out/history.jsonl")" "tail -n 1 OUT/history.jsonl | jq ." +check "history: a target never measured stays unknown" "null null null" \ + "$(jq -rs '.[-1].x["x86_64-multilib"] | "\(.n) \(.m) \(.k)"' "$out/history.jsonl")" "tail -n 1 OUT/history.jsonl | jq ." + +# GHCR does not resolve: no target's registry reads. Nothing says a target +# is not mirrored, and no sum is 0. +out="$work/run6" +mkdir -p "$out" +cp "$out1/history.jsonl" "$out/history.jsonl" +mv "$net/ghcr.io" "$net/ghcr.io.off" +rc=0 +collect "$out" "$((NOW + 3600))" || rc=$? +mv "$net/ghcr.io.off" "$net/ghcr.io" +check "a registry that does not resolve: the collect measures and exits 0" 0 "$rc" "$RP" +check "summary: no registry read: 0 promoted, 5 not read, 0 measured, every sum unknown" "0 5 0 null null null null null" \ + "$(q '.summary | "\(.promoted) \(.mirror_unread) \(.measured) \(.mirrored_files) \(.upstream_files) \(.mirrored_bytes) \(.upstream_bytes) \(.coverage_files)"')" \ + "jq .summary OUT/status.json" #---------------------------------------------------------------------------# # check-staleness: a target with no latest is new while git says its file is @@ -325,6 +382,20 @@ else } add_target aarch64 2026-10-01 add_target loong64 2026-10-13 + # riscv64-core: added with enabled = no long ago, enabled one day ago. Its + # grace starts when it is enabled. + mkdir -p "$net/up.test/riscv64/core" + cp "$net/up.test/aarch64/core/core.db" "$net/up.test/riscv64/core/core.db" + # shellcheck disable=SC2016 # the target file's own placeholders + printf 'arch = riscv64\nrepo = core\ndb_server = https://up.test/$arch/$repo\nenabled = no\n' > "$srepo/targets/riscv64-core.conf" + git -C "$srepo" add targets/riscv64-core.conf + GIT_AUTHOR_DATE="2026-09-01T12:00:00Z" GIT_COMMITTER_DATE="2026-09-01T12:00:00Z" \ + git -C "$srepo" -c user.name=test -c user.email=test@example.invalid commit --quiet -m "add riscv64-core, disabled" + # shellcheck disable=SC2016 # the target file's own placeholders + printf 'arch = riscv64\nrepo = core\ndb_server = https://up.test/$arch/$repo\n' > "$srepo/targets/riscv64-core.conf" + git -C "$srepo" add targets/riscv64-core.conf + GIT_AUTHOR_DATE="2026-10-13T12:00:00Z" GIT_COMMITTER_DATE="2026-10-13T12:00:00Z" \ + git -C "$srepo" -c user.name=test -c user.email=test@example.invalid commit --quiet -m "enable riscv64-core" staleness() { local rc=0 PATH="$bin:$PATH" FAKE_CURL_ROOT="$net" TARGETS_DIR="$2" MIRROR_NAMESPACE=test/mirror \ @@ -340,7 +411,9 @@ else "$(cat "$work/stale1/aarch64-core.state")" "$RS" check "check-staleness: no latest, added 1 day ago, tolerance 3: new" new \ "$(cat "$work/stale1/loong64-core.state")" "$RS" - if grep -q '| target file added | 2026-10-01 |' "$work/stale1/aarch64-core.md"; then + check "check-staleness: no latest, added disabled 43 days ago, enabled 1 day ago: new" new \ + "$(cat "$work/stale1/riscv64-core.state")" "$RS; git -C log -- targets/riscv64-core.conf" + if grep -q '| target file added or enabled | 2026-10-01 |' "$work/stale1/aarch64-core.md"; then ok "check-staleness: the issue body says when the target was added" else fail "check-staleness: the issue body says when the target was added" \ @@ -433,6 +506,15 @@ for name, words in sorted(spec["badges"].items()): lost = [w for w in words if w not in t] say(not lost, "badge %s carries its text: %s" % (name, " | ".join(words)), "text: %s; not found: %s" % (t, lost)) +for name, words in sorted(spec.get("badges_without", {}).items()): + try: + t = texts(os.path.join(site, "badges", name + ".svg")) + except (OSError, ET.ParseError) as e: + say(False, "badge %s does not say %s" % (name, " | ".join(words)), str(e)) + continue + seen = [w for w in words if w in t] + say(not seen, "badge %s does not say %s" % (name, " | ".join(words)), "text: %s; found: %s" % (t, seen)) + for name, words in sorted(spec.get("chart_text", {}).items()): try: t = texts(os.path.join(site, "charts", name + ".svg")) @@ -447,16 +529,23 @@ class Refs(HTMLParser): def __init__(self): super().__init__() self.refs = set() + self.text = [] def handle_starttag(self, tag, attrs): for k, v in attrs: if k in ("src", "href", "srcset") and v and "://" not in v and not v.startswith("#"): self.refs.add(v) + def handle_data(self, data): + self.text.append(data) + r = Refs() with open(os.path.join(site, "index.html"), encoding="utf-8") as f: r.feed(f.read()) +page_text = " ".join(" ".join(r.text).split()) +for words in spec.get("page_text", []): + say(words in page_text, "index.html says: %s" % words, "not found in the page text") want = (["charts/%s.svg" % c for c in spec["charts"]] + ["charts/narrow/%s.svg" % c for c in spec["charts"]] + ["badges/%s.svg" % b for b in spec["badges"]]) lost = [w for w in want if w not in r.refs] @@ -497,8 +586,13 @@ render() { CHARTS='["coverage","freshness","size","history-coverage","history-age","speed"]' spec() { - # spec BENCH_CASES HISTORY SPEED_WORDS - jq -nc --argjson charts "$CHARTS" --argjson cases "$1" --argjson history "$2" --argjson speed "$3" '{ + # spec BENCH_CASES HISTORY SPEED_WORDS [OVERRIDES]: OVERRIDES is merged + # into the spec, an object at a time; a list in it replaces the list. + # page_text_more adds to page_text. + local over="${4:-}" + [ -n "$over" ] || over='{}' + jq -nc --argjson charts "$CHARTS" --argjson cases "$1" --argjson history "$2" --argjson speed "$3" \ + --argjson over "$over" '{ charts: $charts, bench_cases: $cases, history: $history, svgs: (($charts | length) * 2 + 10), files: (["index.html", "status.json", "history.json", "bench.json"] @@ -509,20 +603,28 @@ spec() { "x86_64-multilib": ["x86_64-multilib", "not mirrored"], "aarch64-core": ["aarch64-core", "not mirrored"], "loong64-core": ["loong64-core", "not mirrored"], - "x86_64": ["x86_64", "77.7%"], + "x86_64": ["x86_64", "77.7%", "3.0 h", "2 of 3 measured"], "aarch64": ["aarch64", "not mirrored"], "loong64": ["loong64", "not mirrored"], - "all": ["mirror", "53.8%", "2/5 repos"], + "all": ["mirror", "53.8%", "2/5 repos", "4 of 5 measured"], "speed": $speed}, + badges_without: {"x86_64-core": ["measured"], "speed": ["ago", "date unknown"]}, chart_text: { - coverage: ["Coverage by repository", "53.8%", "x86_64", "aarch64", "loong64", "core", "extra", "multilib", "upstream only"], + coverage: ["Coverage by repository", "53.8%", "in 4 of 5 repositories", "x86_64", "aarch64", "loong64", "core", "extra", "multilib", "upstream only"], freshness: ["Snapshot age", "3.0 h", "sync every 3 h", "not mirrored yet", "matches upstream"], - size: ["Size by repository", "GiB"], + size: ["Size by repository", "GiB", "in 4 of 5 repositories"], "history-coverage": ["Coverage, last 30 days", "x86_64", "aarch64", "Time, UTC"], - "history-age": ["Snapshot age, last 30 days", "sync every 3 h", "log scale"]}}' + "history-age": ["Snapshot age, last 30 days", "sync every 3 h", "log scale"]}, + page_text: ([ + "over the 4 of 5 repositories measured in this run. 1 could not be read.", + "of 13 upstream, in 4 of 5 repositories", + "Failed probes in 2 runs in a row, over 50 minutes or more, open the worker-down issue."] + + ($over.page_text_more // []))} + * ($over | del(.page_text_more))' } -render "$in" "$work/site" "$(spec 12 120 '["pacman speed", "oci 13.3 s", "official 15.6 s"]')" +render "$in" "$work/site" "$(spec 12 120 '["pacman speed", "oci 13.3 s", "official 15.6 s"]' \ + '{"page_text_more": ["official mirror 15.6 s; measured 2026-10-14"]}')" # No bench, and a history line that does not read in2="$work/render-in2" @@ -550,6 +652,67 @@ mkdir -p "$in3" cp "$work/run1/status.json" "$work/run1/history.jsonl" "$in3/" render "$in3" "$work/site3" "$(spec 0 1 '["pacman speed", "no data"]')" +# x86_64-extra's registry answered 503 (run5, an hour after run1): the +# x86_64 and all badges say over how many targets they count, and the +# history keeps x86_64 at 77.7%. +in5="$work/render-in5" +mkdir -p "$in5" +cp "$work/run5/status.json" "$work/run5/history.jsonl" "$in5/" +render "$in5" "$work/site5" "$(spec 0 2 '["pacman speed", "no data"]' '{ + "badges": { + "x86_64-core": ["x86_64-core", "100%", "4.0 h"], + "x86_64-extra": ["x86_64-extra", "unknown"], + "x86_64": ["x86_64", "100%", "4.0 h", "1 of 3 measured"], + "all": ["mirror", "50%", "1/5 repos", "3 of 5 measured"]}, + "chart_text": { + "coverage": ["Coverage by repository", "50%", "in 3 of 5 repositories", "registry not read"], + "freshness": ["Snapshot age", "4.0 h", "sync every 3 h", "not mirrored yet", "matches upstream"], + "size": ["Size by repository", "in 3 of 5 repositories"]}, + "page_text": [ + "over the 3 of 5 repositories measured in this run. 2 could not be read.", + "1 of 4 read", "1 not read from the registry; 1 match upstream now", + "x86_64 2026-10-14 03:00 77.7% 77.7%"]}')" + +# No registry read at all (run6): every group is unknown, never "not +# mirrored", and no sum reads as 0. +in6="$work/render-in6" +mkdir -p "$in6" +cp "$work/run6/status.json" "$work/run6/history.jsonl" "$in6/" +render "$in6" "$work/site6" "$(spec 0 2 '["pacman speed", "no data"]' '{ + "badges": { + "x86_64-core": ["x86_64-core", "unknown"], "x86_64-extra": ["x86_64-extra", "unknown"], + "x86_64-multilib": ["x86_64-multilib", "unknown"], "aarch64-core": ["aarch64-core", "unknown"], + "loong64-core": ["loong64-core", "unknown"], + "x86_64": ["x86_64", "unknown"], "aarch64": ["aarch64", "unknown"], "loong64": ["loong64", "unknown"], + "all": ["mirror", "unknown"]}, + "badges_without": {"x86_64": ["not mirrored"], "all": ["not mirrored"]}, + "chart_text": { + "coverage": ["Coverage by repository", "unknown"], + "freshness": ["Snapshot age", "none", "sync every 3 h"], + "size": ["Size by repository", "unknown"]}, + "page_text": [ + "Coverage is not known: no repository could be read on both sides in this run.", + "the registry did not answer for any of the 5", "Files in the mirror unknown", + "Size in the mirror unknown"]}')" + +# A bench from 2026-08-01, 73 days before the measurement: the badge and the +# tile show its age in the warning colour. +in7="$work/render-in7" +mkdir -p "$in7" +cp "$in/status.json" "$in7/status.json" +jq '.date = "2026-08-01T05:50:00Z"' "$REPO_ROOT/tests/fixtures/status/bench.json" > "$in7/bench.json" +render "$in7" "$work/site7" "$(spec 12 0 '["pacman speed", "oci 13.3 s", "official 15.6 s", "73 d ago"]' '{ + "badges_without": {"speed": []}, + "chart_text": {"speed": ["Measured 2026-08-01 (73 d ago)"], + "history-coverage": ["Coverage, last 30 days", "No history yet"]}, + "page_text_more": ["official mirror 15.6 s; measured 2026-08-01, 73 d ago"]}')" +if grep -q 'fill:#9a6700' "$work/site7/badges/speed.svg"; then + ok "site7: a bench older than 2 days gives the speed badge the warning colour" +else + fail "site7: a bench older than 2 days gives the speed badge the warning colour" \ + "$(sed -n 3,4p "$work/site7/badges/speed.svg")" "python3 -I scripts/status-render IN SITE" +fi + # No status.json: refused, with the path named rc=0 python3 -I "$REPO_ROOT/scripts/status-render" "$work/empty" "$work/site4" > "$work/site4.out" 2> "$work/site4.err" || rc=$? From 6aac3b0bb45d16330b55a982530f289593dbd5c8 Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 23:01:36 +0545 Subject: [PATCH 21/22] pipeline: fix review findings - fetch_upstream: MAX_SECONDS bounds the whole call, retries included (--retry-max-time, and timeout 5 s after it). plan ends its upstream reads within PLAN_BUDGET (1200 s), shared out over the targets; a db_server read gets 1/(n + 1) of the target's share. - mirror-file: with no %PGPSIG%, the .sig comes from the target's servers in their own order, the db_servers first, as verify reads it. A server that fails is asked last by every file of the shard after it. A .sig read ends within 30 s, a download within 120 s plus the time at 64 KiB/s. - A file pushed unsigned is named in the staged manifest (dev.pkgforge.mirror.unsigned). plan asks for its .sig on each run and plans it again once upstream signs it, also with the database unchanged. - registry_token_file: an anonymous read never uses the token cache. - Every chain artifact carries the plan id. stage reads only the results of its plan; the report reads only the summaries of its plan, for pending targets; collect reads only the verdicts and the promotion of the staged digest. So "Re-run all jobs" reads nothing of the earlier attempt. - Integration cases for the files-database rule: a .files that leaves out a package is planned and verified; one that names an extra package is refused. --- .github/workflows/sync-target.yml | 4 +- .github/workflows/sync.yml | 9 +- docs/design.md | 12 +- docs/operations.md | 15 ++ scripts/lib.sh | 72 +++++++-- scripts/mirror-file | 83 ++++++++--- scripts/mirror-shard | 17 ++- scripts/plan | 200 +++++++++++++++++++++---- scripts/report | 99 ++++++++++--- scripts/stage | 80 +++++++++- tests/fixtures/build-repo | 50 ++++++- tests/integration/10-pipeline.sh | 236 +++++++++++++++++++++++++++++- tests/unit/40-lib.sh | 174 +++++++++++++++++++++- 13 files changed, 941 insertions(+), 110 deletions(-) diff --git a/.github/workflows/sync-target.yml b/.github/workflows/sync-target.yml index 3a1f14c..8c02262 100644 --- a/.github/workflows/sync-target.yml +++ b/.github/workflows/sync-target.yml @@ -4,7 +4,9 @@ # # ⛔ The chain reads only its own target's artifacts, and every artifact it # writes carries the target's name: chains of different targets never share a -# file, and one target's failure never holds another. +# file, and one target's failure never holds another. The results, the stage +# and the summary also carry the plan id (scripts/plan): after "Re-run all +# jobs", scripts/stage and scripts/report read nothing of the earlier plan. name: Sync one target on: diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 0dcbcb9..cc46ee3 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -12,6 +12,12 @@ # writes only write-once package tags and its own staging- tag, and # never moves latest. Runs from every branch share one concurrency group, so # two runs never write the same tags at once. +# +# A re-run: "Re-run failed jobs" keeps the plan and its artifacts. "Re-run all +# jobs" plans again under a new plan id; a download by name or pattern still +# returns the earlier attempt's artifact for every name not uploaded again. +# So each artifact a chain makes carries the plan id, and scripts/stage and +# scripts/report read none of another plan (docs/operations.md). name: Sync on: @@ -99,7 +105,8 @@ jobs: done scripts/plan "$RUNNER_TEMP/plan" "${args[@]}" mkdir -p "$RUNNER_TEMP/plan-summary" - cp "$RUNNER_TEMP/plan/summary.tsv" "$RUNNER_TEMP/plan/targets.json" "$RUNNER_TEMP/plan/pending.json" "$RUNNER_TEMP/plan-summary/" + cp "$RUNNER_TEMP/plan/summary.tsv" "$RUNNER_TEMP/plan/targets.json" "$RUNNER_TEMP/plan/pending.json" \ + "$RUNNER_TEMP/plan/plan-id" "$RUNNER_TEMP/plan-summary/" - name: Upload the plan's summary, for the report uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: diff --git a/docs/design.md b/docs/design.md index 7ccf307..4d8a04c 100644 --- a/docs/design.md +++ b/docs/design.md @@ -22,6 +22,13 @@ Each rule below keeps this invariant true for one way it could fail. | ⛔ A package tag is written once. A tag that holds other bytes under a name is never written again. The target's state is `replaced`, and on `main` the `replaced-` issue opens. | The promoted or an older snapshot names those bytes. pacman refuses a file whose sha256 it does not expect, and tries no other Server. | `scripts/lib.sh` `decide_push`, `scripts/mirror-file` | | A signature layer that is missing or wrong beside the right package bytes is repaired: the file is pushed again with the same package blob. plan skips a file only when its sha256 and its `%PGPSIG%` equal the promoted record. | A file that upstream signs again keeps a stale signature layer for ever. | `scripts/plan`, `scripts/mirror-file` | | A package never lacks its signature when the database has one. The layer is the decoded `%PGPSIG%`. With no `%PGPSIG%`, it is the detached `.sig`, which must be one OpenPGP signature packet under 64 KiB. Any other answer is an error, not an absence. A file is pushed unsigned only when every server answers 404 for its `.sig`. | pacman asks for a `.sig` only from the server that served the package. A missing `.sig` ends the install. | `scripts/mirror-file` | +| With no `%PGPSIG%`, mirror-file reads the `.sig` from the target's servers in their own order, the `db_server` hosts first. verify compares with the same order. Only package files are read in an order rotated by shard. | A plain mirror that serves other `.sig` bytes gives the layer, verify refuses it, and the target fails on every run. | `scripts/mirror-file` | +| A file pushed unsigned is named in the staged manifest, in `dev.pkgforge.mirror.unsigned`. plan asks for its `.sig` again on each run. When a server answers, plan puts the file in todo, also when the database did not change, and mirror-file adds the layer. | A `.sig` that upstream publishes later never reaches the mirror. A detached-signature database does not change when a `.sig` is added. | `scripts/stage`, `scripts/plan` | +| Every upstream read has a stall deadline. A database or a signature read has a whole-call deadline that also bounds its retries: `--retry-max-time`, and `timeout` 5 s after it. A package download gets 120 s plus the time at 64 KiB/s. | curl gives each retry a new `--max-time`: a call took 3 x the deadline + 4 s. | `scripts/lib.sh` `fetch_upstream` | +| The upstream reads of plan end within `PLAN_BUDGET` (1200 s), below the job's 30 minutes. Each target gets what is left, divided by the targets not yet read. A `db_server` read gets 1/(n + 1) of the target's share, with n the `db_server` hosts not yet read. A target whose reads do not fit is in error for the run, and its why names `PLAN_BUDGET`. | One slow upstream server makes the plan job time out, and no target is mirrored or promoted. | `scripts/plan` | +| A server that fails for one file of a shard is asked last by every file of the shard after it: a timeout, a failed connection, or a status other than 200, 404 and 410. A `.sig` read ends within 30 s. | A server that stalls costs each file of the shard 184 s, also a file the registry holds. The shard never ends inside its step. | `scripts/mirror-file`, `scripts/lib.sh` `down_mark` | +| The token cache keeps only a token got with `REGISTRY_TOKEN`. A read with `REGISTRY_TOKEN` empty never uses the cache. | On a registry other than GHCR, stage's anonymous read sends the workflow's token. A private package reads as public and is promoted. | `scripts/lib.sh` `registry_token_file` | +| Every artifact that a chain makes carries the plan id (`scripts/plan`). stage reads only the shard results of its plan. The report reads only the summaries of its plan, and only for a target the plan marks pending. collect reads only the verdicts and the promotion of the staged digest. | "Re-run all jobs" plans again, and a download by name or pattern still returns the earlier attempt's artifacts. A stale result holds a target, or a stale summary raises an issue again. | `scripts/stage`, `scripts/report`, `scripts/mirror-shard` | | Verify compares every `.sig` pacman stored with the decoded `%PGPSIG%`. | pacman checks against `%PGPSIG%` and stores the served `.sig` unread. A wrong layer passes verify and breaks a later `pacman -U` from the cache. | `scripts/verify-batch` | | Package manifests are the same bytes whoever pushes them. `org.opencontainers.image.created` is the package's `%BUILDDATE%`, and no annotation names the source server. | Two writers replace each other's tag with a manifest of another digest. | `scripts/mirror-file` | | Absent means the registry says so: only a 404 with `NAME_UNKNOWN` or `MANIFEST_UNKNOWN`. On GHCR, with a token set, a read tries `Bearer QQ==` first, and only its 200 counts. Any other answer is read again with the real token. | GHCR answers `QQ==` with 404 for a repository it does not show. A private package reads as absent, and its tags are pushed over. | `scripts/lib.sh` `registry_manifest` | @@ -77,6 +84,7 @@ A database manifest: - layers `.db` (`application/vnd.pkgforge.archlinux.db.v1`) and `.files` (`application/vnd.pkgforge.archlinux.files.v1`), byte for byte as upstream serves them - a `.db.sig` and a `.files.sig` layer, only when upstream publishes them - annotations: `dev.pkgforge.mirror.arch`, `.repo`, `.upstream` (the server), `.db-sha256`, `.files-sha256`, `.packages`, `.bytes`, `.newest-builddate`, `.fetched` (ISO 8601 UTC), `.run`, and the OCI `source`, `title` and `revision` +- `dev.pkgforge.mirror.unsigned`, only when a file has no signature because upstream publishes none: the filenames, space separated, at most 16 KiB ## Jobs @@ -84,7 +92,7 @@ A database manifest: | job | permissions | timeout (min) | does | | --- | --- | --- | --- | -| `plan` | contents read, packages read | 30 | `scripts/plan` for every enabled target, or for the targets of the `targets` input; uploads `plan-` for each pending target | +| `plan` | contents read, packages read | 30 | `scripts/plan` for every enabled target, or for the targets of the `targets` input, within `PLAN_BUDGET` (1200 s) of upstream reads; uploads `plan-` for each pending target | | `target` | contents read, packages write | (each called job has its own) | one call of `sync-target.yml` per pending target, at most `max_targets` (4) at once | | `report` | actions write, contents read, issues write | 10 | `scripts/report`; on `main` it starts `status.yml` | | `keepalive` | actions write, contents read | 10 | `keep-schedules-enabled`, `check-schedules-fired`, and `dispatch-overdue status.yml 120` on `main` | @@ -107,7 +115,7 @@ A dispatch input reaches an expression as a string. `max-parallel` and a number ### Shards -plan gives each target shards of its own. The count is k = the larger of ceil(bytes / `SHARD_BYTES`) and ceil(files / `SHARD_FILES`). k is at least 1, at most `MAX_SHARDS`, and at most the file count. Files go largest first onto the lightest shard. plan logs when `MAX_SHARDS` binds, with the GiB and files per shard. Each shard reads the servers in an order that starts at its own index, so the shards spread across the mirrors. +plan gives each target shards of its own. The count is k = the larger of ceil(bytes / `SHARD_BYTES`) and ceil(files / `SHARD_FILES`). k is at least 1, at most `MAX_SHARDS`, and at most the file count. Files go largest first onto the lightest shard. plan logs when `MAX_SHARDS` binds, with the GiB and files per shard. Each shard reads package files from the servers in an order that starts at its own index, so the shards spread across the mirrors. A `.sig` comes from the servers in the target's order. In both orders, a server that failed earlier in the shard comes last. | variable | default | | --- | --- | diff --git a/docs/operations.md b/docs/operations.md index b109de3..22a3e59 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -108,6 +108,21 @@ gh workflow run sync.yml --ref main gh workflow run sync.yml --ref main -f targets='x86_64-core x86_64-extra' -f max_targets=2 ``` +### Re-run a sync + +A sync run can be run again from its page. The two choices do not do the same thing. + +| choice | does | +| --- | --- | +| Re-run failed jobs | keeps the plan and every artifact of the jobs that passed. The failed jobs run again with the same plan id. Use this when a job failed for a reason outside the mirror, for example a runner that stopped. | +| Re-run all jobs | plans again, under a new plan id. A download by name or pattern still returns the earlier attempt's artifact for each name that this attempt does not upload again. So stage reads only the shard results of the new plan, and the report reads only the summaries of the new plan. A target whose chain left nothing of the new plan shows `failed`, and its row names the plan. | + +⚠ A new run reads upstream again and leaves nothing of an earlier run behind. Prefer `gh workflow run sync.yml --ref main` to "Re-run all jobs". + +### The plan's budget + +The plan job reads every target's databases from upstream, one target after another, in one job of 30 minutes. `scripts/plan` ends its upstream reads within `PLAN_BUDGET` seconds (1200). Each target gets what is left of the budget, divided by the targets not yet read. A target whose upstream does not answer in time is in error for that run, and its row in the run's table names `PLAN_BUDGET`. The next run reads it again. A target that stays in error raises `stale-` after the tolerance. + ## Renovate `.github/renovate.json` sets the update policy. diff --git a/scripts/lib.sh b/scripts/lib.sh index b736d67..797f342 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -576,7 +576,9 @@ reconcile_results() { # REGISTRY_TOKEN_CACHE, when set, is a directory that keeps each scoped token # in a file of its own. Every process that reads it uses the same token, so a # shard asks the token endpoint once, not once per read. A 401 or a 403 drops -# the file and asks again, once. +# the file and asks again, once. ⛔ Only a token got with REGISTRY_TOKEN is +# kept. A read with REGISTRY_TOKEN empty is a consumer's read: it never uses +# the cache, or it would send the workflow's token and read a private package. # # On GHCR a manifest read tries the dummy token Bearer QQ== first, which every # public package accepts. Only its 200 is taken as the answer: GHCR answers QQ== @@ -589,8 +591,9 @@ REG_STATUS_ERROR=1 REG_STATUS_ABSENT=4 # registry_token_file REPO SCOPE -> the cache file for that token, or nothing +# when there is no cache, or when the read is anonymous registry_token_file() { - [ -n "${REGISTRY_TOKEN_CACHE:-}" ] || return 0 + [ -n "${REGISTRY_TOKEN_CACHE:-}" ] && [ -n "${REGISTRY_TOKEN:-}" ] || return 0 printf '%s/%s\n' "$REGISTRY_TOKEN_CACHE" "$(printf '%s %s %s' "$MIRROR_REGISTRY" "$1" "$2" | sha256sum | cut -d ' ' -f 1)" } @@ -754,16 +757,22 @@ manifest_layer() { } # fetch_upstream URL OUT [MAX_SECONDS] -> 0 fetched, 4 the server says it has -# no such file, 1 anything else. Every fetch has a stall deadline; a database -# or a signature also has a whole-transfer one (MAX_SECONDS), so a server that -# trickles moves the caller on to the next server. A package file has no fixed -# deadline: its size makes any fixed one wrong. +# no such file, 1 anything else. Every fetch has a stall deadline (60 s below +# 4096 B/s). With MAX_SECONDS the whole call also ends within MAX_SECONDS, its +# retries included, so a server that trickles moves the caller on to the next +# server. The caller sets MAX_SECONDS: a fixed one for a database or a +# signature, one from the size for a package file. +# +# curl gives each retry a new --max-time. --retry-max-time starts no retry +# after MAX_SECONDS, and timeout(1) ends a retry that started just before it. +# Without them a call took 3 x MAX_SECONDS + 4 s (measured with curl 8.22). fetch_upstream() { - local url="$1" outfile="$2" code rc=0 max=() + local url="$1" outfile="$2" code rc=0 max=() limit=() if [ -n "${3:-}" ]; then - max=(--max-time "$3") + max=(--max-time "$3" --retry-max-time "$3") + limit=(timeout -k 5 "$(($3 + 5))") fi - code="$(curl -q --location --silent --show-error --max-redirs 5 \ + code="$("${limit[@]}" curl -q --location --silent --show-error --max-redirs 5 \ --proto "$UPSTREAM_PROTO" --proto-redir "$UPSTREAM_PROTO" \ --connect-timeout 20 --speed-limit 4096 --speed-time 60 "${max[@]}" \ --retry 2 --retry-connrefused --retry-delay 2 \ @@ -786,6 +795,51 @@ fetch_upstream() { esac } +# sig_packet FILE -> 0 when FILE can be a detached signature: 1 to 65535 +# bytes, starting with an OpenPGP signature packet tag (0x88, 0x89 or 0xc2) +sig_packet() { + local size first_byte + size="$(wc -c < "$1" | tr -d ' ')" + first_byte="$(od -An -tx1 -N1 "$1" | tr -d ' \n')" + [ "$size" -gt 0 ] && [ "$size" -lt 65536 ] && [[ "$first_byte" =~ ^(88|89|c2)$ ]] +} + +#---------------------------------------------------------------------------# +# Servers that failed +# +# A directory holds one empty file per host that failed: no answer in time, a +# connection that failed, or an HTTP status other than 200, 404 and 410. The +# processes that share the directory then ask that host last. A host that +# stalls costs one deadline per directory, not one per file. It is still asked +# when every other server fails. +# +# down_mark DIR URL: remember that the host of URL failed +# down_last DIR URL... -> the URLs, one per line, in the order given, except +# that those whose host failed come last +#---------------------------------------------------------------------------# +# down_key URL -> the file name for the host and port of URL +down_key() { + local h="${1#*://}" + h="${h%%/*}" + printf 'h-%s\n' "$(printf '%s' "$h" | tr -c 'A-Za-z0-9.:_-' '_')" +} + +down_mark() { + mkdir -p -- "$1" + : > "$1/$(down_key "$2")" +} + +down_last() { + local dir="$1" u + shift + for u in "$@"; do + [ -e "$dir/$(down_key "$u")" ] || printf '%s\n' "$u" + done + for u in "$@"; do + [ ! -e "$dir/$(down_key "$u")" ] || printf '%s\n' "$u" + done +} + # Seconds since the epoch, from the environment when a test pins the clock. now_epoch() { printf '%s\n' "${MIRROR_NOW:-$(date -u +%s)}" diff --git a/scripts/mirror-file b/scripts/mirror-file index 47e6fe5..7eedd3d 100755 --- a/scripts/mirror-file +++ b/scripts/mirror-file @@ -19,6 +19,10 @@ # present the tag already held them; nothing written # replaced the tag holds other bytes under this name; never overwritten # failed anything else +# A file that the tag now holds with no signature, because upstream publishes +# none for it, is pushed or present with a detail that starts "unsigned: ". +# scripts/stage lists these files in the staged manifest, and scripts/plan +# asks upstream for their .sig again on each run. # # ⛔ A package tag is written once (lib.sh decide_push). The promoted database # or an older snapshot may name the bytes a tag holds, and a consumer handed new @@ -29,9 +33,16 @@ # ⭐ The signature layer is the database's own %PGPSIG%, decoded, whenever the # database carries one: pacman checks against that copy, and asks for the .sig # of the server that served the package and of no other. Without %PGPSIG% the -# detached .sig is downloaded, from the server that served the file first; it -# is absent only when every server answers 404 for it, and then the file is -# mirrored unsigned, as upstream ships it, and counts as present from then on. +# detached .sig is downloaded from the target's servers in their own order, +# the db_servers first, as scripts/verify-batch reads it: the layer is the +# signature verify compares it with. It is absent only when every server +# answers 404 for it, and then the file is mirrored unsigned, as upstream +# ships it. +# +# The package file comes from the servers in an order rotated by shard, so +# shards spread across mirrors. A server that fails (lib.sh down_mark) is +# asked last by every file of the shard after it. A .sig read ends within +# 30 s, and a package download within 120 s plus the time at 64 KiB/s. # # ⭐ The manifest is the same bytes whoever pushes it: its created annotation is # the package's %BUILDDATE%, and no annotation names the server the bytes came @@ -96,8 +107,8 @@ rm -rf -- "$dir" mkdir -p -- "$dir" #---------------------------------------------------------------------------# -# The servers, the db_servers first, in an order rotated by shard so shards -# spread across mirrors. +# The servers: the db_servers first. Package files are read in an order +# rotated by shard; in both orders a server that failed comes last. #---------------------------------------------------------------------------# mapfile -t servers < <(target_sources "$conf") n="${#servers[@]}" @@ -107,30 +118,44 @@ order=() for ((i = 0; i < n; i++)); do order+=("${servers[$(((start + i) % n))]}") done +down_dir="$WORK/down" +SIG_MAX_TIME=30 + +# failed SERVER RC: the server answered with an error, not with the file or a +# 404. Every file of the shard asks it last from now on. +failed() { + if [ ! -e "$down_dir/$(down_key "$1")" ]; then + log "$1 failed for $file (status $2); every file of this shard asks it last from now on" + fi + down_mark "$down_dir" "$1" +} -# fetch_sig [FIRST]: the detached signature, from FIRST and then every server. +# fetch_sig: the detached signature, from every server in the target's order. # Sets sig_sha, or unsigned=1 when every one answers 404; returns 1 when a # server failed some other way, because an error is not an absence. unsigned=0 fetch_sig() { - local first="${1:-}" server size rc missing=0 errors=0 - for server in ${first:+"$first"} "${order[@]}"; do + local server size rc missing=0 errors=0 sig_order=() + mapfile -t sig_order < <(down_last "$down_dir" "${servers[@]}") + for server in "${sig_order[@]}"; do rc=0 - fetch_upstream "$server/$file.sig" "$dir/$file.sig" || rc=$? + fetch_upstream "$server/$file.sig" "$dir/$file.sig" "$SIG_MAX_TIME" || rc=$? case "$rc" in 0) - size="$(wc -c < "$dir/$file.sig" | tr -d ' ')" - first_byte="$(od -An -tx1 -N1 "$dir/$file.sig" | tr -d ' \n')" - if [ "$size" -gt 0 ] && [ "$size" -lt 65536 ] && [[ "$first_byte" =~ ^(88|89|c2)$ ]]; then + if sig_packet "$dir/$file.sig"; then sig_sha="$(sha256sum -- "$dir/$file.sig" | cut -d ' ' -f 1)" return 0 fi + size="$(wc -c < "$dir/$file.sig" | tr -d ' ')" warn "$file.sig from $server is not an OpenPGP signature packet ($size bytes)" rm -f -- "$dir/$file.sig" errors=$((errors + 1)) ;; 4) missing=$((missing + 1)) ;; - *) errors=$((errors + 1)) ;; + *) + failed "$server" "$rc" + errors=$((errors + 1)) + ;; esac done if [ "$errors" -gt 0 ]; then @@ -160,7 +185,7 @@ fi #---------------------------------------------------------------------------# if [ -z "$sig_sha" ]; then sig_error="" - if ! fetch_sig ""; then + if ! fetch_sig; then result failed "its .sig could not be read ($sig_error)" exit 1 fi @@ -191,21 +216,29 @@ case "$decision" in ;; needsig) # Only reached with no signature known: upstream publishes none for it. - result present "$ref holds it unsigned, and upstream publishes no signature for it" + result present "unsigned: $ref holds it with no signature, and upstream publishes none for it" exit 0 ;; esac #---------------------------------------------------------------------------# -# Download, from the db_servers and then the other servers, in the order set -# above. +# Download, in the shard's order, a server that failed last. A server that +# answers 200 with other bytes is behind, not down: it is not marked. #---------------------------------------------------------------------------# - got_from="" -for server in "${order[@]}"; do +dl_order=() +mapfile -t dl_order < <(down_last "$down_dir" "${order[@]}") +for server in "${dl_order[@]}"; do rc=0 - fetch_upstream "$server/$file" "$dir/$file" || rc=$? - [ "$rc" -eq 0 ] || continue + fetch_upstream "$server/$file" "$dir/$file" "$((120 + csize / 65536))" || rc=$? + case "$rc" in + 0) ;; + 4) continue ;; + *) + failed "$server" "$rc" + continue + ;; + esac size="$(wc -c < "$dir/$file" | tr -d ' ')" got="$(sha256sum -- "$dir/$file" | cut -d ' ' -f 1)" if [ "$size" = "$csize" ] && [ "$got" = "$sha256" ]; then @@ -320,10 +353,14 @@ for attempt in 1 2 3 4 5; do if [ "$rc" -eq "$REG_STATUS_OK" ]; then got_decision="$(decide_push "$dir/pushed.json" "$file" "$sha256" "$csize" "$sig_sha")" fi + if [ "$got_decision" = present ]; then + result pushed "from $got_from, pushed in ${push_seconds}s ($((rate / 1024)) KiB/s), read back on try $attempt" + exit 0 + fi # Pushed unsigned on purpose, the tag reads back as "needs a signature", # which for a file upstream signs nowhere is what present means. - if [ "$got_decision" = present ] || { [ "$unsigned" -eq 1 ] && [ "$got_decision" = needsig ]; }; then - result pushed "from $got_from, pushed in ${push_seconds}s ($((rate / 1024)) KiB/s), read back on try $attempt" + if [ "$unsigned" -eq 1 ] && [ "$got_decision" = needsig ]; then + result pushed "unsigned: from $got_from, pushed in ${push_seconds}s ($((rate / 1024)) KiB/s) with no signature, read back on try $attempt" exit 0 fi sleep $((attempt + 1)) diff --git a/scripts/mirror-shard b/scripts/mirror-shard index 0b79136..e870c1a 100755 --- a/scripts/mirror-shard +++ b/scripts/mirror-shard @@ -18,7 +18,14 @@ # those, and scripts/stage reads them: what was pushed counts. # # Every mirror-file of the shard shares one pull token, kept in a file in the -# work directory (lib.sh REGISTRY_TOKEN_CACHE). +# work directory (lib.sh REGISTRY_TOKEN_CACHE), and one record of the servers +# that failed (lib.sh down_mark): a server that fails for one file is asked +# last for every file after it. +# +# OUT/plan-id is the id of the plan the shard comes from, read from +# SHARD.tsv's plan directory (scripts/plan writes PLAN//plan-id and +# PLAN//shards/.tsv). It is written first, so even a shard that is +# killed leaves it. scripts/stage reads no result of another plan. # # The work goes in the directory with the most free space among $RUNNER_TEMP, # /mnt and /tmp, counting each filesystem once. scripts/debloat-runner runs @@ -56,6 +63,14 @@ case "$DEBLOAT" in *) die "DEBLOAT must be auto, always or never, not '$DEBLOAT'" ;; esac mkdir -p "$out" +plan_id_file="$(dirname "$(dirname "$shard")")/plan-id" +plan_id="" +if [ -f "$plan_id_file" ]; then + plan_id="$(cat "$plan_id_file")" +else + warn "$plan_id_file does not exist, so these results name no plan, and scripts/stage reads them only with a plan that names none" +fi +printf '%s\n' "$plan_id" > "$out/plan-id" lines="$(awk 'END { print NR + 0 }' "$shard")" [ "$lines" -gt 0 ] || die "$shard has no work line" diff --git a/scripts/plan b/scripts/plan index 431a093..5b00c80 100755 --- a/scripts/plan +++ b/scripts/plan @@ -23,17 +23,37 @@ # while a db_server still serves the promoted one. ALLOW_OLDER (target # names, comma separated) skips the promoted comparison for a deliberate # upstream downgrade. -# 4. The same bytes as the promoted database: the target is current. -# 5. Otherwise every file is skip, when the promoted database names it with +# 4. The files the promoted snapshot holds with no signature (its +# dev.pkgforge.mirror.unsigned annotation, scripts/stage): each one's +# .sig is asked for again. One that upstream signs now is planned again, +# so mirror-file adds its signature layer. +# 5. The same bytes as the promoted database, and no file signed since: the +# target is current. +# 6. Otherwise every file is skip, when the promoted database names it with # the same sha256 and the same %PGPSIG% (no request made), or todo. # scripts/mirror-file reads each todo file's tag before writing it: it # never overwrites a package layer, and it repairs a signature layer that # upstream signed again. -# 6. The todo files of the target go into its own shards: k = the larger of +# 7. The todo files of the target go into its own shards: k = the larger of # ceil(bytes / SHARD_BYTES) and ceil(files / SHARD_FILES), at least 1, at # most MAX_SHARDS and the file count; largest file first onto the lightest # shard. A target never waits for another target's shards. # +# ⛔ The plan ends within PLAN_BUDGET seconds of upstream reads (default 1200), +# below the plan job's timeout-minutes (30). Each target gets a share: what is +# left of the budget, divided by the targets not yet read. A fast target +# leaves its unused share to the targets after it; a slow one cannot take +# theirs. Each upstream read ends within its own deadline, or within a part of +# what is left of the share when that is less: a db_server gets 1/(n + 1) of +# it, with n the db_servers not yet read, so a slow first db_server leaves +# time for the next one and for the files database. A target whose reads do +# not fit is in error for this run, and its why names PLAN_BUDGET. +# +# OUT/plan-id this plan's id: the run, its attempt and a random +# number, also in OUT//plan-id. Every +# artifact a chain makes from this plan carries it +# (scripts/mirror-shard, stage, report), and stage +# and report read nothing of another plan. # OUT//state current pending behind held error # OUT//why one line saying what was measured # OUT//upstream.db the chosen database, byte for byte, and .files and @@ -41,6 +61,9 @@ # OUT//records.tsv its records (lib.sh db_records) # OUT//nul.txt its package directories whose desc is NUL bytes only # OUT//todo.tsv the files this run mirrors +# OUT//unsigned.txt the files promoted with no signature that are not +# planned again: upstream still has no .sig for them, +# or did not answer. scripts/stage lists them again. # OUT//shards/.tsv the work lines of shard n, for scripts/mirror-shard # OUT//meta.env KEY=VALUE facts stage writes into the manifest # OUT/targets.json {"include":[{"target":T,"shards":[0,1,..]},..]}: the @@ -58,8 +81,9 @@ # workflow's upload steps read to upload OUT/ as plan-. # # Environment: SHARD_BYTES (default 536870912), SHARD_FILES (400), MAX_SHARDS -# (32, per target), ALLOW_OLDER, MIRROR_NAMESPACE, REGISTRY_USER and -# REGISTRY_TOKEN (used only to tell a private package from an absent one). +# (32, per target), PLAN_BUDGET (1200 s), ALLOW_OLDER, MIRROR_NAMESPACE, +# REGISTRY_USER and REGISTRY_TOKEN (used only to tell a private package from an +# absent one). # # Exit status: 0 planned, whatever state each target is in; 1 no plan could be # made at all. @@ -68,7 +92,7 @@ SCRIPT_NAME=plan # shellcheck source=lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -need curl jq awk tar sha256sum +need curl jq awk tar sha256sum timeout out="${1:-}" [ -n "$out" ] || die "usage: scripts/plan OUT [TARGET.conf ...]" @@ -78,7 +102,8 @@ shift SHARD_BYTES="${SHARD_BYTES:-536870912}" SHARD_FILES="${SHARD_FILES:-400}" MAX_SHARDS="${MAX_SHARDS:-32}" -for v in SHARD_BYTES SHARD_FILES MAX_SHARDS; do +PLAN_BUDGET="${PLAN_BUDGET:-1200}" +for v in SHARD_BYTES SHARD_FILES MAX_SHARDS PLAN_BUDGET; do case "${!v}" in '' | *[!0-9]* | 0) die "$v must be a positive integer, not '${!v}'" ;; esac @@ -88,6 +113,13 @@ done DB_MAX_TIME=300 FILES_MAX_TIME=900 SIG_MAX_TIME=60 +run_deadline=$((SECONDS + PLAN_BUDGET)) + +if [ -n "${GITHUB_RUN_ID:-}" ]; then + plan_id="$GITHUB_RUN_ID.${GITHUB_RUN_ATTEMPT:-1}.$RANDOM" +else + plan_id="local.$(date -u +%s).$$.$RANDOM" +fi if [ "$#" -eq 0 ]; then for conf in "$TARGETS_DIR"/*.conf; do @@ -101,6 +133,8 @@ fi mkdir -p "$out" fetched="$(iso_now)" +printf '%s\n' "$plan_id" > "$out/plan-id" +log "plan $plan_id: $# target(s), PLAN_BUDGET=${PLAN_BUDGET}s" # set_state DIR STATE WHY set_state() { @@ -109,6 +143,49 @@ set_state() { log "$(basename "$1"): $2: $3" } +#---------------------------------------------------------------------------# +# The budget +#---------------------------------------------------------------------------# +# share_start TARGETS_LEFT: this target's share of what is left of PLAN_BUDGET +share_start() { + local left=$((run_deadline - SECONDS)) + if [ "$left" -lt 0 ]; then + left=0 + fi + share=$((left / $1)) + share_deadline=$((SECONDS + share)) + budget_cut=0 +} + +# fetch_within URL OUT MAX PARTS -> fetch_upstream's status. The deadline is +# MAX, or what is left of the share divided by PARTS when that is less; with +# nothing left, status 1 and no request. Sets fetch_max to the deadline used, +# and budget_cut when a read failed with less time than MAX, or was not made. +fetch_within() { + local url="$1" outfile="$2" own="$3" parts="$4" rc=0 + fetch_max=$(((share_deadline - SECONDS) / parts)) + if [ "$fetch_max" -gt "$own" ]; then + fetch_max="$own" + fi + if [ "$fetch_max" -lt 1 ]; then + fetch_max=0 + budget_cut=1 + log "$url: not read, nothing is left of the target's share of PLAN_BUDGET (${share}s)" + return 1 + fi + fetch_upstream "$url" "$outfile" "$fetch_max" || rc=$? + if [ "$rc" -ne 0 ] && [ "$rc" -ne 4 ] && [ "$fetch_max" -lt "$own" ]; then + budget_cut=1 + fi + return "$rc" +} + +# budget_note -> "; ..." when a read of this target failed for want of time +budget_note() { + [ "$budget_cut" -eq 1 ] || return 0 + printf '; a read had less time than its own deadline, to fit the target'\''s share (%ss) of PLAN_BUDGET=%ss' "$share" "$PLAN_BUDGET" +} + allowed_older() { case ",${ALLOW_OLDER:-}," in *",$1,"*) return 0 ;; @@ -157,11 +234,12 @@ prune() { esac done rm -f -- "$dir/promoted.db" "$dir/promoted.db.part" "$dir/promoted.tsv" "$dir/pairs.tsv" \ - "$dir/files.tsv" "$dir"/files.tsv.* "$dir/latest.json" + "$dir/files.tsv" "$dir"/files.tsv.* "$dir/latest.json" "$dir/listed.txt" "$dir/recheck.txt" \ + "$dir/recheck.sig" "$dir/signed-now.txt" } plan_target() { - local conf="$1" arch repo name dir repo_path server rc why i + local conf="$1" arch repo name dir repo_path server rc why i dbs=() target_check "$conf" arch="$(target_get "$conf" arch)" repo="$(target_get "$conf" repo)" @@ -173,19 +251,23 @@ plan_target() { : > "$dir/todo.tsv" : > "$dir/meta.env" : > "$dir/nul.txt" + : > "$dir/unsigned.txt" + : > "$dir/signed-now.txt" + printf '%s\n' "$plan_id" > "$dir/plan-id" #-------------------------------------------------------------------------# # 1. Candidates, from the distribution's own servers only #-------------------------------------------------------------------------# i=0 : > "$dir/cand/list.tsv" - while IFS= read -r server; do + mapfile -t dbs < <(target_all "$conf" db_server) + for server in "${dbs[@]}"; do [ -n "$server" ] || continue i=$((i + 1)) rc=0 - fetch_upstream "$server/$repo.db" "$dir/cand/$i.db" "$DB_MAX_TIME" || rc=$? + fetch_within "$server/$repo.db" "$dir/cand/$i.db" "$DB_MAX_TIME" "$((${#dbs[@]} - i + 2))" || rc=$? if [ "$rc" -ne 0 ]; then - log "$name: $server/$repo.db did not answer within ${DB_MAX_TIME}s (status $rc); the next db_server is tried" + log "$name: $server/$repo.db did not answer within ${fetch_max}s (status $rc); the next db_server is tried" continue fi rc=0 @@ -199,10 +281,10 @@ plan_target() { log "$name: $server/$repo.db: $(awk 'END { print NR + 0 }' "$dir/cand/$i.tsv.nul") package directory(ies) whose desc holds only NUL bytes: $(tr '\n' ' ' < "$dir/cand/$i.tsv.nul")" fi printf '%s\t%s\t%s\n' "$i" "$server" "$(sha256sum "$dir/cand/$i.db" | cut -d ' ' -f 1)" >> "$dir/cand/list.tsv" - done <<< "$(target_all "$conf" db_server)" + done if [ ! -s "$dir/cand/list.tsv" ]; then - set_state "$dir" error "no db_server answered with a whole, well-formed database" + set_state "$dir" error "no db_server answered with a whole, well-formed database$(budget_note)" return 0 fi @@ -318,9 +400,59 @@ plan_target() { cp "$dir/cand/$best.tsv.nul" "$dir/nul.txt" #-------------------------------------------------------------------------# - # 4. Current? + # 4. The files promoted with no signature: does upstream sign them now? Only + # a name of a file the plan would skip counts (same sha256, no %PGPSIG%); + # any other is planned anyway, and mirror-file reads its .sig afresh. + #-------------------------------------------------------------------------# + local f src probe_rc answer srcs=() + if [ -n "$promoted_sha" ]; then + jq -r '.annotations["dev.pkgforge.mirror.unsigned"] // empty' "$dir/latest.json" | tr ' ' '\n' > "$dir/listed.txt" + awk -F '\t' ' + FILENAME == ARGV[1] { if ($0 != "") want[$0] = 1; next } + FILENAME == ARGV[2] { if ($8 == "") p[$1] = $5; next } + ($1 in want) && ($1 in p) && p[$1] == $5 && $8 == "" { print $1 } + ' "$dir/listed.txt" "$dir/promoted.tsv" "$dir/records.tsv" > "$dir/recheck.txt" + mapfile -t srcs < <(target_sources "$conf") + while IFS= read -r f; do + answer=404 + for src in "${srcs[@]}"; do + probe_rc=0 + fetch_within "$src/$f.sig" "$dir/recheck.sig" "$SIG_MAX_TIME" 4 || probe_rc=$? + case "$probe_rc" in + 0) + if sig_packet "$dir/recheck.sig"; then + answer=200 + else + answer="$src answered 200 with no OpenPGP signature packet" + fi + break + ;; + 4) ;; + *) + answer="error from $src (status $probe_rc)" + break + ;; + esac + done + rm -f "$dir/recheck.sig" + case "$answer" in + 200) + log "$name: $f was promoted with no signature, and upstream has a .sig for it now; it is planned again" + printf '%s\n' "$f" >> "$dir/signed-now.txt" + ;; + 404) printf '%s\n' "$f" >> "$dir/unsigned.txt" ;; + *) + log "$name: $f was promoted with no signature, and its .sig is not known this run: $answer" + printf '%s\n' "$f" >> "$dir/unsigned.txt" + ;; + esac + done < "$dir/recheck.txt" + fi + + #-------------------------------------------------------------------------# + # 5. Current? #-------------------------------------------------------------------------# - local packages bytes newest + local packages bytes newest signed_now packages="$(awk 'END { print NR + 0 }' "$dir/records.tsv")" bytes="$(awk -F '\t' '{ s += $6 } END { printf "%.0f\n", s }' "$dir/records.tsv")" newest="$(awk -F '\t' '$7 + 0 > m { m = $7 + 0 } END { print m + 0 }' "$dir/records.tsv")" @@ -331,7 +463,8 @@ plan_target() { printf 'NUL_MEMBERS=%s\n' "$(awk 'END { print NR + 0 }' "$dir/nul.txt")" } > "$dir/meta.env" - if [ "$chosen_sha" = "$promoted_sha" ]; then + signed_now="$(awk 'END { print NR + 0 }' "$dir/signed-now.txt")" + if [ "$chosen_sha" = "$promoted_sha" ] && [ "$signed_now" -eq 0 ]; then set_state "$dir" current "the promoted $repo.db is upstream's ($chosen_sha)$(nul_note "$dir")" return 0 fi @@ -342,13 +475,15 @@ plan_target() { # the same sha256, or pacman -F would describe other packages. A server that # fails gives way to the next one with the same bytes. #-------------------------------------------------------------------------# - local files_ok=0 s sig_rc sig_done + local files_ok=0 s sig_rc sig_done cands tried=0 + cands="$(awk -F '\t' -v c="$chosen_sha" '$3 == c { n++ } END { print n + 0 }' "$dir/cand/list.tsv")" while IFS=$'\t' read -r idx srv sha; do [ "$sha" = "$chosen_sha" ] || continue + tried=$((tried + 1)) rc=0 - fetch_upstream "$srv/$repo.files" "$dir/upstream.files" "$FILES_MAX_TIME" || rc=$? + fetch_within "$srv/$repo.files" "$dir/upstream.files" "$FILES_MAX_TIME" "$((cands - tried + 2))" || rc=$? if [ "$rc" -ne 0 ]; then - log "$name: $srv/$repo.files did not answer within ${FILES_MAX_TIME}s (status $rc)" + log "$name: $srv/$repo.files did not answer within ${fetch_max}s (status $rc)" continue fi if ! db_records "$dir/upstream.files" "$dir/files.tsv"; then @@ -377,7 +512,7 @@ plan_target() { for s in db files; do rm -f "$dir/upstream.$s.sig" sig_rc=0 - fetch_upstream "$srv/$repo.$s.sig" "$dir/upstream.$s.sig" "$SIG_MAX_TIME" || sig_rc=$? + fetch_within "$srv/$repo.$s.sig" "$dir/upstream.$s.sig" "$SIG_MAX_TIME" "$((cands - tried + 1))" || sig_rc=$? case "$sig_rc" in 0) log "$name: upstream signs $repo.$s" ;; 4) ;; @@ -392,7 +527,7 @@ plan_target() { break done < "$dir/cand/list.tsv" if [ "$files_ok" -ne 1 ]; then - set_state "$dir" error "no db_server that served $repo.db $chosen_sha also served a $repo.files that matches it, and an answer for its signatures" + set_state "$dir" error "no db_server that served $repo.db $chosen_sha also served a $repo.files that matches it, and an answer for its signatures$(budget_note)" return 0 fi if [ "$(target_get "$conf" db_signed)" = yes ]; then @@ -405,34 +540,43 @@ plan_target() { fi #-------------------------------------------------------------------------# - # 5. Skip or todo + # 6. Skip or todo. A file promoted with no signature that upstream signs now + # is todo, whatever its record says. #-------------------------------------------------------------------------# cut -f 1 "$dir/records.tsv" | "$WRAPPER" --tags > "$dir/pairs.tsv" awk -F '\t' -v OFS='\t' -v target="$name" ' - FILENAME == ARGV[1] { promoted[$1] = $5; psig[$1] = $8; next } - FILENAME == ARGV[2] { tag[$1] = $2; next } - ($1 in promoted) && promoted[$1] == $5 && psig[$1] == $8 { next } + FILENAME == ARGV[1] { again[$1] = 1; next } + FILENAME == ARGV[2] { promoted[$1] = $5; psig[$1] = $8; next } + FILENAME == ARGV[3] { tag[$1] = $2; next } + !($1 in again) && ($1 in promoted) && promoted[$1] == $5 && psig[$1] == $8 { next } { print target, $1, tag[$1], $5, $6, $2, "todo", $7, $8 } - ' "$dir/promoted.tsv" "$dir/pairs.tsv" "$dir/records.tsv" > "$dir/todo.tsv" + ' "$dir/signed-now.txt" "$dir/promoted.tsv" "$dir/pairs.tsv" "$dir/records.tsv" > "$dir/todo.tsv" local resigned resigned="$(awk -F '\t' 'FILENAME == ARGV[1] { p[$1] = $5; s[$1] = $8; next } ($2 in p) && p[$2] == $4 && s[$2] != $9 { n++ } END { print n + 0 }' \ "$dir/promoted.tsv" "$dir/todo.tsv")" if [ "$resigned" -gt 0 ]; then log "$name: $resigned file(s) keep their bytes and carry a new %PGPSIG%; mirror-file repairs their signature layers" fi + if [ "$signed_now" -gt 0 ]; then + log "$name: $signed_now file(s) promoted with no signature have a .sig upstream now; mirror-file adds their signature layers" + fi #-------------------------------------------------------------------------# - # 6. This target's shards + # 7. This target's shards #-------------------------------------------------------------------------# shard_target "$dir" "$name" set_state "$dir" pending "$(awk 'END { print NR + 0 }' "$dir/todo.tsv") file(s) to mirror, $(awk -F '\t' '{ s += $5 } END { printf "%.0f", s / 1048576 }' "$dir/todo.tsv") MiB, in $(find "$dir/shards" -name '*.tsv' | awk 'END { print NR + 0 }') shard(s), database from $chosen_server$(nul_note "$dir")" } +left="$#" for conf in "$@"; do + share_start "$left" plan_target "$conf" prune "$out/$(target_get "$conf" arch)-$(target_get "$conf" repo)" + left=$((left - 1)) done +log "the upstream reads took $((SECONDS + PLAN_BUDGET - run_deadline))s of PLAN_BUDGET=${PLAN_BUDGET}s" #---------------------------------------------------------------------------# # The matrices: one entry per pending target, with its shards diff --git a/scripts/report b/scripts/report index 867c9f6..62b9876 100755 --- a/scripts/report +++ b/scripts/report @@ -15,9 +15,12 @@ # did not run. OUT gets: # state.tsv target state why: promote's word wins over verify's, over # stage's -# verify.tsv every verdict line of the target -# refused.tsv file reason: every upstream-refused package of the target +# plan-id the plan the stage came from (STAGE/TARGET/plan-id) +# verify.tsv every verdict line of the target about the staged digest +# refused.tsv file reason: every upstream-refused package of those verdicts # failing.tsv, replaced.tsv, settings-url as stage left them +# A verdict about another digest, and a promotion of another digest, are +# left over from an earlier attempt of the run: they are not read. # # The second form writes one table for the run: every target of the plan # (PLAN/summary.tsv), with the state each chain collected into @@ -25,6 +28,13 @@ # the upstream-refused packages, per target. To $GITHUB_STEP_SUMMARY when set, # otherwise to stdout. # +# ⛔ A chain's summary counts only when its plan-id is PLAN/plan-id, and only +# for a target this plan marks pending. "Re-run all jobs" plans again, and the +# download of summary-* still returns the earlier attempt's summary for every +# target that has no chain now. Read, it could show a state that is no longer +# true, and raise an issue again. A summary left out is named in the log. +# "Re-run failed jobs" keeps the plan, and every summary of that plan counts. +# # Issues (scripts/track-issue), only when TRACK_ISSUES=1: # visibility- raised when the package reads only with the workflow's # token; cleared once a run reads it anonymously @@ -58,17 +68,18 @@ if [ "${1:-}" = collect ]; then '' | *[!a-z0-9_.-]*) usage ;; esac mkdir -p "$sout" + work_c="$(mktemp -d)" + trap 'rm -rf "$work_c"' EXIT + : > "$work_c/verify.tsv" + : > "$work_c/refused.tsv" : > "$sout/verify.tsv" : > "$sout/refused.tsv" - if [ -d "$verify" ]; then - find "$verify" -name "verify-$target-*.tsv" -type f -exec cat {} + | sort -t $'\t' -k2,2n >> "$sout/verify.tsv" - find "$verify" -name "refused-$target-*.tsv" -type f -exec cat {} + | awk -F '\t' '!seen[$1]++' >> "$sout/refused.tsv" - fi - state="" why="" + : > "$sout/plan-id" + state="" why="" digest="" if [ -f "$stage/$target/state" ]; then state="$(cat "$stage/$target/state")" why="$(cat "$stage/$target/why")" - for f in failing.tsv replaced.tsv settings-url; do + for f in failing.tsv replaced.tsv settings-url plan-id; do if [ -f "$stage/$target/$f" ]; then cp "$stage/$target/$f" "$sout/$f" fi @@ -79,7 +90,26 @@ if [ "${1:-}" = collect ]; then fi if [ "$state" = staged ]; then digest="$(sed -n 's/^DIGEST=//p' "$stage/$target/staged.env")" - failed_line="$(awk -F '\t' -v d="$digest" '$3 != "passed" || $5 != d { print "batch " $2 ": " $3 ": " $4; exit }' "$sout/verify.tsv")" + fi + # The verdicts about the staged digest, and the refusals beside them + if [ -d "$verify" ] && [ -n "$digest" ]; then + find "$verify" -name "verify-$target-*.tsv" -type f > "$work_c/verdicts.txt" + while IFS= read -r v; do + if [ "$(cut -f 5 "$v")" != "$digest" ]; then + log "$target: $v is about $(cut -f 5 "$v"), not the staged $digest; it is not read" + continue + fi + cat "$v" >> "$work_c/verify.tsv" + r="$(dirname "$v")/refused-${v##*/verify-}" + if [ -f "$r" ]; then + cat "$r" >> "$work_c/refused.tsv" + fi + done < "$work_c/verdicts.txt" + sort -t $'\t' -k2,2n "$work_c/verify.tsv" > "$sout/verify.tsv" + awk -F '\t' '!seen[$1]++' "$work_c/refused.tsv" > "$sout/refused.tsv" + fi + if [ "$state" = staged ]; then + failed_line="$(awk -F '\t' '$3 != "passed" && f == "" { f = "batch " $2 ": " $3 ": " $4 } END { print f }' "$sout/verify.tsv")" if [ -n "$failed_line" ]; then state=failed why="$failed_line" @@ -88,10 +118,18 @@ if [ "${1:-}" = collect ]; then why="$(awk 'END { print NR + 0 }' "$sout/verify.tsv") verdict(s) for $(find "$stage/$target/batches" -name '*.txt' | awk 'END { print NR + 0 }') batch(es)" fi fi - if [ -f "$promoted" ] && [ -s "$promoted" ]; then + # promote runs only after a stage of this attempt that succeeded, and its + # "promoted" names the digest it tagged. + if [ -f "$promoted" ] && [ -s "$promoted" ] && [ -n "$digest" ]; then line="$(awk -F '\t' -v t="$target" '$1 == t { l = $2 "\t" $3 } END { print l }' "$promoted")" if [ -n "$line" ]; then - IFS=$'\t' read -r state why <<< "$line" + IFS=$'\t' read -r p_state p_why <<< "$line" + if [ "$p_state" = promoted ] && [ "${p_why%% *}" != "$digest" ]; then + log "$target: the promotion in $promoted is of ${p_why%% *}, not the staged $digest; it is not read" + else + state="$p_state" + why="$p_why" + fi fi fi refused_n="$(awk 'END { print NR + 0 }' "$sout/refused.tsv")" @@ -120,25 +158,48 @@ fi work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT -# target -> final state and why: the chain's word wins over the plan's -tail -n +2 "$plan/summary.tsv" | cut -f 1,2,5 > "$work/states.tsv" +# The summaries of this plan's chains: target, then the directory +plan_id="" +if [ -f "$plan/plan-id" ]; then + plan_id="$(cat "$plan/plan-id")" +else + warn "$plan/plan-id does not exist: this plan names no plan id, so only summaries that name none are read" +fi +: > "$work/dirs.tsv" : > "$work/chains.tsv" if [ -d "$summaries" ]; then - find "$summaries" -name state.tsv -type f -exec cat {} + > "$work/chains.tsv" + find "$summaries" -name state.tsv -type f > "$work/state-files.txt" + while IFS= read -r f; do + d="${f%/state.tsv}" + got="" + if [ -f "$d/plan-id" ]; then + got="$(cat "$d/plan-id")" + fi + if [ "$got" != "$plan_id" ]; then + log "$d is the summary of plan '$got', not of this plan '$plan_id'; it is not read" + continue + fi + cat "$f" >> "$work/chains.tsv" + awk -F '\t' -v d="$d" '{ print $1 "\t" d }' "$f" >> "$work/dirs.tsv" + done < "$work/state-files.txt" fi -# Every pending target has a chain; one that is still pending here left no -# summary, and nothing says what became of it. + +# target -> final state and why. The chain's word wins over the plan's, for a +# target this plan marks pending only. Every pending target has a chain; one +# that is still pending here left no summary, and nothing says what became of +# it. +tail -n +2 "$plan/summary.tsv" | cut -f 1,2,5 > "$work/states.tsv" awk -F '\t' -v OFS='\t' ' FILENAME == ARGV[1] { s[$1] = $2; w[$1] = $3; next } - ($1 in s) { $2 = s[$1]; $3 = w[$1] } - $2 == "pending" { $2 = "failed"; $3 = "its chain left no summary; read the run: " $3 } + ($1 in s) && $2 == "pending" { $2 = s[$1]; $3 = w[$1] } + $2 == "pending" { $2 = "failed"; $3 = "its chain left no summary of this plan; read the run: " $3 } { print } ' "$work/chains.tsv" "$work/states.tsv" > "$work/merged.tsv" mv -f "$work/merged.tsv" "$work/states.tsv" # dir_of TARGET -> the directory its chain collected, or nothing dir_of() { - find "$summaries" -name state.tsv -type f -exec awk -F '\t' -v t="$1" '$1 == t { print FILENAME }' {} + | sed -n 's|/state.tsv$||p' | sed -n 1p + awk -F '\t' -v t="$1" '$1 == t && d == "" { d = $2 } END { print d }' "$work/dirs.tsv" } { diff --git a/scripts/stage b/scripts/stage index 1615b08..d42878d 100755 --- a/scripts/stage +++ b/scripts/stage @@ -11,9 +11,18 @@ # read, and every lines/*.tsv, which a shard killed before its results.tsv # leaves behind. # +# ⛔ Only the results of this plan are read: a shard's directory must hold a +# plan-id that is PLAN/TARGET/plan-id (scripts/mirror-shard). A "Re-run all +# jobs" plans again, and the download of results--* still returns the +# earlier attempt's artifacts for every name this attempt did not upload +# again. Their lines are about another plan; read, a stale "failed" could +# hold the target, or a stale "pushed" could count a file that is not there. +# Each directory left out is named in a warning. +# # OUT/TARGET/state staged held replaced private error, or the plan's own # state for a target that was not pending # OUT/TARGET/why what was measured +# OUT/TARGET/plan-id the plan this stage comes from, for scripts/report # OUT/TARGET/staged.env DIGEST, DB_SHA256, FILES_SHA256, TAG of the staged manifest # OUT/TARGET/batches/.txt package names, one per line # OUT/TARGET/canaries.txt the names verify reads first, through the @@ -30,6 +39,14 @@ # itself. A file whose tag holds other bytes (replaced) holds the target until # a person decides, and stage says so at once. # +# The staged manifest names the files whose tag holds no signature, because +# upstream publishes none for them, in its dev.pkgforge.mirror.unsigned +# annotation, space separated: those plan carried over (PLAN/TARGET/ +# unsigned.txt), and those mirror-file left unsigned in this run (a result +# detail that starts "unsigned: "). scripts/plan asks upstream for their .sig +# on every run, and plans a file again once upstream signs it. The list stops +# at 16 KiB, so the manifest stays far below the 64 KiB the wrapper reads. +# # Verification batches: every file new since the promoted database, plus the # canaries, which are always present: the smallest package and every keyring # package of the target, so a run with nothing new still proves the path a @@ -87,14 +104,39 @@ ORAS="$("$REPO_ROOT/scripts/install-oras" "$work/bin")" ORAS_CONFIG="$work/oras-auth.json" oras_login "$ORAS" "$ORAS_CONFIG" -# Every result line from every shard: target filename status seconds detail. No -# results at all is ordinary: a database that changed by dropping packages -# names no new file. A results.tsv and the lines/ beside it say the same. +# Every result line from every shard of this plan: target filename status +# seconds detail. No results at all is ordinary: a database that changed by +# dropping packages names no new file. A results.tsv and the lines/ beside it +# say the same. +plan_id="" +if [ -f "$plan/$target/plan-id" ]; then + plan_id="$(cat "$plan/$target/plan-id")" +else + warn "$plan/$target/plan-id does not exist: this plan names no plan id, so only results that name none are read" +fi : > "$work/results.tsv" +: > "$work/result-dirs.txt" if [ -d "$results" ]; then - find "$results" -name results.tsv -type f -exec cat {} + >> "$work/results.tsv" - find "$results" -path '*/lines/*' -name '[0-9]*.tsv' -type f -exec cat {} + >> "$work/results.tsv" + find "$results" \( -name results.tsv -type f \) -o \( -name lines -type d \) > "$work/result-paths.txt" + sed 's|/[^/]*$||' "$work/result-paths.txt" | sort -u > "$work/result-dirs.txt" fi +while IFS= read -r d; do + got="" + if [ -f "$d/plan-id" ]; then + got="$(cat "$d/plan-id")" + fi + if [ "$got" != "$plan_id" ]; then + warn "$d holds the results of plan '$got', not of this plan '$plan_id'; they are not read" \ + "an earlier attempt of the run left them: Re-run all jobs plans again, and the artifacts of the earlier attempt stay" + continue + fi + if [ -f "$d/results.tsv" ]; then + cat "$d/results.tsv" >> "$work/results.tsv" + fi + if [ -d "$d/lines" ]; then + find "$d/lines" -maxdepth 1 -name '[0-9]*.tsv' -type f -exec cat {} + >> "$work/results.tsv" + fi +done < "$work/result-dirs.txt" set_state() { printf '%s\n' "$2" > "$1/state" @@ -163,6 +205,27 @@ stage_target() { : "${PRIVATE:=0}" "${ARCH:?}" "${REPO:?}" "${UPSTREAM:?}" "${PACKAGES:?}" "${BYTES:?}" "${NEWEST_BUILDDATE:?}" "${FETCHED:?}" local source_url source_url="${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:-pkgforge-dev/mirror-archlinux}" + + # The files whose tag holds no signature, as far as the database names them + local unsigned_list="" unsigned_n unsigned_cut + { + if [ -f "$pdir/unsigned.txt" ]; then + cat "$pdir/unsigned.txt" + fi + awk -F '\t' -v t="$name" '$1 == t && ($3 == "pushed" || $3 == "present") && index($5, "unsigned: ") == 1 { print $2 }' "$work/results.tsv" + } | sort -u > "$work/unsigned-all.txt" + awk -F '\t' 'FILENAME == ARGV[1] { r[$1] = 1; next } ($0 in r)' "$pdir/records.tsv" "$work/unsigned-all.txt" > "$work/unsigned-named.txt" + awk -v max=16384 '{ n += length($0) + 1; if (n <= max) print }' "$work/unsigned-named.txt" > "$work/unsigned.txt" + unsigned_n="$(awk 'END { print NR + 0 }' "$work/unsigned.txt")" + unsigned_cut=$(($(awk 'END { print NR + 0 }' "$work/unsigned-named.txt") - unsigned_n)) + if [ "$unsigned_n" -gt 0 ]; then + unsigned_list="$(tr '\n' ' ' < "$work/unsigned.txt" | sed 's/ $//')" + log "$name: $unsigned_n file(s) have no signature, as upstream ships them; the manifest names them, and plan asks for their .sig again on each run" + fi + if [ "$unsigned_cut" -gt 0 ]; then + warn "$name: $unsigned_cut more file(s) have no signature and are not named: the list stops at 16 KiB" \ + "plan does not ask for their .sig again until upstream changes their database record" + fi local oargs=() mapfile -t oargs < <(oras_args) local args=( @@ -182,9 +245,11 @@ stage_target() { --annotation "dev.pkgforge.mirror.newest-builddate=$NEWEST_BUILDDATE" --annotation "dev.pkgforge.mirror.fetched=$FETCHED" --annotation "dev.pkgforge.mirror.run=${GITHUB_RUN_ID:-local}" - "$MIRROR_REGISTRY/$repo_path:$STAGING_TAG" - "${layers[@]}" ) + if [ -n "$unsigned_list" ]; then + args+=(--annotation "dev.pkgforge.mirror.unsigned=$unsigned_list") + fi + args+=("$MIRROR_REGISTRY/$repo_path:$STAGING_TAG" "${layers[@]}") local rc=0 attempt for attempt in 1 2 3; do rc=0 @@ -279,6 +344,7 @@ stage_target() { } stage_target "$plan/$target" +printf '%s\n' "$plan_id" > "$out/$target/plan-id" #---------------------------------------------------------------------------# # The verify matrix diff --git a/tests/fixtures/build-repo b/tests/fixtures/build-repo index e4c8727..db842b3 100644 --- a/tests/fixtures/build-repo +++ b/tests/fixtures/build-repo @@ -32,6 +32,14 @@ # desc is a GNU sparse member of NUL bytes only, as aarch64 extra # findnewest-0.3-4 is upstream # port-junk as port, with junk-1-1/desc holding bytes that are no record +# port-files-missing as port, with expired-1-1 left out of port.files only, +# as Arch Linux ARM aarch64 aur.files left out localepurge-0.7.3.4-1 +# and alarm.files dkms-mt7601 (2026-10-14) +# port-files-extra as port, with port.files naming one more package, +# stray 1-1, that port.db does not +# port-signed the unsigned package of DIR (aarch64/port) signed now, beside +# it: the database and every other file stay as they are, as when +# upstream uploads a .sig late for a database that embeds none # # Every database carries %PGPSIG% for every signed package, as Arch's own core # and extra do, except nopgpsig. pacman 7's repo-add embeds a signature only @@ -94,9 +102,6 @@ if [ ! -d "$GNUPGHOME" ]; then mkdir -p /out/sigcache fi -rm -rf "$repo" -mkdir -p "$repo" - # sign FILE HOME [fresh]: FILE.sig, made once per distinct file and key sign() { local file="$1" home="$2" sum cached faked=() @@ -112,6 +117,17 @@ sign() { cp "$cached" "$file.sig" } +if [ "$state" = port-signed ]; then + [ -f "$repo/unsigned-1-1-any.pkg.tar.zst" ] + [ ! -e "$repo/unsigned-1-1-any.pkg.tar.zst.sig" ] + sign "$repo/unsigned-1-1-any.pkg.tar.zst" "$GNUPGHOME" + ls -l "$repo" + exit 0 +fi + +rm -rf "$repo" +mkdir -p "$repo" + # mkpkg NAME VERSION ARCH BUILDDATE CONTENT [GNUPGHOME|none] [fresh] mkpkg() { local name="$1" version="$2" arch="$3" built="$4" content="$5" home="${6:-$GNUPGHOME}" d file @@ -164,6 +180,19 @@ add_member() { rm -rf "$t" } +# drop_member DB.tar.gz DIR: the archive without the package directory DIR +drop_member() { + local t + t="$(mktemp -d)" + mkdir -p "$t/m" + tar -xzf "$1" -C "$t/m" + [ -d "$t/m/$2" ] + rm -rf "${t:?}/m/$2" + (cd "$t/m" && bsdtar -czf "$t/db.tar.gz" -- *) + mv -f "$t/db.tar.gz" "$1" + rm -rf "$t" +} + case "$state" in keyring) mkpkg test-keyring 1-1 any 1700000050 "the keyring of the test keys" @@ -171,11 +200,24 @@ case "$state" in ls -l "$repo" exit 0 ;; - port | port-junk) + port | port-junk | port-files-missing | port-files-extra) mkpkg port 1-1 aarch64 1700000000 "an aarch64 package" mkpkg unsigned 1-1 any 1700000010 "upstream signs this nowhere" none mkpkg expired 1-1 any 1700000020 "signed by a key that has since expired" /out/gnupg-expired repo-add --quiet --include-sigs "$repo/$db.db.tar.gz" "$repo"/*.pkg.tar.zst + case "$state" in + port-files-missing) drop_member "$repo/$db.files.tar.gz" expired-1-1 ;; + port-files-extra) + # The files database of a repository with stray in it, and stray + # itself nowhere else. + mkdir -p "$repo/.stray" + mkpkg stray 1-1 any 1700000030 "named by the files database only" + mv "$repo"/stray-1-1-any.pkg.tar.zst* "$repo/.stray/" + repo-add --quiet --include-sigs "$repo/.stray/x.db.tar.gz" "$repo"/*.pkg.tar.zst "$repo/.stray/stray-1-1-any.pkg.tar.zst" + cp "$repo/.stray/x.files.tar.gz" "$repo/$db.files.tar.gz" + rm -rf "$repo/.stray" + ;; + esac for f in "$db.db.tar.gz" "$db.files.tar.gz"; do add_member "$repo/$f" ghost-1-1 nul if [ "$state" = port-junk ]; then diff --git a/tests/integration/10-pipeline.sh b/tests/integration/10-pipeline.sh index c8f43f5..1fa003e 100644 --- a/tests/integration/10-pipeline.sh +++ b/tests/integration/10-pipeline.sh @@ -3,7 +3,9 @@ # The whole mirror, end to end, against a registry and an upstream of its own. # # A local registry (registry:3) stands in for GHCR and a local web server for -# the upstream mirrors. Three targets are built, each a signed repository: +# the upstream mirrors. A second web server stands in for an upstream host +# that never answers, or answers 503. Three targets are built, each a signed +# repository: # x86_64-test filenames that stress the tag mapping; its keyring package # comes from x86_64-kr (keyring_from) # x86_64-kr the keyring package of the test keys @@ -41,9 +43,10 @@ work="$(work_dir)" suffix="$$-$RANDOM" reg_name="mirror-it-registry-$suffix" up_name="mirror-it-upstream-$suffix" +broken_name="mirror-it-broken-$suffix" kimage_local="mirror-it-keyring-$suffix" cleanup() { - "$RUNTIME" rm -f "$reg_name" "$up_name" > "$work/cleanup.log" 2>&1 || echo "# cleanup: $(cat "$work/cleanup.log")" >&2 + "$RUNTIME" rm -f "$reg_name" "$up_name" "$broken_name" > "$work/cleanup.log" 2>&1 || echo "# cleanup: $(cat "$work/cleanup.log")" >&2 if [ "${IT_KEEP:-0}" = 1 ]; then echo "# kept $work" >&2 else @@ -63,7 +66,21 @@ case "$(basename "$RUNTIME")" in ;; esac -mkdir -p "$work/upstream" "$work/targets" "$work/out" +mkdir -p "$work/upstream" "$work/targets" "$work/out" "$work/broken/cgi-bin" "$work/broken-log" + +# A second upstream host that fails, on a port of its own: each request is +# logged in broken-log/hits, then a path under /stall/ gets no answer for +# 300 s, and a path under /fail/ gets 503 at once. +cat > "$work/broken/cgi-bin/b" << 'EOF' +#!/bin/sh +printf '%s\n' "$PATH_INFO" >> /log/hits +case "$PATH_INFO" in + /stall/*) sleep 300 ;; +esac +printf 'HTTP/1.1 503 Service Unavailable\r\nContent-Length: 5\r\n\r\nbusy\n' +EOF +chmod 0755 "$work/broken/cgi-bin/b" +: > "$work/broken-log/hits" #---------------------------------------------------------------------------# # The registry and the upstream, on ports the runtime picks @@ -75,13 +92,17 @@ port_of() { "$RUNTIME" run -d --rm --name "$reg_name" -p 127.0.0.1::5000 "$REGISTRY_IMAGE" > /dev/null "$RUNTIME" run -d --rm --name "$up_name" -p 127.0.0.1::8000 -v "$work/upstream:/srv:ro" \ "$HTTPD_IMAGE" httpd -f -p 8000 -h /srv > /dev/null +"$RUNTIME" run -d --rm --name "$broken_name" "${as_me[@]}" -p 127.0.0.1::8000 -v "$work/broken:/srv:ro" \ + -v "$work/broken-log:/log" "$HTTPD_IMAGE" httpd -f -p 8000 -h /srv > /dev/null REG_PORT="$(port_of "$reg_name" 5000)" UP_PORT="$(port_of "$up_name" 8000)" +BROKEN_PORT="$(port_of "$broken_name" 8000)" ready=0 -if [ -n "$REG_PORT" ] && [ -n "$UP_PORT" ]; then +if [ -n "$REG_PORT" ] && [ -n "$UP_PORT" ] && [ -n "$BROKEN_PORT" ]; then for _ in $(seq 1 30); do if curl -q -fsS --connect-timeout 2 --max-time 5 "http://127.0.0.1:$REG_PORT/v2/" > /dev/null && - curl -q -sS --connect-timeout 2 --max-time 5 -o /dev/null "http://127.0.0.1:$UP_PORT/"; then + curl -q -sS --connect-timeout 2 --max-time 5 -o /dev/null "http://127.0.0.1:$UP_PORT/" && + curl -q -sS --connect-timeout 2 --max-time 5 -o /dev/null "http://127.0.0.1:$BROKEN_PORT/"; then ready=1 break fi @@ -89,12 +110,12 @@ if [ -n "$REG_PORT" ] && [ -n "$UP_PORT" ]; then done fi if [ "$ready" -ne 1 ]; then - fail "the local registry and upstream answer" "registry port '$REG_PORT', upstream port '$UP_PORT', after 30 s" \ - "reproduce: $RUNTIME logs $reg_name; $RUNTIME logs $up_name" + fail "the local registry and upstream answer" "registry port '$REG_PORT', upstream ports '$UP_PORT' and '$BROKEN_PORT', after 30 s" \ + "reproduce: $RUNTIME logs $reg_name; $RUNTIME logs $up_name; $RUNTIME logs $broken_name" summary exit 1 fi -echo "# registry on 127.0.0.1:$REG_PORT, upstream on 127.0.0.1:$UP_PORT" +echo "# registry on 127.0.0.1:$REG_PORT, upstream on 127.0.0.1:$UP_PORT, failing upstream on 127.0.0.1:$BROKEN_PORT" # podman reads a registry over plain http only when its configuration says # so; docker does for any registry on the loopback. @@ -124,6 +145,7 @@ export ORAS_CACHE="${IT_ORAS_CACHE:-$work/oras-cache}" REPO="it/mirror/x86_64/test" BASE="http://127.0.0.1:$REG_PORT/v2/$REPO" UP="http://127.0.0.1:$UP_PORT" +BROKEN="http://127.0.0.1:$BROKEN_PORT/cgi-bin/b" #---------------------------------------------------------------------------# # The upstream repositories, and the suite's keyring image @@ -476,6 +498,79 @@ else fail "the refusal names the member" "$(cat "$work/out/junk.log")" fi +#---------------------------------------------------------------------------# +# 1f. A file promoted with no signature, because upstream had none, gets its +# signature once upstream publishes a .sig, while the database stays the +# same: the snapshot names the file, plan asks for its .sig on each run, +# and plans it again when one answers. +#---------------------------------------------------------------------------# +unsigned_f='unsigned-1-1-any.pkg.tar.zst' +port_repo=it/mirror/aarch64/port +check "the snapshot names the file it holds with no signature" "$unsigned_f" \ + "$(manifest latest "$port_repo" | jq -r '.annotations["dev.pkgforge.mirror.unsigned"] // "none"')" \ + "curl http://127.0.0.1:$REG_PORT/v2/$port_repo/manifests/latest | jq .annotations; read out/1/stage-aarch64-port.log" +port_db_before="$(sha256sum "$work/upstream/aarch64/port/port.db.tar.gz" | cut -d ' ' -f 1)" +port_latest_before="$(digest_of latest "$port_repo")" +build port-signed aarch64/port port +sync 1f +check "upstream signs it later, and its database stays the same" "$port_db_before" \ + "$(sha256sum "$work/upstream/aarch64/port/port.db.tar.gz" | cut -d ' ' -f 1)" "read $work/build-port-signed.log" +check "a file promoted with no signature that upstream signs now is planned again, alone" "pending $unsigned_f" \ + "$(state_of 1f plan aarch64-port) $(cut -f 2 "$work/out/1f/plan/aarch64-port/todo.tsv" 2> /dev/null | tr '\n' ' ' | sed 's/ $//')" \ + "read out/1f/plan.log" +check "its tag gets upstream's new .sig as its signature layer" \ + "sha256:$(sha256sum "$work/upstream/aarch64/port/$unsigned_f.sig" | cut -d ' ' -f 1)" \ + "$(manifest "$("$REPO_ROOT/client/pacman-oci" --tag "$unsigned_f")" "$port_repo" | jq -r --arg t "$unsigned_f.sig" '[.layers[] | select(.annotations["org.opencontainers.image.title"] == $t) | .digest] | first // "none"')" \ + "read out/1f/mirror-aarch64-port-0.log" +port_latest_after="$(digest_of latest "$port_repo")" +check "the snapshot is verified and promoted, and names no file with no signature" "passed moved none" \ + "$(verdict_of 1f aarch64-port | cut -f 1) $([ -n "$port_latest_after" ] && [ "$port_latest_after" != "$port_latest_before" ] && echo moved || echo stayed) $(manifest latest "$port_repo" | jq -r '.annotations["dev.pkgforge.mirror.unsigned"] // "none"')" \ + "read out/1f/verify/verify-aarch64-port-0.log out/1f/promote-aarch64-port.log" + +#---------------------------------------------------------------------------# +# 1g. A files database that leaves out a package of the database is upstream's +# own defect: planned, the package named, and verify runs pacman -Fl only +# over the names the files database holds. One that names a package the +# database does not is refused. Each in a namespace of its own. +#---------------------------------------------------------------------------# +build port-files-missing aarch64/port-fmiss port +mkdir -p "$work/targets-fmiss" +# shellcheck disable=SC2016 # $arch and $repo are the target file's, not the shell's +sed 's|/\$arch/\$repo$|/$arch/port-fmiss|; s|/\$arch/\./\$repo$|/$arch/./port-fmiss|' "$work/targets/aarch64-port.conf" > "$work/targets-fmiss/aarch64-port.conf" +TARGETS_DIR="$work/targets-fmiss" MIRROR_NAMESPACE=it/fmiss sync fmiss aarch64-port +check "a files database that leaves out a package of the database is planned" pending "$(state_of fmiss plan aarch64-port)" \ + "read out/fmiss/plan.log" +if grep -q 'port.files leaves out 1 package(s) of port.db, as upstream serves it: expired-1-1-any.pkg.tar.zst' "$work/out/fmiss/plan.log"; then + ok "the plan names the package the files database leaves out" +else + fail "the plan names the package the files database leaves out" "read out/fmiss/plan.log" +fi +if grep -q 'the files database holds no entry for 1 package(s) of the batch, as upstream serves it: expired' "$work/out/fmiss/verify/verify-aarch64-port-0.log"; then + ok "verify names the package the files database has no entry for" +else + fail "verify names the package the files database has no entry for" "read out/fmiss/verify/verify-aarch64-port-0.log" +fi +check "verify runs pacman -Fl over the other names, passes, and the snapshot is promoted" "passed promoted" \ + "$(verdict_of fmiss aarch64-port | cut -f 1) $(cut -f 2 "$work/out/fmiss/promote-aarch64-port.tsv" 2> /dev/null)" \ + "read out/fmiss/verify/verify-aarch64-port-0.log out/fmiss/promote-aarch64-port.log" + +build port-files-extra aarch64/port-fextra port +mkdir -p "$work/targets-fextra" +# shellcheck disable=SC2016 # $arch and $repo are the target file's, not the shell's +sed 's|/\$arch/\$repo$|/$arch/port-fextra|; s|/\$arch/\./\$repo$|/$arch/./port-fextra|' "$work/targets/aarch64-port.conf" > "$work/targets-fextra/aarch64-port.conf" +rc=0 +TARGETS_DIR="$work/targets-fextra" MIRROR_NAMESPACE=it/fextra "$REPO_ROOT/scripts/plan" "$work/out/fextra" > "$work/out/fextra.log" 2>&1 || rc=$? +case "$(cat "$work/out/fextra/aarch64-port/state" 2> /dev/null):$(cat "$work/out/fextra/aarch64-port/why" 2> /dev/null)" in + "error:no db_server that served port.db "*" also served a port.files that matches it"*) ok "a files database that names a package the database does not is refused" ;; + *) fail "a files database that names a package the database does not is refused" "plan exit $rc: $(cat "$work/out/fextra/aarch64-port/why" 2> /dev/null)" \ + "read out/fextra.log" ;; +esac +if grep -q 'port.files names 1 package file(s) that port.db does not, e.g. stray-1-1-any.pkg.tar.zst' "$work/out/fextra.log"; then + ok "the refusal names the package" +else + fail "the refusal names the package" "read out/fextra.log" +fi + #---------------------------------------------------------------------------# # 2. Consumers: pacman through the wrapper, nothing else #---------------------------------------------------------------------------# @@ -540,6 +635,37 @@ check "every target is current, and the report says so" "current current current "$(state_of 2 plan x86_64-test) $(state_of 2 plan x86_64-kr) $(state_of 2 plan aarch64-port) $(cat "$work/out/2/report.rc")" \ "cat out/2/plan/summary.tsv out/2/report.md" +#---------------------------------------------------------------------------# +# 3b. A db_server that accepts and never answers, listed first. Each read of +# it ends within its part of the target's share of PLAN_BUDGET, the next +# db_server is read, and the plan ends within PLAN_BUDGET. A target whose +# only db_server never answers is in error, and its why names the budget. +# With no deadline over the retries, each read of it took 184 s. +#---------------------------------------------------------------------------# +mkdir -p "$work/targets-slow" +for t in x86_64-kr x86_64-test; do + # shellcheck disable=SC2016 # $arch and $repo are the target file's, not the shell's + { printf 'db_server = %s/stall/$arch/$repo\n' "$BROKEN"; cat "$work/targets/$t.conf"; } > "$work/targets-slow/$t.conf" +done +# shellcheck disable=SC2016 # $arch and $repo are the target file's, not the shell's +printf 'arch = x86_64\nrepo = stalled\ndb_server = %s/stall/$arch/$repo\n' "$BROKEN" > "$work/targets-slow/x86_64-stalled.conf" +started_at=$SECONDS +rc=0 +TARGETS_DIR="$work/targets-slow" PLAN_BUDGET=40 "$REPO_ROOT/scripts/plan" "$work/out/slow" > "$work/out/slow.log" 2>&1 || rc=$? +took=$((SECONDS - started_at)) +check "a db_server that never answers gives way to the next one, and the targets are current" "0 current current" \ + "$rc $(cat "$work/out/slow/x86_64-kr/state" 2> /dev/null || echo none) $(cat "$work/out/slow/x86_64-test/state" 2> /dev/null || echo none)" \ + "read out/slow.log" +check "the plan ends within PLAN_BUDGET=40 s, with 15 s for its registry reads" "within 55 s" \ + "$([ "$took" -le 55 ] && echo 'within 55 s' || echo "after $took s")" "read out/slow.log" +check "each read of the stalled db_server ends within its part of the share, not DB_MAX_TIME (300 s)" 3 \ + "$(grep -cE "/stall/x86_64/(kr|test|stalled)/(kr|test|stalled)\.db did not answer within [0-9]{1,2}s" "$work/out/slow.log")" "read out/slow.log" +case "$(cat "$work/out/slow/x86_64-stalled/state" 2> /dev/null):$(cat "$work/out/slow/x86_64-stalled/why" 2> /dev/null)" in + error:*PLAN_BUDGET=40s*) ok "a target whose only db_server never answers is in error, and its why names the budget" ;; + *) fail "a target whose only db_server never answers is in error, and its why names the budget" \ + "$(cat "$work/out/slow/x86_64-stalled/why" 2> /dev/null)" "read out/slow.log" ;; +esac + #---------------------------------------------------------------------------# # 4. Two packages new, one of them signed by a key only the keyring package # of x86_64-kr holds: only they are mirrored, verify installs that keyring @@ -586,6 +712,37 @@ else fi check "latest stays on the newer snapshot" "$latest3" "$(digest_of latest)" "curl -I $BASE/manifests/latest" +#---------------------------------------------------------------------------# +# 4a. "Re-run all jobs" plans again, and the download of results--* +# still returns an earlier attempt's results beside this plan's. stage +# reads only this plan's: a stale "failed" for a file this plan's shard +# pushed holds nothing. The layout is the workflow's, one directory per +# artifact, and the stale line comes last, as it would win. +#---------------------------------------------------------------------------# +rr="$work/out/rerun-results" +rm -rf "$rr" +for d in "$work/out/3/results/x86_64-test"/*/; do + n="$(basename "$d")" + mkdir -p "$rr/results-x86_64-test-$n/$n" + cp "$d/results.tsv" "$d/plan-id" "$rr/results-x86_64-test-$n/$n/" +done +stale_f="$(cut -f 2 "$work/out/3/plan/x86_64-test/todo.tsv" | sed -n 1p)" +mkdir -p "$rr/results-x86_64-test-7/7/lines" +printf 'earlier-attempt\n' > "$rr/results-x86_64-test-7/7/plan-id" +printf 'x86_64-test\t%s\tfailed\t600\tpush timed out\n' "$stale_f" > "$rr/results-x86_64-test-7/7/lines/1.tsv" +rc=0 +STAGING_TAG=staging-rerun "$REPO_ROOT/scripts/stage" "$work/out/3/plan" "$rr" "$work/out/rerun-stage" x86_64-test \ + > "$work/out/rerun-stage.log" 2>&1 || rc=$? +check "a re-run's stage reads no result of an earlier plan" "0 staged" \ + "$rc $(cat "$work/out/rerun-stage/x86_64-test/state" 2> /dev/null || echo 'no state')" "read out/rerun-stage.log" +if grep -q "results-x86_64-test-7/7 holds the results of plan 'earlier-attempt'" "$work/out/rerun-stage.log"; then + ok "the stage log names the results it left out" +else + fail "the stage log names the results it left out" "read out/rerun-stage.log" +fi +check "the stage carries its plan id, for the report" "$(cat "$work/out/3/plan/plan-id")" \ + "$(cat "$work/out/rerun-stage/x86_64-test/plan-id" 2> /dev/null || echo none)" "ls out/rerun-stage/x86_64-test" + #---------------------------------------------------------------------------# # 4b. Upstream signs a file again, its bytes unchanged: the file is planned, # and its signature layer is repaired under the same tag @@ -754,6 +911,69 @@ esac check "it is never promoted" "$latest10" "$(digest_of latest)" "curl -I $BASE/manifests/latest" put_manifest "$canary_tag" "$work/canary-good.json" || diag "the canary manifest could not be written back" +#---------------------------------------------------------------------------# +# 9e. With no %PGPSIG%, the signature layer is the .sig of the target's +# servers in their own order, the db_servers first, as verify reads it. +# A shard whose package reads start at a plain server that serves other +# .sig bytes still pushes the db_server's. Each in a namespace of its own. +#---------------------------------------------------------------------------# +rm -rf "$work/upstream/x86_64/test-plain" +cp -r "$work/upstream/x86_64/test" "$work/upstream/x86_64/test-plain" +mapfile -t plain_sigs < <(cd "$work/upstream/x86_64/test-plain" && ls -- *.pkg.tar.zst.sig) +for ((i = 0; i < ${#plain_sigs[@]}; i++)); do + cp "$work/upstream/x86_64/test/${plain_sigs[$(((i + 1) % ${#plain_sigs[@]}))]}" "$work/upstream/x86_64/test-plain/${plain_sigs[$i]}" +done +mkdir -p "$work/targets-sigsrc" +cp "$work/targets/x86_64-kr.conf" "$work/targets-sigsrc/" +# shellcheck disable=SC2016 # $arch is the target file's, not the shell's +sed 's|^server = .*$|server = '"$UP"'/$arch/test-plain|' "$work/targets/x86_64-test.conf" > "$work/targets-sigsrc/x86_64-test.conf" +rc=0 +TARGETS_DIR="$work/targets-sigsrc" MIRROR_NAMESPACE=it/sigsrc MAX_SHARDS=1 "$REPO_ROOT/scripts/plan" "$work/out/sigsrc" \ + "$work/targets-sigsrc/x86_64-test.conf" > "$work/out/sigsrc.log" 2>&1 || rc=$? +# Shard index 1 of two servers: the shard reads packages from test-plain first. +TARGETS_DIR="$work/targets-sigsrc" MIRROR_NAMESPACE=it/sigsrc SHARD_INDEX=1 "$REPO_ROOT/scripts/mirror-shard" \ + "$work/out/sigsrc/x86_64-test/shards/0.tsv" "$work/out/sigsrc-results" >> "$work/out/sigsrc.log" 2>&1 || rc=$? +same=0 +differ="" +while IFS=$'\t' read -r _ f tag _; do + if [ "$(manifest "$tag" it/sigsrc/x86_64/test | jq -r --arg t "$f.sig" '[.layers[] | select(.annotations["org.opencontainers.image.title"] == $t) | .digest] | first // "none"')" = \ + "sha256:$(sha256sum "$work/upstream/x86_64/test/$f.sig" | cut -d ' ' -f 1)" ]; then + same=$((same + 1)) + else + differ="$differ $f" + fi +done < "$work/out/sigsrc/x86_64-test/shards/0.tsv" +check "a shard that reads packages from a plain server first takes each .sig from the db_server" \ + "${#plain_sigs[@]} of ${#plain_sigs[@]}" "$same of $(awk 'END { print NR + 0 }' "$work/out/sigsrc/x86_64-test/shards/0.tsv")" \ + "status $rc; layers not the db_server's .sig:$differ; read out/sigsrc.log" + +#---------------------------------------------------------------------------# +# 9f. A server that fails, first in a shard's order: after it fails for one +# file, every file of the shard asks it last. It gets one file's requests +# (curl's three tries of a 503), not every file's. +#---------------------------------------------------------------------------# +mkdir -p "$work/targets-down" +cp "$work/targets/x86_64-kr.conf" "$work/targets-down/" +# shellcheck disable=SC2016 # $arch and $repo are the target file's, not the shell's +sed 's|^server = .*$|server = '"$BROKEN"'/fail/$arch/$repo|' "$work/targets/x86_64-test.conf" > "$work/targets-down/x86_64-test.conf" +rc=0 +TARGETS_DIR="$work/targets-down" MIRROR_NAMESPACE=it/down MAX_SHARDS=1 "$REPO_ROOT/scripts/plan" "$work/out/down" \ + "$work/targets-down/x86_64-test.conf" > "$work/out/down.log" 2>&1 || rc=$? +: > "$work/broken-log/hits" +TARGETS_DIR="$work/targets-down" MIRROR_NAMESPACE=it/down SHARD_INDEX=1 PARALLEL=1 "$REPO_ROOT/scripts/mirror-shard" \ + "$work/out/down/x86_64-test/shards/0.tsv" "$work/out/down-results" >> "$work/out/down.log" 2>&1 || rc=$? +down_files="$(awk 'END { print NR + 0 }' "$work/out/down/x86_64-test/shards/0.tsv")" +check "every file of the shard is pushed from the next server" "$down_files pushed" \ + "$(awk -F '\t' '$3 == "pushed" { n++ } END { print n + 0 }' "$work/out/down-results/results.tsv" 2> /dev/null) pushed" \ + "status $rc; read out/down.log" +check "the failing server gets the requests of one file only, for $down_files files" 3 \ + "$(awk '/^\/fail\// { n++ } END { print n + 0 }' "$work/broken-log/hits")" "cat $work/broken-log/hits; read out/down.log" +if grep -q 'every file of this shard asks it last from now on' "$work/out/down.log"; then + ok "the shard log names the server it asks last" +else + fail "the shard log names the server it asks last" "read out/down.log" +fi + #---------------------------------------------------------------------------# # 10. What the wrapper must refuse, served by a plain web server standing in # for a registry that lies: a manifest whose bytes are not the digest the diff --git a/tests/unit/40-lib.sh b/tests/unit/40-lib.sh index 80329b6..9db6cf1 100644 --- a/tests/unit/40-lib.sh +++ b/tests/unit/40-lib.sh @@ -3,7 +3,9 @@ # The decisions the sync makes, each on fixed inputs: which target files are # valid, which database fields are refused, which database is behind another, # when a package tag may be written, when a target is stale, which snapshot is -# newer, and how a shard accounts for every file. +# newer, how a shard accounts for every file, which token a registry read +# sends, which server is asked last, how long an upstream read may take, and +# which artifacts of a re-run the report reads. set -euo pipefail # shellcheck source=../lib/harness.sh . "$REPO_ROOT/tests/lib/harness.sh" @@ -372,14 +374,17 @@ fake="$work/fake" mkdir -p "$fake/bin" cat > "$fake/bin/curl" << 'EOF' #!/usr/bin/env bash -# Logs "URLAUTH" to $FAKE/calls.log; answers a token request with -# $FAKE/next-token, and a manifest request from $FAKE/answers.tsv: -# AUTHCODEBODY-FILE, the first line whose AUTH matches. -out="" auth="" url="" +# Logs "URLAUTH" to $FAKE/calls.log; answers a token request that +# carries a credential (a user line on stdin, --config -) with +# $FAKE/next-token, one with none with "anon", and a manifest request from +# $FAKE/answers.tsv: AUTHCODEBODY-FILE, the first line whose AUTH +# matches. +out="" auth="" url="" cred="" while [ "$#" -gt 0 ]; do case "$1" in --output) out="$2"; shift 2 ;; --header) case "$2" in Authorization:*) auth="${2#Authorization: }" ;; esac; shift 2 ;; + --config) if [ "$2" = - ]; then cred="$(cat)"; fi; shift 2 ;; --write-out | --user | --data-urlencode | --connect-timeout | --max-time | --retry | --retry-delay | \ --max-filesize | --proto | --proto-redir | --max-redirs | --speed-limit | --speed-time | --user-agent) shift 2 ;; -*) shift ;; @@ -388,7 +393,12 @@ while [ "$#" -gt 0 ]; do done printf '%s\t%s\n' "$url" "$auth" >> "$FAKE/calls.log" case "$url" in - */token) printf '{"token":"%s"}' "$(cat "$FAKE/next-token")" > "$out" ;; + */token) + case "$cred" in + *'user = '*) printf '{"token":"%s"}' "$(cat "$FAKE/next-token")" > "$out" ;; + *) printf '{"token":"anon"}' > "$out" ;; + esac + ;; */manifests/*) line="$(awk -F '\t' -v a="$auth" '$1 == a && !f { print $2 "\t" $3; f = 1 }' "$FAKE/answers.tsv")" [ -n "$line" ] || { printf '000'; exit 7; } @@ -406,7 +416,7 @@ reg() { local rc=0 printf '%b' "$2" > "$fake/answers.tsv" : > "$fake/calls.log" - FAKE="$fake" PATH="$fake/bin:$PATH" MIRROR_REGISTRY=ghcr.io MIRROR_REGISTRY_AUTH=token MIRROR_REGISTRY_SCHEME=https \ + FAKE="$fake" PATH="$fake/bin:$PATH" MIRROR_REGISTRY="${REG_REGISTRY:-ghcr.io}" MIRROR_REGISTRY_AUTH=token MIRROR_REGISTRY_SCHEME=https \ REGISTRY_TOKEN="${REG_TOKEN-secret}" REGISTRY_TOKEN_CACHE="${3:-}" \ in_lib registry_manifest it/repo latest "$fake/out.json" > /dev/null 2> "$fake/err" || rc=$? printf '%s %s %s\n' "$rc" "$(awk 'END { print NR + 0 }' "$fake/calls.log")" "$(awk -F '\t' '$1 ~ /\/token$/ { n++ } END { print n + 0 }' "$fake/calls.log")" @@ -431,4 +441,154 @@ check "registry: a cached token the registry refuses is asked for again, once" " "lib.sh registry_manifest, REGISTRY_TOKEN_CACHE" check "registry: the cache then holds the new token" tok2 "$(cat "$fake"/cache/* 2> /dev/null)" "ls $fake/cache" +# On a token registry other than GHCR there is no dummy token: an anonymous +# read asks the token endpoint with no credential. It must never send the +# token an authenticated read cached, or a private package reads as public. +mkdir -m 0700 "$fake/cache-other" +echo tok1 > "$fake/next-token" +answers='Bearer tok1\t200\tmanifest.json\nBearer anon\t401\tdenied.json\n' +check "registry, another registry: an authenticated read caches its token and reads the private package" "0 2 1" \ + "$(REG_REGISTRY=registry.example reg auth "$answers" "$fake/cache-other")" "lib.sh registry_manifest, REGISTRY_TOKEN_CACHE" +check "registry, another registry: the anonymous read after it asks for a token of its own, and is refused" "1 2 1" \ + "$(REG_REGISTRY=registry.example REG_TOKEN='' reg anon "$answers" "$fake/cache-other")" \ + "lib.sh registry_token_file must give no cache file when REGISTRY_TOKEN is empty; cat $fake/calls.log" +check "registry, another registry: the anonymous read sent no cached token" "Bearer anon" \ + "$(awk -F '\t' '$1 ~ /manifests/ { print $2 }' "$fake/calls.log")" "cat $fake/calls.log" +check "registry, another registry: the cache holds only the authenticated token" tok1 "$(cat "$fake"/cache-other/*)" "ls $fake/cache-other" + +#---------------------------------------------------------------------------# +# Servers that failed: asked last, the rest in their own order +#---------------------------------------------------------------------------# +down="$work/down" +order_of() { + in_lib down_last "$down" https://a.example/r https://b.example/r/x https://c.example:8443/r https://c.example/r | tr '\n' ' ' | sed 's/ $//' +} +check "down: with no host marked, the order is kept" \ + "https://a.example/r https://b.example/r/x https://c.example:8443/r https://c.example/r" "$(order_of)" "lib.sh down_last" +in_lib down_mark "$down" https://b.example/other/path +in_lib down_mark "$down" https://c.example:8443/z +check "down: a host that failed comes last, by its host and port, and the rest keep their order" \ + "https://a.example/r https://c.example/r https://b.example/r/x https://c.example:8443/r" "$(order_of)" "lib.sh down_mark, down_last" + +#---------------------------------------------------------------------------# +# "Re-run all jobs" plans again, and a download by name or pattern still +# returns the earlier attempt's artifacts. The report reads only the summaries +# of its own plan, for targets its plan marks pending; collect reads only the +# verdicts and the promotion of the staged digest. +#---------------------------------------------------------------------------# +rr="$work/rerun" +mkdir -p "$rr/plan" "$rr/sum/x86_64-core" "$rr/sum/x86_64-extra" "$rr/sum/x86_64-multilib" +printf 'plan-2\n' > "$rr/plan/plan-id" +{ + printf 'target\tstate\tfiles\tMiB\twhy\n' + printf 'x86_64-core\tpending\t1\t1\t1 file(s) to mirror\n' + printf 'x86_64-extra\tcurrent\t0\t0\tthe promoted extra.db is upstream'\''s\n' + printf 'x86_64-multilib\tpending\t1\t1\t1 file(s) to mirror\n' +} > "$rr/plan/summary.tsv" +printf 'x86_64-core\tpromoted\tsha256:new 2026.10.14 latest\n' > "$rr/sum/x86_64-core/state.tsv" +printf 'plan-2\n' > "$rr/sum/x86_64-core/plan-id" +# Left by attempt 1: extra was replaced then, and the issue was resolved since. +printf 'x86_64-extra\treplaced\t1 file(s) have a tag holding other bytes\n' > "$rr/sum/x86_64-extra/state.tsv" +printf 'plan-1\n' > "$rr/sum/x86_64-extra/plan-id" +# Left by attempt 1 too: this attempt's chain of multilib left no summary. +printf 'x86_64-multilib\tpromoted\tsha256:old latest\n' > "$rr/sum/x86_64-multilib/state.tsv" +printf 'plan-1\n' > "$rr/sum/x86_64-multilib/plan-id" +rc=0 +env -u GITHUB_STEP_SUMMARY TRACK_ISSUES=0 bash "$REPO_ROOT/scripts/report" "$rr/plan" "$rr/sum" > "$rr/table.md" 2> "$rr/report.err" || rc=$? +check "re-run: a summary of an earlier plan is not read for a target this plan finds current" "| x86_64-extra | current |" \ + "$(grep -o '^| x86_64-extra | [a-z]* |' "$rr/table.md")" "scripts/report PLAN SUMMARIES; cat $rr/table.md $rr/report.err" +check "re-run: a pending target whose chain left only an earlier plan's summary failed" "| x86_64-multilib | failed |" \ + "$(grep -o '^| x86_64-multilib | [a-z]* |' "$rr/table.md")" "scripts/report PLAN SUMMARIES; cat $rr/table.md" +check "re-run: the summary of this plan is read" "| x86_64-core | promoted |" \ + "$(grep -o '^| x86_64-core | [a-z]* |' "$rr/table.md")" "scripts/report PLAN SUMMARIES; cat $rr/table.md" +check "re-run: the report fails the run for the target with no summary of this plan, and only for it" "1 0" \ + "$rc $(awk '/replaced/ { n++ } END { print n + 0 }' "$rr/table.md")" "scripts/report PLAN SUMMARIES; cat $rr/table.md" +# A summary of this very plan counts only for a target the plan marks pending. +printf 'plan-2\n' > "$rr/sum/x86_64-extra/plan-id" +rc=0 +env -u GITHUB_STEP_SUMMARY TRACK_ISSUES=0 bash "$REPO_ROOT/scripts/report" "$rr/plan" "$rr/sum" > "$rr/table.md" 2> "$rr/report.err" || rc=$? +check "re-run: a chain's word applies only to a target the plan marks pending" "| x86_64-extra | current |" \ + "$(grep -o '^| x86_64-extra | [a-z]* |' "$rr/table.md")" "scripts/report PLAN SUMMARIES; cat $rr/table.md" + +# collect: this attempt staged sha256:new in one batch, and it passed. Attempt +# 1 left a failed verdict of a batch 5, its refusal, and a promotion, all of +# sha256:old. +mkdir -p "$rr/stage/x86_64-core/batches" "$rr/verify/verify-x86_64-core-0" "$rr/verify/verify-x86_64-core-5" +printf 'staged\n' > "$rr/stage/x86_64-core/state" +printf 'it/x86_64/core@sha256:new\n' > "$rr/stage/x86_64-core/why" +printf 'DIGEST=sha256:new\n' > "$rr/stage/x86_64-core/staged.env" +printf 'a\n' > "$rr/stage/x86_64-core/batches/0.txt" +printf 'plan-2\n' > "$rr/stage/x86_64-core/plan-id" +printf 'x86_64-core\t0\tpassed\t1 package(s)\tsha256:new\n' > "$rr/verify/verify-x86_64-core-0/verify-x86_64-core-0.tsv" +printf 'x86_64-core\t5\tfailed\tbatch 5 failed\tsha256:old\n' > "$rr/verify/verify-x86_64-core-5/verify-x86_64-core-5.tsv" +printf 'stale-1-1-any.pkg.tar.zst\tsig-missing: x\n' > "$rr/verify/verify-x86_64-core-5/refused-x86_64-core-5.tsv" +printf 'x86_64-core\tpromoted\tsha256:old 2026.10.13 latest\n' > "$rr/promote-old.tsv" +rc=0 +bash "$REPO_ROOT/scripts/report" collect x86_64-core "$rr/stage" "$rr/verify" "$rr/promote-old.tsv" "$rr/out/x86_64-core" 2> "$rr/collect.err" || rc=$? +check "re-run: collect reads no verdict, refusal or promotion of another digest" "0 staged 0 plan-2" \ + "$rc $(cut -f 2 "$rr/out/x86_64-core/state.tsv") $(awk 'END { print NR + 0 }' "$rr/out/x86_64-core/refused.tsv") $(cat "$rr/out/x86_64-core/plan-id")" \ + "scripts/report collect x86_64-core STAGE VERIFY PROMOTED OUT; cat $rr/collect.err" +printf 'x86_64-core\tpromoted\tsha256:new 2026.10.14 latest\n' > "$rr/promote-new.tsv" +rc=0 +bash "$REPO_ROOT/scripts/report" collect x86_64-core "$rr/stage" "$rr/verify" "$rr/promote-new.tsv" "$rr/out2/x86_64-core" 2> "$rr/collect.err" || rc=$? +check "re-run: collect takes the promotion of the staged digest" "0 promoted" \ + "$rc $(cut -f 2 "$rr/out2/x86_64-core/state.tsv")" "scripts/report collect x86_64-core STAGE VERIFY PROMOTED OUT; cat $rr/collect.err" + +#---------------------------------------------------------------------------# +# fetch_upstream with MAX_SECONDS ends within it, its retries included: a +# server that sends 16 KiB/s, above the 4096 B/s stall floor, is asked once. +# curl gives each retry a new --max-time; without --retry-max-time the call +# took 3 x MAX + 4 s. +#---------------------------------------------------------------------------# +if need_tool python3 "fetch_upstream: a server that trickles is given up within MAX_SECONDS, with one request"; then + cat > "$work/trickle.py" << 'EOF' +import socket, sys, threading, time +s = socket.socket() +s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +s.bind(("127.0.0.1", 0)) +s.listen(16) +print(s.getsockname()[1], flush=True) +def serve(c): + try: + c.recv(65536) + with open(sys.argv[1], "a") as f: + f.write("hit\n") + c.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 104857600\r\n\r\n") + while True: + c.sendall(b"\0" * 8192) + time.sleep(0.5) + except OSError: + pass + finally: + c.close() +while True: + conn, _ = s.accept() + threading.Thread(target=serve, args=(conn,), daemon=True).start() +EOF + : > "$work/trickle.hits" + python3 -I "$work/trickle.py" "$work/trickle.hits" > "$work/trickle.port" 2> "$work/trickle.err" & + trickle_pid=$! + trap 'kill "$trickle_pid"; rm -rf "$work"' EXIT + port="" + for _ in $(seq 1 50); do + port="$(sed -n 1p "$work/trickle.port")" + [ -z "$port" ] || break + sleep 0.1 + done + started=$SECONDS + rc=0 + UPSTREAM_PROTO='=http,https' in_lib fetch_upstream "http://127.0.0.1:$port/big.db" "$work/trickle.out" 3 > /dev/null 2>&1 || rc=$? + took=$((SECONDS - started)) + kill "$trickle_pid" + trap 'rm -rf "$work"' EXIT + # The server ends by the signal (status 143), as it should. + wait_rc=0 + wait "$trickle_pid" || wait_rc=$? + [ "$wait_rc" -ne 0 ] || diag "the trickle server ended with status 0 before it was stopped: $(cat "$work/trickle.err")" + check "fetch_upstream: a server that trickles is given up within MAX_SECONDS, with one request" \ + "status 1, 1 request(s), within 8 s" \ + "status $rc, $(awk 'END { print NR + 0 }' "$work/trickle.hits") request(s), $([ "$took" -lt 8 ] && echo 'within 8 s' || echo "after $took s")" \ + "bash -c '. scripts/lib.sh; UPSTREAM_PROTO==http fetch_upstream http://127.0.0.1:$port/big.db /tmp/out 3' against $work/trickle.py ($(cat "$work/trickle.err"))" +fi + summary From 3a9caa5946568b1903846cf018496f2d5caa3f73 Mon Sep 17 00:00:00 2001 From: Azathothas Date: Wed, 14 Oct 2026 23:05:22 +0545 Subject: [PATCH 22/22] docs: seen-to-fail pairs of the pipeline fixes --- docs/tests.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/tests.md b/docs/tests.md index 965ac4c..9127131 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -213,6 +213,23 @@ The tree itself is never changed. An integration fault runs with its own registr | integration/10-pipeline | `keyring_repo` emptied after `keyring_from`'s `latest` is read | not ok: `verify installs the keyring package from keyring_from's repository` | | integration/10-pipeline | plan's `%PGPSIG%` condition reverted | not ok: a file signed again with the same bytes is planned, its layer repaired, promoted | | integration/10-pipeline | the signature comparison turned off | not ok: `a signature layer that is not the %PGPSIG% fails verify, and is named`, `a batch with a bad signature layer is never promoted` | +| unit/40-lib | `fetch_upstream` with `--max-time` and no `--retry-max-time` | not ok: a server at 16 KiB/s gets 3 requests in about 13 s, over a deadline of 3 s | +| integration/10-pipeline | no share of `PLAN_BUDGET` per target | not ok: assertions 62 to 64 (a stalled first db_server), and the run takes 944 s | +| integration/10-pipeline | a failed db_server ends the db_server loop | not ok: assertion 61 | +| integration/10-pipeline | `fetch_sig` reads the shard's rotated order | not ok: assertion 108 (the `.sig` of a plain mirror becomes the layer) | +| unit/40-lib | `down_last` ignores the down marks | not ok: assertion 95 | +| unit/40-lib | `down_last` drops the hosts with no mark | not ok: assertion 94 | +| integration/10-pipeline | no host is marked down | not ok: assertion 110 (a server that answers 503 gets more than 3 requests) | +| integration/10-pipeline | the first failed server ends the search | not ok: assertion 109 | +| integration/10-pipeline | the unsigned list is not written to the staged manifest | not ok: assertions 42, 44, 45, 46 | +| integration/10-pipeline | plan ignores the unsigned list of `latest` | not ok: assertions 44, 45, 46 (a signature published later never reaches the mirror) | +| unit/40-lib | an anonymous read uses the cached token | not ok: assertions 91, 92 | +| unit/40-lib | the report reads every summary, of any plan | not ok: assertions 96, 97, 99, 100 | +| unit/40-lib | `collect` reads verdicts of another digest | not ok: assertion 101 | +| integration/10-pipeline | no plan id is written | not ok: assertions 76, 37, 38 (an earlier attempt's result is read) | +| integration/10-pipeline | plan refuses a files database that leaves a package out | not ok: assertions 47 to 50 | +| integration/10-pipeline | verify runs `-Fl` over every name of the batch | not ok: assertion 50 | +| integration/10-pipeline | plan accepts a files database with an extra package | not ok: assertion 51 | | integration/10-pipeline | every `sig-refused` package excused | not ok: with no `%PGPSIG%`, a layer that is not upstream's `.sig` is not excused | | integration/10-pipeline | `build-repo` signs on every build | not ok: an update is planned as new files only; `latest` does not move backwards | | static/60-targets | five edits in a copy: a `docker.io` keyring image, a removed `# tag: aarch64`, another verify digest, renamed plan slots, a `keyring_from` that does not exist | one not ok for each edit, among them `every target verifies in the one image, on linux/amd64` |