diff --git a/render.txt b/render.txt index 7368f36..b20b33c 100644 --- a/render.txt +++ b/render.txt @@ -23,22 +23,18 @@ echo "due: RECONCILE=$r RECONCILE_HOURS=$h, last reconcile $ago: $(test -f /work task: [list] rm -f /work/.run/publish.txt /work/.run/labeled.txt /work/.run/orphans.txt /work/.run/unlanded.txt /work/.run/fresh.txt /work/.run/index-same.txt task: [list] mkdir -p /work/.run /work/staging task: [retry] for i in 1 2 3 4 5; do - rc=0; ( TZ=UTC rsync --timeout=300 --contimeout=60 --no-h -rL --list-only rsync://ftp.gnu.org/alpha/ > /work/.run/listing.txt ) || rc=$? # subshell: an exit in CMD cannot stop the loop. ||: the errexit of Task cannot stop it. + rc=0; ( TZ=UTC rsync --timeout=300 --contimeout=60 --no-h -8 -rL --list-only rsync://ftp.gnu.org/alpha/ > /work/.run/listing.txt ) || rc=$? # subshell: an exit in CMD cannot stop the loop. ||: the errexit of Task cannot stop it. case $rc in 0|23|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac test $i -lt 5 || break s=$((15 * 2 ** i + RANDOM % (2 * 15 + 1))); echo "rsync exit $rc, retry $i in ${s}s" >&2; sleep $s done; exit $rc -task: [normalise] awk ' - /^-/ && match($0, / [0-9][0-9][0-9][0-9]\/[0-9][0-9]\/[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9] /) { - path = substr($0, RSTART + RLENGTH); mtime = substr($0, RSTART + 1, RLENGTH - 2) - size = substr($0, 11, RSTART - 11); gsub(/[ ,]/, "", size) - print path "\t" size + 0 "\t" mtime }' /work/.run/listing.txt \ -| LC_ALL=C sort > /work/.run/upstream.txt -task: [normalise] awk ' - /^d/ && match($0, / [0-9][0-9][0-9][0-9]\/[0-9][0-9]\/[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9] /) { - path = substr($0, RSTART + RLENGTH); if (path != ".") print path "/" }' /work/.run/listing.txt \ -| LC_ALL=C sort > /work/.run/upstream-dirs.txt +task: [normalise] awk '/^-/ && match($0, / [0-9][0-9][0-9][0-9]\/[0-9][0-9]\/[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9] /) { + path = substr($0, RSTART + RLENGTH); mtime = substr($0, RSTART + 1, RLENGTH - 2) + size = substr($0, 11, RSTART - 11); gsub(/[ ,]/, "", size) + print path "\t" size + 0 "\t" mtime }' /work/.run/listing.txt | LC_ALL=C sort > /work/.run/upstream.txt +task: [normalise] awk '/^d/ && match($0, / [0-9][0-9][0-9][0-9]\/[0-9][0-9]\/[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9] /) { + path = substr($0, RSTART + RLENGTH); if (path != ".") print path "/" }' /work/.run/listing.txt | LC_ALL=C sort > /work/.run/upstream-dirs.txt task: [normalise] test "$(wc -l < /work/.run/upstream.txt)" -gt 6500 task: [state] mkdir -p /work/.run task: [pull] rc=0; aws s3 ls s3://gnu-alpha/.state/applied.txt.xz > /dev/null || rc=$? @@ -55,9 +51,10 @@ task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout task: [rebuild] mv /work/.run/applied.new /work/.run/applied.txt task: [rebuild] rm -f /work/.run/rebuild-now task: [diff] LC_ALL=C comm -13 /work/.run/applied.txt /work/.run/upstream.txt > /work/.run/changed.txt -task: [diff] awk -F'\t' -v TL="" 'NR == FNR { if ($1 ~ "^" TL "/(tlpkg/texlive\\.tlpdb|[^/]+)\\.sha512$") want[substr($1, 1, length($1) - 7)]; next } $1 in want' /work/.run/changed.txt /work/.run/upstream.txt | LC_ALL=C sort -u -o /work/.run/changed.txt - /work/.run/changed.txt +task: [diff] awk -F'\t' -v TL="" 'FILENAME == ARGV[1] { if ($1 ~ "^" TL "/(tlpkg/texlive\\.tlpdb|[^/]+)\\.sha512$") want[substr($1, 1, length($1) - 7)]; next } $1 in want' /work/.run/changed.txt /work/.run/upstream.txt | LC_ALL=C sort -u -o /work/.run/changed.txt - /work/.run/changed.txt task: [diff] cut -f1 /work/.run/upstream.txt > /work/.run/paths.txt task: [diff] cut -f1 /work/.run/applied.txt | LC_ALL=C comm -23 - /work/.run/paths.txt > /work/.run/deleted.txt +task: [split] b=4; case "$b" in ''|0?*|*[!0-9]*) echo "split: BATCH_GB is a whole number, not $b" >&2; exit 1 ;; esac task: [split] awk -F'\t' -v gb=160 '{ s += $2 } END { printf "upstream: %d objects, %.2f GB, ceiling %d GB\n", NR, s / 1e9, gb; exit (s > gb * 1e9) }' \ /work/.run/upstream.txt @@ -67,16 +64,14 @@ task: [split] awk -F'\t' -v kb="$(df -Pk /work/staging | awk 'NR == 2 { print $4 '$2 / 1024 > kb - 1048576 { print "no room for " $1 " (" $2 " bytes)"; bad = 1 } END { exit bad }' \ /work/.run/changed.txt task: [split] rm -f /work/.run/batch-*.txt -task: [split] awk -F'\t' -v OFS='\t' -v CAP=$((4 * 1000000000)) -v DIR=/work/.run -v TL="" ' - index($1, TL "/tlpkg/") == 1 || $1 !~ /\// { dec[++nd] = $0; next } - $2 > CAP { f = name(++n); print > f; close(f); next } - { if (sz + $2 > CAP && index($1, prev) != 1) flush(); buf[++nb] = $0; sz += $2; prev = $1 } - END { flush(); if (nd) { f = name(++n); for (i = 1; i <= nd; i++) print dec[i] > f; close(f) } - printf "%d lines in %d batches\n", NR, n } - function name(i) { return sprintf("%s/batch-%04d.txt", DIR, i) } - function flush( i, f) { if (!nb) return; f = name(++n) - for (i = 1; i <= nb; i++) print buf[i] > f; close(f); nb = sz = 0; split("", buf) }' \ -/work/.run/changed.txt +task: [split] awk -F'\t' -v OFS='\t' -v CAP=$((4 * 1000000000)) -v DIR=/work/.run -v TL="" 'index($1, TL "/tlpkg/") == 1 || $1 !~ /\// { dec[++nd] = $0; next } +$2 > CAP { f = name(++n); print > f; close(f); next } +{ if (sz + $2 > CAP && index($1, prev) != 1) flush(); buf[++nb] = $0; sz += $2; prev = $1 } +END { flush(); if (nd) { f = name(++n); for (i = 1; i <= nd; i++) print dec[i] > f; close(f) } + printf "%d lines in %d batches\n", NR, n } +function name(i) { return sprintf("%s/batch-%04d.txt", DIR, i) } +function flush( i, f) { if (!nb) return; f = name(++n) + for (i = 1; i <= nb; i++) print buf[i] > f; close(f); nb = sz = 0; split("", buf) }' /work/.run/changed.txt task: [prepare] mkdir -p /work/.run/tl task: [prepare] printf '%s\n' tlpkg/texlive.tlpdb.xz tlpkg/texlive.tlpdb.sha512 tlpkg/texlive.tlpdb.sha512.asc tlpkg/gpg/pubring.gpg > /work/.run/tl/files.txt task: [retry] for i in 1 2 3 4 5; do @@ -98,21 +93,19 @@ task: [retry] for i in 1 2 3 4 5; do done; exit $rc task: [verify] cd /work/staging && cut -f1 | grep -E '^/(tlpkg/texlive\.tlpdb|[^/]+)\.sha512$' | while IFS= read -r f; do (cd "$(dirname "$f")" && f=$(basename "$f") && shasum -a 512 -c "$f" && gpgv --status-fd 1 --keyring /work/.run/tl/tlpkg/gpg/pubring.gpg "$f.asc" "$f" | awk -v k= '/^\[GNUPG:\] GOODSIG /{g=1} /^\[GNUPG:\] VALIDSIG / && $NF == k {v=1} END{exit !(g&&v)}') || exit 1; done -task: [verify] cd /work/staging && { test "$(grep -c '^/archive/' )" = 0 || { awk -F'\t' -v TL="" 'NR == FNR { if (index($1, TL "/archive/") == 1) want[$1] = 1; next } - /^name /{split($0, a, " "); n = a[2]} /^revision /{split($0, a, " "); r = a[2]} - /^containerchecksum /{split($0, a, " "); emit(a[2], n)} - /^doccontainerchecksum /{split($0, a, " "); emit(a[2], n ".doc")} - /^srccontainerchecksum /{split($0, a, " "); emit(a[2], n ".source")} - function emit(sum, base, p, q) { p = TL "/archive/" base ".tar.xz"; q = TL "/archive/" base ".r" r ".tar.xz" - if (p in want) { print sum " " p; delete want[p] } - if (q in want) { print sum " " q; delete want[q] } } - END { for (p in want) { print "no checksum in the tlpdb for " p > "/dev/stderr"; bad = 1 } exit bad }' \ - /work/.run/tl/tlpkg/texlive.tlpdb > /work/.run/tl/sums.txt && shasum -a 512 -c --quiet /work/.run/tl/sums.txt; }; } +task: [verify] cd /work/staging && { test "$(grep -c '^/archive/' )" = 0 || { awk -F'\t' -v TL="" 'FILENAME == ARGV[1] { if (index($1, TL "/archive/") == 1) want[$1] = 1; next } +/^name /{split($0, a, " "); n = a[2]} /^revision /{split($0, a, " "); r = a[2]} +/^containerchecksum /{split($0, a, " "); emit(a[2], n)} +/^doccontainerchecksum /{split($0, a, " "); emit(a[2], n ".doc")} +/^srccontainerchecksum /{split($0, a, " "); emit(a[2], n ".source")} +function emit(sum, base, p, q) { p = TL "/archive/" base ".tar.xz"; q = TL "/archive/" base ".r" r ".tar.xz" + if (p in want) { print sum " " p; delete want[p] } + if (q in want) { print sum " " q; delete want[q] } } +END { for (p in want) { print "no checksum in the tlpdb for " p > "/dev/stderr"; bad = 1 } exit bad }' /work/.run/tl/tlpkg/texlive.tlpdb > /work/.run/tl/sums.txt && shasum -a 512 -c --quiet /work/.run/tl/sums.txt; }; } task: [verify] cd /work/staging && { test ! -f /tlpkg/texlive.tlpdb || { cmp /tlpkg/texlive.tlpdb /work/.run/tl/tlpkg/texlive.tlpdb && xz -dc /tlpkg/texlive.tlpdb.xz | cmp - /tlpkg/texlive.tlpdb && { cut -f1 /work/.run/applied.txt; cut -f1 ; } | LC_ALL=C sort -u | LC_ALL=C comm -23 - /work/.run/deleted.txt > /work/.run/tl/after.txt && awk -v TL="" '/^name /{n=$2} - /^containerchecksum /{print TL "/archive/" n ".tar.xz"} - /^doccontainerchecksum /{print TL "/archive/" n ".doc.tar.xz"} - /^srccontainerchecksum /{print TL "/archive/" n ".source.tar.xz"}' /tlpkg/texlive.tlpdb \ -| LC_ALL=C sort | LC_ALL=C comm -23 - /work/.run/tl/after.txt > /work/.run/tl/missing.txt && test ! -s /work/.run/tl/missing.txt; }; } +/^containerchecksum /{print TL "/archive/" n ".tar.xz"} +/^doccontainerchecksum /{print TL "/archive/" n ".doc.tar.xz"} +/^srccontainerchecksum /{print TL "/archive/" n ".source.tar.xz"}' /tlpkg/texlive.tlpdb | LC_ALL=C sort | LC_ALL=C comm -23 - /work/.run/tl/after.txt > /work/.run/tl/missing.txt && test ! -s /work/.run/tl/missing.txt; }; } task: [label] python3 -I - /work/staging $'application/x-gzip .gz .tgz .taz .tz .svgz\napplication/x-bzip2 .bz2 .tbz2 .tbz\napplication/x-xz .xz .txz\napplication/x-compress .Z\napplication/zstd .zst .tzst\napplication/x-lzip .lz\napplication/x-lzma .lzma' > /work/.run/labels.raw <<'PY' import codecs, mimetypes, os, sys @@ -172,10 +165,10 @@ PY task: [label] LC_ALL=C sort /work/.run/labels.raw > /work/.run/labels.txt && rm /work/.run/labels.raw task: [publish] cat /work/.run/labels.txt >> /work/.run/labeled.txt -task: [label-trees] python3 -I - /work/staging /work/.run/up /work/.run/labels.txt up <<'PY' +task: [label-trees] python3 -I - /work/staging /work/.run/up /work/.run/labels.txt up "$FRESH_KEY" <<'PY' import os, shutil, sys -staging, up, labels, to = (os.fsencode(a) for a in sys.argv[1:5]) +staging, up, labels, to, fresh = (os.fsencode(a) for a in sys.argv[1:6]) def move(a, b): os.makedirs(os.path.dirname(b), exist_ok=True) @@ -187,7 +180,7 @@ if to == b"up": with open(labels, "rb") as f: for line in f: path, t = line.rstrip(b"\n").split(b"\t") - if path != b"mirror-updated-timestamp.txt": + if path != fresh: n = trees.setdefault(t, str(len(trees) + 1).encode()) move(os.path.join(staging, path), os.path.join(up, n, path)) for t, n in trees.items(): @@ -204,10 +197,10 @@ else: PY task: [publish] for d in /work/.run/up/*/; do test -d "$d" || continue; aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type "$(cat "${d%/}.type")" "$d" s3://gnu-alpha/ | tee -a /work/.run/publish.txt; done -task: [label-trees] python3 -I - /work/staging /work/.run/up /work/.run/labels.txt staging <<'PY' +task: [label-trees] python3 -I - /work/staging /work/.run/up /work/.run/labels.txt staging "$FRESH_KEY" <<'PY' import os, shutil, sys -staging, up, labels, to = (os.fsencode(a) for a in sys.argv[1:5]) +staging, up, labels, to, fresh = (os.fsencode(a) for a in sys.argv[1:6]) def move(a, b): os.makedirs(os.path.dirname(b), exist_ok=True) @@ -219,7 +212,7 @@ if to == b"up": with open(labels, "rb") as f: for line in f: path, t = line.rstrip(b"\n").split(b"\t") - if path != b"mirror-updated-timestamp.txt": + if path != fresh: n = trees.setdefault(t, str(len(trees) + 1).encode()) move(os.path.join(staging, path), os.path.join(up, n, path)) for t, n in trees.items(): @@ -235,9 +228,9 @@ else: shutil.rmtree(up, ignore_errors=True) PY -task: [publish] test ! -f /work/staging/mirror-updated-timestamp.txt || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --content-type "$(awk -F'\t' '$1 == "mirror-updated-timestamp.txt" { print $2 }' /work/.run/labels.txt)" /work/staging/mirror-updated-timestamp.txt s3://gnu-alpha/mirror-updated-timestamp.txt | tee -a /work/.run/publish.txt +task: [publish] test ! -f /work/staging/"$FRESH_KEY" || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --content-type "$(awk -F'\t' '$1 == ENVIRON["FRESH_KEY"] { print $2 }' /work/.run/labels.txt)" /work/staging/"$FRESH_KEY" s3://gnu-alpha/"$FRESH_KEY" | tee -a /work/.run/publish.txt task: [merge] (cd /work/staging && find . -type f | sed 's|^\./||' | LC_ALL=C sort) > /work/.run/landed.txt -task: [merge] awk -F'\t' 'NR == FNR { drop[$1] = 1; next } !($1 in drop)' /work/.run/applied.txt | cat - <(LC_ALL=C join -t "$(printf '\t')" /work/.run/landed.txt ) | LC_ALL=C sort > /work/.run/applied.new +task: [merge] awk -F'\t' 'FILENAME == ARGV[1] { drop[$1] = 1; next } !($1 in drop)' /work/.run/applied.txt | cat - <(LC_ALL=C join -t "$(printf '\t')" /work/.run/landed.txt ) | LC_ALL=C sort > /work/.run/applied.new task: [push] xz -T0 -c /work/.run/applied.new > /work/.run/applied.new.xz task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/.run/applied.new.xz s3://gnu-alpha/.state/applied.txt.xz task: [checkpoint] mv /work/.run/applied.new /work/.run/applied.txt @@ -245,10 +238,9 @@ task: [checkpoint] find /work/staging -mindepth 1 -delete task: [batches] test "$(ls /work/.run/batch-*.txt 2>/dev/null | wc -l)" -le 4 || touch /work/.run/chain task: [remove] rm -f /work/.run/del-*; split -l 1000 -a 4 /work/.run/deleted.txt /work/.run/del- task: [remove] for f in /work/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"") - printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" \ -> "$f.json" && aws s3api delete-objects --bucket gnu-alpha --delete "file://$f.json" | tee -a /work/.run/publish.txt | { test "$(grep -c Errors)" = 0; }; done + printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" > "$f.json" && aws s3api delete-objects --bucket gnu-alpha --delete "file://$f.json" | tee -a /work/.run/publish.txt | { test "$(grep -c Errors)" = 0; }; done task: [merge] (cd /work/staging && find . -type f | sed 's|^\./||' | LC_ALL=C sort) > /work/.run/landed.txt -task: [merge] awk -F'\t' 'NR == FNR { drop[$1] = 1; next } !($1 in drop)' /work/.run/deleted.txt /work/.run/applied.txt | cat - <(LC_ALL=C join -t "$(printf '\t')" /work/.run/landed.txt /work/.run/deleted.txt) | LC_ALL=C sort > /work/.run/applied.new +task: [merge] awk -F'\t' 'FILENAME == ARGV[1] { drop[$1] = 1; next } !($1 in drop)' /work/.run/deleted.txt /work/.run/applied.txt | cat - <(LC_ALL=C join -t "$(printf '\t')" /work/.run/landed.txt /work/.run/deleted.txt) | LC_ALL=C sort > /work/.run/applied.new task: [push] xz -T0 -c /work/.run/applied.new > /work/.run/applied.new.xz task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/.run/applied.new.xz s3://gnu-alpha/.state/applied.txt.xz task: [checkpoint] mv /work/.run/applied.new /work/.run/applied.txt @@ -265,10 +257,9 @@ task: [reconcile] cat /work/.run/applied.txt /work/.run/upstream-dirs.txt | awk task: [reconcile] cut -f1 /work/.run/bucket.txt | grep -vE '^\.state/' | awk -v i="/katoptra.org.directory.index.html" '{ s = "/" $0 } substr(s, length(s) - length(i) + 1) != i' | LC_ALL=C comm -23 - /work/.run/index-keys.txt | LC_ALL=C comm -23 - /work/.run/paths.txt > /work/.run/orphans.txt task: [remove] rm -f /work/.run/del-*; split -l 1000 -a 4 /work/.run/orphans.txt /work/.run/del- task: [remove] for f in /work/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"") - printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" \ -> "$f.json" && aws s3api delete-objects --bucket gnu-alpha --delete "file://$f.json" | tee -a /work/.run/publish.txt | { test "$(grep -c Errors)" = 0; }; done + printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" > "$f.json" && aws s3api delete-objects --bucket gnu-alpha --delete "file://$f.json" | tee -a /work/.run/publish.txt | { test "$(grep -c Errors)" = 0; }; done task: [merge] (cd /work/staging && find . -type f | sed 's|^\./||' | LC_ALL=C sort) > /work/.run/landed.txt -task: [merge] awk -F'\t' 'NR == FNR { drop[$1] = 1; next } !($1 in drop)' /work/.run/orphans.txt /work/.run/applied.txt | cat - <(LC_ALL=C join -t "$(printf '\t')" /work/.run/landed.txt /work/.run/orphans.txt) | LC_ALL=C sort > /work/.run/applied.new +task: [merge] awk -F'\t' 'FILENAME == ARGV[1] { drop[$1] = 1; next } !($1 in drop)' /work/.run/orphans.txt /work/.run/applied.txt | cat - <(LC_ALL=C join -t "$(printf '\t')" /work/.run/landed.txt /work/.run/orphans.txt) | LC_ALL=C sort > /work/.run/applied.new task: [push] xz -T0 -c /work/.run/applied.new > /work/.run/applied.new.xz task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/.run/applied.new.xz s3://gnu-alpha/.state/applied.txt.xz task: [checkpoint] mv /work/.run/applied.new /work/.run/applied.txt @@ -283,106 +274,88 @@ case $rc in *) exit $rc ;; esac +task: [pages] awk 'BEGIN { g = ENVIRON["PAGE_FOOT"]; gsub(/%%/, "", g) + if (g !~ /^[^%]*(%s[^%]*)?$/) { print "pages: PAGE_FOOT can have one %s, and %%, not " ENVIRON["PAGE_FOOT"] > "/dev/stderr"; exit 1 } }' task: [pages] mkdir -p /work/staging /work/.run/slash && find /work/staging /work/.run/slash -mindepth 1 -delete && : > /work/.run/index-real.txt task: [pages] { cat /work/.run/applied.txt; awk '{ print $0 "\t\t" }' /work/.run/upstream-dirs.txt; } | LC_ALL=C sort > /work/.run/page-state.txt -task: [pages] printf '%s\n' 3 katoptra.org.directory.index.html '' '' https://alpha.gnu.org/gnu/ '' gnu-alpha.katoptra.org | sha256sum | awk '{ print "#pages " substr($1, 1, 16) }' > /work/.run/index-stamp.txt +task: [pages] printf '%s\n' 3 katoptra.org.directory.index.html "$INDEX_FILES" "$PAGE_FOOT" "$UPSTREAM" "$CONTACT" gnu-alpha.katoptra.org | sha256sum | awk '{ print "#pages " substr($1, 1, 16) }' > /work/.run/index-stamp.txt task: [pages] awk 'NR == 1 && /^#pages / { next } { print }' /work/.run/indexed.txt > /work/.run/index-old.txt task: [pages] { LC_ALL=C comm -3 /work/.run/index-old.txt /work/.run/page-state.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }'; head -1 /work/.run/indexed.txt | cmp -s - /work/.run/index-stamp.txt || cat /work/.run/page-state.txt; } | awk -F'\t' '!("" in s) { s[""] = 1; print "" } { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' | LC_ALL=C sort > /work/.run/index-dirty.txt task: [pages] awk -F'\t' '!("" in s) { s[""] = 1; print "" } { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/.run/page-state.txt | LC_ALL=C sort > /work/.run/index-dirs.txt task: [pages] cat /work/.run/index-stamp.txt /work/.run/page-state.txt > /work/.run/indexed.new -task: [pages] LC_ALL=C comm -23 /work/.run/index-dirty.txt /work/.run/index-dirs.txt | awk -v i="katoptra.org.directory.index.html" '{ print $0 i } $0 != "" { print substr($0, 1, length($0) - 1) }' | awk -F'\t' 'NR == FNR { f[$1] = 1; next } !($0 in f)' /work/.run/page-state.txt - > /work/.run/index-gone.txt +task: [pages] LC_ALL=C comm -23 /work/.run/index-dirty.txt /work/.run/index-dirs.txt | awk -v i="katoptra.org.directory.index.html" '{ print $0 i } $0 != "" { print substr($0, 1, length($0) - 1) }' | awk -F'\t' 'FILENAME == ARGV[1] { f[$1] = 1; next } !($0 in f)' /work/.run/page-state.txt - > /work/.run/index-gone.txt task: [pages] LC_ALL=C comm -12 /work/.run/index-dirty.txt /work/.run/index-dirs.txt > /work/.run/index-make.txt task: [pages] test ! -s /work/.run/index-make.txt || awk -v s="/work/staging/" '{ print s $0 }' /work/.run/index-make.txt | tr '\n' '\0' | xargs -0 mkdir -p task: [pages] test ! -s /work/.run/index-make.txt || awk -v r="/work/.run/slash/" '$0 != "" { n = split($0, c, "/"); d = "" for (i = 1; i < n - 1; i++) d = d c[i] "/"; print r (n - 1) "/" d }' /work/.run/index-make.txt \ | LC_ALL=C sort -u | tr '\n' '\0' | xargs -0r mkdir -p -task: [pages] test ! -s /work/.run/index-make.txt || awk -F'\t' -v OFS='\t' 'NR == FNR { want[$0] = 1; next } - { n = split($1, c, "/"); d = "" - for (i = 1; i < n; i++) { if ((d in want) && !((d c[i]) in seen)) { seen[d c[i]] = 1; print d, "d", c[i], "", "" }; d = d c[i] "/" } - if (d in want) print d, (c[n] == "" ? "e" : "f"), c[n], $2, $3 }' /work/.run/index-make.txt /work/.run/page-state.txt \ -| LC_ALL=C sort -t "$(printf '\t')" -k1,1 -k2,2 -k3,3V | awk -F'\t' -v S="/work/staging/" -v R="/work/.run/slash/" -v I="katoptra.org.directory.index.html" -v F='' \ - -v N='' -v X="/work/.run/index-real.txt" -v U=https://alpha.gnu.org/gnu/ \ - -v C='' -v H="gnu-alpha.katoptra.org" 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } +task: [pages] test ! -s /work/.run/index-make.txt || awk -F'\t' -v OFS='\t' 'FILENAME == ARGV[1] { want[$0] = 1; next } +{ n = split($1, c, "/"); d = "" + for (i = 1; i < n; i++) { if ((d in want) && !((d c[i]) in seen)) { seen[d c[i]] = 1; print d, "d", c[i], "", "" }; d = d c[i] "/" } + if (d in want) print d, (c[n] == "" ? "e" : "f"), c[n], $2, $3 }' /work/.run/index-make.txt /work/.run/page-state.txt | LC_ALL=C sort -t "$(printf '\t')" -k1,1 -k2,2 -k3,3V | awk -F'\t' -v S="/work/staging/" -v R="/work/.run/slash/" -v I="katoptra.org.directory.index.html" -v X="/work/.run/index-real.txt" -v H="gnu-alpha.katoptra.org" -v css=':root{color-scheme:light dark;--bg:#f6f5f2;--fg:#1c1c28;--mut:#6e6e7e;--line:#e2e0da;--hov:#edebe5;--a:#3b45d9}@media (prefers-color-scheme:dark){:root{--bg:#121218;--fg:#f0f0f5;--mut:#8b8b9c;--line:#262630;--hov:#1a1a22;--a:#8b93ff}}body{margin:0 auto;max-width:60rem;padding:2rem 1rem 3rem;background:var(--bg);color:var(--fg);font:15px/1.45 system-ui,-apple-system,sans-serif}h1{font-size:1.35rem;font-weight:650;margin:0;overflow-wrap:anywhere}p{color:var(--mut);margin:.2rem 0 1.25rem;font-size:.9rem}table{width:100%;border-collapse:collapse;font:13.5px/1.4 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}th{font:600 .7rem/1 system-ui,sans-serif;letter-spacing:.06em;text-transform:uppercase;color:var(--mut);text-align:left;padding:0 .5rem .6rem;white-space:nowrap}td{padding:.4rem .5rem;border-top:1px solid var(--line);vertical-align:baseline}td:first-child{overflow-wrap:anywhere;width:100%}td+td{white-space:nowrap;color:var(--mut);font-variant-numeric:tabular-nums}th:last-child,td:last-child{text-align:right}tr:hover td{background:var(--hov)}a{color:var(--a);text-decoration:none}a:hover{text-decoration:underline}.s a{color:var(--mut)}.k{font-size:.72rem;margin-left:.6rem;padding:.05rem .4rem;border:1px solid var(--line);border-radius:.3rem;color:var(--mut)}.k:hover{color:var(--a);border-color:var(--a);text-decoration:none}footer{margin-top:2rem;padding-top:1rem;border-top:1px solid var(--line);color:var(--mut);font-size:.85rem;overflow-wrap:anywhere}footer a{color:inherit;text-decoration:underline}@media (max-width:36rem){thead{display:none}tr{display:grid;grid-template-columns:1fr auto;border-top:1px solid var(--line)}td{border:0;padding:.1rem .5rem}td:first-child{grid-column:1/-1;padding-top:.45rem}td+td{padding-bottom:.45rem;font-size:.8rem}}' 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } - BEGIN { n = split(N, nm, " "); for (i = n; i >= 1; i--) rank[nm[i]] = i - G = "\\.(sig|sign|asc|gpg|dsig|proof|md5|md5sum|sha|sha1|sha224|sha256|sha256sum|sha512|sum|checksums)$" - if (C == "" && (j = index(H, ".")) > 1) C = substr(H, 1, j - 1) "@" substr(H, j + 1) - css = ":root{color-scheme:light dark;--bg:#f6f5f2;--fg:#1c1c28;--mut:#6e6e7e;--line:#e2e0da;--hov:#edebe5;--a:#3b45d9}" - css = css "@media (prefers-color-scheme:dark){:root{--bg:#121218;--fg:#f0f0f5;--mut:#8b8b9c;--line:#262630;--hov:#1a1a22;--a:#8b93ff}}" - css = css "body{margin:0 auto;max-width:60rem;padding:2rem 1rem 3rem;background:var(--bg);color:var(--fg);font:15px/1.45 system-ui,-apple-system,sans-serif}" - css = css "h1{font-size:1.35rem;font-weight:650;margin:0;overflow-wrap:anywhere}p{color:var(--mut);margin:.2rem 0 1.25rem;font-size:.9rem}" - css = css "table{width:100%;border-collapse:collapse;font:13.5px/1.4 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}" - css = css "th{font:600 .7rem/1 system-ui,sans-serif;letter-spacing:.06em;text-transform:uppercase;color:var(--mut);text-align:left;padding:0 .5rem .6rem;white-space:nowrap}" - css = css "td{padding:.4rem .5rem;border-top:1px solid var(--line);vertical-align:baseline}td:first-child{overflow-wrap:anywhere;width:100%}" - css = css "td+td{white-space:nowrap;color:var(--mut);font-variant-numeric:tabular-nums}th:last-child,td:last-child{text-align:right}" - css = css "tr:hover td{background:var(--hov)}a{color:var(--a);text-decoration:none}a:hover{text-decoration:underline}.s a{color:var(--mut)}" - css = css ".k{font-size:.72rem;margin-left:.6rem;padding:.05rem .4rem;border:1px solid var(--line);border-radius:.3rem;color:var(--mut)}" - css = css ".k:hover{color:var(--a);border-color:var(--a);text-decoration:none}" - css = css "footer{margin-top:2rem;padding-top:1rem;border-top:1px solid var(--line);color:var(--mut);font-size:.85rem;overflow-wrap:anywhere}" - css = css "footer a{color:inherit;text-decoration:underline}@media (max-width:36rem){thead{display:none}" - css = css "tr{display:grid;grid-template-columns:1fr auto;border-top:1px solid var(--line)}td{border:0;padding:.1rem .5rem}" - css = css "td:first-child{grid-column:1/-1;padding-top:.45rem}td+td{padding-bottom:.45rem;font-size:.8rem}}" } - function esc(s) { gsub(/&/, "\\&", s); gsub(/, "\\<", s); gsub(/>/, "\\>", s); gsub(/"/, "\\"", s); return s } - function grp(n, s, o) { s = sprintf("%.0f", n); o = "" - while (length(s) > 3) { o = "," substr(s, length(s) - 2) o; s = substr(s, 1, length(s) - 3) } return s o } - function iec(n, u, i) { if (n < 1024) return sprintf("%.0f B", n) - split("KiB MiB GiB TiB", u, " "); n /= 1024; i = 1; while (n >= 1024 && i < 4) { n /= 1024; i++ } - return sprintf(n < 10 ? "%.1f %s" : "%.0f %s", n, u[i]) } - function ymd(t) { gsub(/\//, "-", t); return substr(t, 1, 16) } - function foot( t) { if (F != "") return "" - t = (U == "" ? "" : "Mirror of " esc(U) enc(cur) ".") - if (C != "") t = t (t == "" ? "" : " ") "Report a problem to " esc(C) "." - return (t == "" ? "" : "") } - function open(d) { cur = d; live = 1; real = ""; nd = 0; nfl = 0; tot = 0; split("", has) } - function w(s) { if (real == "") printf("%s", s) > fa; if (cur != "") printf("%s", s) > fb } - function shut( i, r, x, c, e, k, b) { split("", fold); split("", links) - for (i = 1; i <= nfl; i++) { r = fname[i]; x = "" - while (match(tolower(r), G)) { r = substr(r, 1, RSTART - 1); if (r in has) x = r } - if (x != "") { fold[i] = 1 - links[x] = links[x] "" esc(substr(fname[i], length(x) + 1)) "" } } - c = (nd ? grp(nd) (nd == 1 ? " directory" : " directories") : "") - if (nfl) c = c (c == "" ? "" : ", ") grp(nfl) (nfl == 1 ? " file" : " files") ", " iec(tot) - if (c == "") c = "This directory is empty." - fa = S cur I; if (real != "") print cur real "\t" cur I > X - if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/"); fb = R k "/" b } - w("
\n" c "
\n") - w("| Name | Modified (UTC) | Size |
|---|---|---|
| ../ | ||
| " esc(dname[i]) "/ | ||
| " esc(fname[i]) "" links[fname[i]] " | " ymd(ftime[i]) " | " iec(fsize[i]) " |
" c "
\n") + w("| Name | Modified (UTC) | Size |
|---|---|---|
| ../ | ||
| " esc(dname[i]) "/ | ||
| " esc(fname[i]) "" links[fname[i]] " | " ymd(ftime[i]) " | " iec(fsize[i]) " |