From ba8f97d6a7acb6d01a9cec71d64cf5c52c0df1be Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Mon, 19 Oct 2026 17:57:10 -0700 Subject: [PATCH 1/5] feat(rsync): record upstream directories and list in UTC --- engines/rsync.yml | 15 +++++++++++++-- examples/rsync/Taskfile.yml | 4 +++- examples/rsync/render.txt | 6 +++++- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/engines/rsync.yml b/engines/rsync.yml index d5648fd..e379e93 100644 --- a/engines/rsync.yml +++ b/engines/rsync.yml @@ -196,12 +196,15 @@ tasks: # not upstream, rsync reports the symlink and ignores it. rsync then exits 23, but # the listing has all the other files. retry accepts exit 23. LIST_FLOOR, not the # exit code, finds a truncated listing. + # + # rsync shows each time in the time zone of the container. TZ=UTC makes each time + # UTC, and the directory pages show the times as UTC. - task: retry - vars: {CMD: '{{.RSYNC}}{{if .FILTER}} {{.FILTER}}{{end}} -rL --list-only {{.SOURCE}} > {{.RUN}}/listing.txt'} + vars: {CMD: 'TZ=UTC {{.RSYNC}}{{if .FILTER}} {{.FILTER}}{{end}} -rL --list-only {{.SOURCE}} > {{.RUN}}/listing.txt'} - {task: normalise} normalise: - desc: RUN/listing.txt (rsync --list-only) to RUN/upstream.txt; every regular file, as "path TAB size TAB mtime", byte-sorted + desc: RUN/listing.txt (rsync --list-only) to RUN/upstream.txt, every regular file as "path TAB size TAB mtime", and RUN/upstream-dirs.txt, every directory but the root as "path/"; both byte-sorted set: [pipefail] cmds: # The size column can contain commas (rsync 3.x), or it can be empty for an empty @@ -220,6 +223,14 @@ tasks: size = substr($0, 11, RSTART - 11); gsub(/[ ,]/, "", size) print path "\t" size + 0 "\t" mtime }' {{.RUN}}/listing.txt \ | LC_ALL=C sort > {{.RUN}}/upstream.txt + # The directories of upstream, each with a slash at the end. The root (".") is not + # in the file. pages makes a page for each directory, also for a directory with no + # file. + - >- + awk ' + /^d/ && match($0, / [0-9][0-9][0-9][0-9]\/[0-9][0-9]\/[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9] /) { + path = substr($0, RSTART + RLENGTH); if (path != ".") print path "/" }' {{.RUN}}/listing.txt \ + | LC_ALL=C sort > {{.RUN}}/upstream-dirs.txt # A truncated listing must not become a list of deletions. LIST_FLOOR is a line # count, approximately 90% of the usual size of the listing. A correct listing # always has more lines. When LIST_FLOOR is not set, there is no guard. diff --git a/examples/rsync/Taskfile.yml b/examples/rsync/Taskfile.yml index 0fe1393..880da9e 100644 --- a/examples/rsync/Taskfile.yml +++ b/examples/rsync/Taskfile.yml @@ -59,9 +59,11 @@ tasks: CTAN_FOOT: 'This directory on ctan.org. Sizes in bytes, times as CTAN lists them.' cmds: # list and normalise on a local tree: its two regular files, sorted with byte - # collation, and no directory. + # collation, and no directory. upstream-dirs.txt gets its two directories, each with + # a slash at the end, and not the root. - {task: list, vars: {SOURCE: '{{.ROOT_DIR}}/fixtures/run-root/staging/', RSYNC: 'rsync --no-h'}} - test "$(cut -f1 {{.RUN}}/upstream.txt | tr '\n' ' ')" = "a/b/f.txt timestamp " + - test "$(tr '\n' ' ' < {{.RUN}}/upstream-dirs.txt)" = "a/ a/b/ " # The verbs write their files adjacent to the files that they read. Thus, they # operate on a copy. - rm -rf {{.F}} && cp -R {{.ROOT_DIR}}/fixtures {{.F}} diff --git a/examples/rsync/render.txt b/examples/rsync/render.txt index ece27e4..1640b37 100644 --- a/examples/rsync/render.txt +++ b/examples/rsync/render.txt @@ -23,7 +23,7 @@ echo "due: RECONCILE=$r RECONCILE_HOURS=$h, last reconcile $ago: $(test -f /work task: [list] rm -f /work/examples/rsync/.run/publish.txt /work/examples/rsync/.run/labeled.txt /work/examples/rsync/.run/orphans.txt /work/examples/rsync/.run/unlanded.txt /work/examples/rsync/.run/fresh.txt task: [list] mkdir -p /work/examples/rsync/.run /work/examples/rsync/staging task: [retry] for i in 1 2 3 4 5; do - rc=0; ( rsync --timeout=300 --contimeout=60 --no-h -rL --list-only rsync://rsync.example.org/pub/ > /work/examples/rsync/.run/listing.txt ) || rc=$? # subshell: an exit in CMD cannot stop the loop. ||: the errexit of Task cannot stop it. + rc=0; ( TZ=UTC rsync --timeout=300 --contimeout=60 --no-h -rL --list-only rsync://rsync.example.org/pub/ > /work/examples/rsync/.run/listing.txt ) || rc=$? # subshell: an exit in CMD cannot stop the loop. ||: the errexit of Task cannot stop it. case $rc in 0|23|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac test $i -lt 5 || break s=$((15 * 2 ** i + RANDOM % (2 * 15 + 1))); echo "rsync exit $rc, retry $i in ${s}s" >&2; sleep $s @@ -35,6 +35,10 @@ task: [normalise] awk ' size = substr($0, 11, RSTART - 11); gsub(/[ ,]/, "", size) print path "\t" size + 0 "\t" mtime }' /work/examples/rsync/.run/listing.txt \ | LC_ALL=C sort > /work/examples/rsync/.run/upstream.txt +task: [normalise] awk ' + /^d/ && match($0, / [0-9][0-9][0-9][0-9]\/[0-9][0-9]\/[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9] /) { + path = substr($0, RSTART + RLENGTH); if (path != ".") print path "/" }' /work/examples/rsync/.run/listing.txt \ +| LC_ALL=C sort > /work/examples/rsync/.run/upstream-dirs.txt task: [normalise] test "$(wc -l < /work/examples/rsync/.run/upstream.txt)" -gt 1 task: [state] mkdir -p /work/examples/rsync/.run task: [pull] rc=0; aws s3 ls s3://example/.state/applied.txt.xz > /dev/null || rc=$? From dc36bab62aa553ced1aabf9985021d00ae1caec1 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Mon, 19 Oct 2026 18:01:57 -0700 Subject: [PATCH 2/5] feat(rsync): draw pages for empty directories, redraw on change --- engines/rsync.yml | 69 +++++++++++++------ examples/rsync/Taskfile.yml | 29 +++++++- .../fixtures/run-pages/upstream-dirs.txt | 6 ++ .../fixtures/run-pages/want/index-make.txt | 2 + .../rsync/fixtures/run-pages/want/indexed.new | 13 ++++ .../rsync/fixtures/run-pages/want/slash/1/e1 | 7 ++ .../fixtures/run-pages/want/slash/2/e1/e2 | 6 ++ .../ctan.katoptra.org.directory.index.html | 1 + .../e1/ctan.katoptra.org.directory.index.html | 7 ++ .../e2/ctan.katoptra.org.directory.index.html | 6 ++ examples/rsync/render.txt | 20 ++++-- 11 files changed, 136 insertions(+), 30 deletions(-) create mode 100644 examples/rsync/fixtures/run-pages/upstream-dirs.txt create mode 100644 examples/rsync/fixtures/run-pages/want/indexed.new create mode 100644 examples/rsync/fixtures/run-pages/want/slash/1/e1 create mode 100644 examples/rsync/fixtures/run-pages/want/slash/2/e1/e2 create mode 100644 examples/rsync/fixtures/run-pages/want/staging/e1/ctan.katoptra.org.directory.index.html create mode 100644 examples/rsync/fixtures/run-pages/want/staging/e1/e2/ctan.katoptra.org.directory.index.html diff --git a/engines/rsync.yml b/engines/rsync.yml index e379e93..bc59342 100644 --- a/engines/rsync.yml +++ b/engines/rsync.yml @@ -79,8 +79,12 @@ vars: URL: https://{{.HOST}} # The listing of the bucket. The key is not an upstream path. STATE: .state/applied.txt.xz - # The state as the directory pages showed it the last time. + # The page state as the directory pages showed it the last time, after a stamp line. INDEXED: .state/indexed.txt.xz + # The format of the directory pages. Increase it at each change to the HTML or the style + # of a page. The stamp in INDEXED includes it. Thus, when a mirror gets a release with a + # new format, its next run makes all of its pages again. + PAGE_FORMAT: '2' # Batches for each run. The remaining batches chain the next run. BATCHES_MAX: '{{.MAX_BATCHES | default 4}}' # Seconds. A retry waits RETRY_S * 2^i, plus jitter. @@ -713,15 +717,16 @@ tasks: # - only when INDEX is set, the directory pages that an index hook writes to two # keys. # One key of a page has the INDEX suffix, which identifies it. The other key is the - # path of a directory of the state, without the slash at the end. It is the same as - # an upstream path. Thus, the list is the directory set of the state. When upstream - # changes a directory into a file, the path goes out of the set, and reconcile - # examines it as a file. + # path of a directory of the state or of upstream, without the slash at the end. It + # is the same as an upstream path. Thus, the list is the directory set of the page + # state, which includes the directories with no file. When upstream changes a + # directory into a file, the path goes out of the set, and reconcile examines it as + # a file. # # Without INDEX, the two lists are empty. If not, the suffix becomes "/", and # reconcile keeps each key that ends in "/". - >- - {{if .INDEX}}{{.ANCESTORS}} {{.RUN}}/applied.txt + {{if .INDEX}}cat {{.RUN}}/applied.txt {{.RUN}}/upstream-dirs.txt | {{.ANCESTORS}} | awk '$0 != "" { print substr($0, 1, length($0) - 1) }' | LC_ALL=C sort > {{.RUN}}/index-keys.txt{{else}}: > {{.RUN}}/index-keys.txt{{end}} - >- cut -f1 {{.RUN}}/bucket.txt | grep -vE '^\.state/'{{if .OWN}} | grep -vxF -f <(printf '%s\n' {{.OWN}}){{end}} @@ -758,11 +763,11 @@ tasks: aws s3 cp {{.AWS_FLAGS}} --recursive --content-type 'text/html; charset=utf-8' "$d" {{.S3}}/; done - {task: index-copy} - {task: remove, vars: {K: '{{.RUN}}/index-gone.txt'}} - # At this point, the pages show the state, and the bucket holds that state, because - # checkpoint put it there before this verb started. One server-side copy makes - # INDEXED the same as the state. No bytes go out of the runner. If a run stops - # before this line, the next run makes the same pages again. - - aws s3 cp {{.AWS_FLAGS}} {{.S3}}/{{.STATE}} {{.S3}}/{{.INDEXED}} + # At this point, the pages show the page state, and the bucket holds the state, + # because checkpoint put it there before this verb started. INDEXED gets + # indexed.new: the stamp, then the page state. If a run stops before this line, the + # next run makes the same pages again. + - {task: push, vars: {FILE: '{{.RUN}}/indexed.new', KEY: '{{.INDEXED}}'}} # Not internal: the offline check of lib starts it with its name. index-copy: @@ -798,9 +803,25 @@ tasks: SLASH: '{{.RUN}}/slash' cmds: - 'mkdir -p {{.STAGING}} {{.SLASH}} && find {{.STAGING}} {{.SLASH}} -mindepth 1 -delete && : > {{.RUN}}/index-real.txt' + # The page state is the state and a line "dir/ TAB TAB" for each directory of + # upstream. Thus, a directory with no file also gets a page, and the page of its + # parent shows it. + - >- + { cat {{.RUN}}/applied.txt; awk '{ print $0 "\t\t" }' {{.RUN}}/upstream-dirs.txt; } + | LC_ALL=C sort > {{.RUN}}/page-state.txt + # The stamp identifies the bytes of the pages: PAGE_FORMAT and each var that changes + # a page. INDEXED starts with the stamp of the pages that it records. The state of + # an earlier release has no stamp. + - >- + printf '%s\n' {{shellQuote .PAGE_FORMAT}} {{shellQuote .INDEX}} {{shellQuote (.INDEX_FILES | default "")}} + {{shellQuote (.PAGE_FOOT | default "")}} {{shellQuote (.UPSTREAM | default "")}} {{shellQuote (.CONTACT | default "")}} + {{shellQuote .HOST}} | sha256sum | awk '{ print "#pages " substr($1, 1, 16) }' > {{.RUN}}/index-stamp.txt + - awk 'NR == 1 && /^#pages / { next } { print }' {{.RUN}}/indexed.txt > {{.RUN}}/index-old.txt # A page changes when a file in its directory, or a directory below it, is added, # changed or removed. Thus, the verb makes the page of each ancestor of each changed - # line again. + # line again. When the stamps do not agree, each directory of the page state is also + # dirty. The changed lines stay in the set. Thus, a directory that upstream removed + # in this run still goes to index-gone. # # ponytail: approximately five pages for each changed directory. Frequently, one # page is sufficient. Ceiling: Class A operations, some hundreds each day, of the @@ -808,9 +829,11 @@ tasks: # the two files. Then make a parent page again only for a new child or a removed # child. - >- - LC_ALL=C comm -3 {{.RUN}}/indexed.txt {{.RUN}}/applied.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }' + { LC_ALL=C comm -3 {{.RUN}}/index-old.txt {{.RUN}}/page-state.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }'; + head -1 {{.RUN}}/indexed.txt | cmp -s - {{.RUN}}/index-stamp.txt || cat {{.RUN}}/page-state.txt; } | {{.ANCESTORS}} | LC_ALL=C sort > {{.RUN}}/index-dirty.txt - - '{{.ANCESTORS}} {{.RUN}}/applied.txt | LC_ALL=C sort > {{.RUN}}/index-dirs.txt' + - '{{.ANCESTORS}} {{.RUN}}/page-state.txt | LC_ALL=C sort > {{.RUN}}/index-dirs.txt' + - cat {{.RUN}}/index-stamp.txt {{.RUN}}/page-state.txt > {{.RUN}}/indexed.new # When a directory is removed, the verb removes its two keys. The root has no # slashless key. - >- @@ -834,11 +857,14 @@ tasks: awk -v r="{{.SLASH}}/" '$0 != "" { n = split($0, c, "/"); d = "" for (i = 1; i < n - 1; i++) d = d c[i] "/"; print r (n - 1) "/" d }' {{.RUN}}/index-make.txt \ | LC_ALL=C sort -u | tr '\n' '\0' | xargs -0r mkdir -p - # The verb gets the entries, "dir TAB d|f TAB name TAB size TAB mtime", from the - # state: the objects in the bucket, not upstream. After the sort, the lines of a - # directory are together, with its subdirectories first. Thus, the verb writes each - # page one time and closes it. The verb percent-encodes each name in href and - # HTML-escapes it in text. + # The verb gets the entries, "dir TAB d|e|f TAB name TAB size TAB mtime", from the + # page state: the files in the bucket and the directories of upstream. A line "dir/" + # of the page state gives an e entry for that directory. The e entry opens the page + # and adds no row. Thus, a directory with no file also gets a page. After the sort, + # the lines of a directory are together, with its subdirectories first and the names + # in version sequence (sort -V: x-2.9 before x-2.10). Thus, the verb writes each page + # one time and closes it. The verb percent-encodes each name in href and HTML-escapes + # it in text. # # The verb makes the full page, and then writes it to its two keys. Thus, the two # keys have the same bytes. With the base href, one document is correct for /dir/ @@ -853,8 +879,8 @@ tasks: awk -F'\t' -v OFS='\t' 'NR == FNR { want[$0] = 1; next } { n = split($1, c, "/"); d = "" for (i = 1; i < n; i++) { if ((d in want) && !((d c[i]) in seen)) { seen[d c[i]] = 1; print d, "d", c[i], "", "" }; d = d c[i] "/" } - if (d in want) print d, "f", c[n], $2, $3 }' {{.RUN}}/index-make.txt {{.RUN}}/applied.txt \ - | LC_ALL=C sort | awk -F'\t' -v S="{{.STAGING}}/" -v R="{{.SLASH}}/" -v I="{{.INDEX}}" -v F={{shellQuote (.PAGE_FOOT | default "")}} \ + if (d in want) print d, (c[n] == "" ? "e" : "f"), c[n], $2, $3 }' {{.RUN}}/index-make.txt {{.RUN}}/page-state.txt \ + | LC_ALL=C sort -t "$(printf '\t')" -k1,1 -k2,2 -k3,3V | awk -F'\t' -v S="{{.STAGING}}/" -v R="{{.SLASH}}/" -v I="{{.INDEX}}" -v F={{shellQuote (.PAGE_FOOT | default "")}} \ -v N={{shellQuote (.INDEX_FILES | default "")}} -v X="{{.RUN}}/index-real.txt" '{{.URLENC}} BEGIN { n = split(N, nm, " "); for (i = n; i >= 1; i--) rank[nm[i]] = i } function esc(s) { gsub(/&/, "\\&", s); gsub(//, "\\>", s); gsub(/"/, "\\"", s); return s } @@ -869,6 +895,7 @@ tasks: if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/") f = R k "/" b; printf("%s", p) > f; close(f) } } !live || $1 != cur { if (live) shut(); open($1) } + $2 == "e" { next } $2 == "d" { p = p "" esc($3) "/\n"; next } ($3 in rank) && (real == "" || rank[$3] < rank[real]) { real = $3 } { p = p "" esc($3) "" $5 "" $4 "\n" } diff --git a/examples/rsync/Taskfile.yml b/examples/rsync/Taskfile.yml index 880da9e..3359d00 100644 --- a/examples/rsync/Taskfile.yml +++ b/examples/rsync/Taskfile.yml @@ -148,11 +148,36 @@ tasks: # index.html, the first name in INDEX_FILES, as the source of its page key. Each byte # must agree with want/. staging keeps the empty directory a/doc/, which git cannot # keep in want/. The upload sends no empty directory. - - {task: pages, vars: {RUN: '{{.F}}/run-pages', STAGING: '{{.F}}/run-pages/staging', INDEX: '{{.CTAN_INDEX}}', PAGE_FOOT: '{{.CTAN_FOOT}}'}} + # + # upstream-dirs.txt also has e1/ and e1/e2/, two directories with no file. Each gets a + # page, and the root page and the page of e1/ show them. indexed.txt has no stamp + # (the format before PAGE_FORMAT 2). Thus, pages makes all the pages again, and it + # still gives the two keys of gone/ for removal. indexed.new starts with the stamp. + - {task: pages, vars: {RUN: '{{.F}}/run-pages', STAGING: '{{.F}}/run-pages/staging', INDEX: '{{.CTAN_INDEX}}', PAGE_FOOT: '{{.CTAN_FOOT}}', UPSTREAM: 'https://ctan.org/tex-archive/'}} + - test "$(tr '\n' '|' < {{.F}}/run-pages/index-make.txt)" = "|a/|a/b/|a/doc/|a/sp ace/|e1/|e1/e2/|" + - test -f "{{.F}}/run-pages/staging/e1/e2/{{.CTAN_INDEX}}" && grep -q 'href="e2/"' "{{.F}}/run-pages/staging/e1/{{.CTAN_INDEX}}" + - head -1 {{.F}}/run-pages/indexed.new | grep -q '^#pages [0-9a-f]\{16\}$' - find {{.F}}/run-pages/staging -type d -empty -delete && diff -r {{.F}}/run-pages/want/staging {{.F}}/run-pages/staging - diff -r {{.F}}/run-pages/want/slash {{.F}}/run-pages/slash - cmp {{.F}}/run-pages/want/index-make.txt {{.F}}/run-pages/index-make.txt && cmp {{.F}}/run-pages/want/index-gone.txt {{.F}}/run-pages/index-gone.txt - - cmp {{.F}}/run-pages/want/index-real.txt {{.F}}/run-pages/index-real.txt + - cmp {{.F}}/run-pages/want/index-real.txt {{.F}}/run-pages/index-real.txt && cmp {{.F}}/run-pages/want/indexed.new {{.F}}/run-pages/indexed.new + # smoke reads the page of the last line of index-make.txt, e1/e2/, which has no file. + # The page must be in staging. fresh then reads timestamp, as in run-smoke, with its + # size in upstream.txt. + - >- + r={{.F}}/run-pages; date -u +%s > $r/staging/timestamp && + printf 'timestamp\t%s\t-\n' "$(wc -c < $r/staging/timestamp | tr -d ' ')" > $r/upstream.txt + - {task: smoke, vars: {RUN: '{{.F}}/run-pages', STAGING: '{{.F}}/run-pages/staging', URL: 'file://{{.F}}/run-pages/staging', INDEX: '{{.CTAN_INDEX}}'}} + # run-pages-stamp: INDEXED is the indexed.new of run-pages. With the same vars, the + # stamp agrees, and pages makes no page. With a different PAGE_FOOT, the stamp does + # not agree, and pages makes the page of each directory again. + - | + r={{.F}}/run-pages-stamp; mkdir -p $r + cp {{.F}}/run-pages/applied.txt {{.F}}/run-pages/upstream-dirs.txt $r/ && cp {{.F}}/run-pages/indexed.new $r/indexed.txt + - {task: pages, vars: {RUN: '{{.F}}/run-pages-stamp', STAGING: '{{.F}}/run-pages-stamp/staging', INDEX: '{{.CTAN_INDEX}}', PAGE_FOOT: '{{.CTAN_FOOT}}', UPSTREAM: 'https://ctan.org/tex-archive/'}} + - test ! -s {{.F}}/run-pages-stamp/index-make.txt + - {task: pages, vars: {RUN: '{{.F}}/run-pages-stamp', STAGING: '{{.F}}/run-pages-stamp/staging', INDEX: '{{.CTAN_INDEX}}', PAGE_FOOT: x, UPSTREAM: 'https://ctan.org/tex-archive/'}} + - cmp {{.F}}/run-pages-stamp/index-make.txt {{.F}}/run-pages-stamp/index-dirs.txt # CTYPE gives the full value of the header, with its charset. - "test \"$(printf 'HTTP/2 200\\r\\nContent-Type: text/plain; charset=utf-8\\r\\n' | {{.CTYPE}})\" = 'text/plain; charset=utf-8'" # run-smoke: an hour with one change only, timestamp, and one new page only, the diff --git a/examples/rsync/fixtures/run-pages/upstream-dirs.txt b/examples/rsync/fixtures/run-pages/upstream-dirs.txt new file mode 100644 index 0000000..5911069 --- /dev/null +++ b/examples/rsync/fixtures/run-pages/upstream-dirs.txt @@ -0,0 +1,6 @@ +a/ +a/b/ +a/doc/ +a/sp ace/ +e1/ +e1/e2/ diff --git a/examples/rsync/fixtures/run-pages/want/index-make.txt b/examples/rsync/fixtures/run-pages/want/index-make.txt index dc77e4a..e44a1c0 100644 --- a/examples/rsync/fixtures/run-pages/want/index-make.txt +++ b/examples/rsync/fixtures/run-pages/want/index-make.txt @@ -3,3 +3,5 @@ a/ a/b/ a/doc/ a/sp ace/ +e1/ +e1/e2/ diff --git a/examples/rsync/fixtures/run-pages/want/indexed.new b/examples/rsync/fixtures/run-pages/want/indexed.new new file mode 100644 index 0000000..a1a1e4d --- /dev/null +++ b/examples/rsync/fixtures/run-pages/want/indexed.new @@ -0,0 +1,13 @@ +#pages 4ea9d5547efb43a3 +a/ +a/b/ +a/b/f.txt 10 2026/08/27 01:00:00 +a/b/new & .txt 7 2026/08/28 06:00:00 +a/doc/ +a/doc/index.htm 4 2026/08/28 06:00:00 +a/doc/index.html 5 2026/08/28 06:00:00 +a/sp ace/ +a/sp ace/g.tar.gz 3 2026/08/28 06:00:00 +e1/ +e1/e2/ +timestamp 20 2026/08/28 06:00:00 diff --git a/examples/rsync/fixtures/run-pages/want/slash/1/e1 b/examples/rsync/fixtures/run-pages/want/slash/1/e1 new file mode 100644 index 0000000..aaaa22d --- /dev/null +++ b/examples/rsync/fixtures/run-pages/want/slash/1/e1 @@ -0,0 +1,7 @@ +Index of /e1/ + + +

Index of /e1/

+ + +
NameLast modifiedSize
../
e2/

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

diff --git a/examples/rsync/fixtures/run-pages/want/slash/2/e1/e2 b/examples/rsync/fixtures/run-pages/want/slash/2/e1/e2 new file mode 100644 index 0000000..3535035 --- /dev/null +++ b/examples/rsync/fixtures/run-pages/want/slash/2/e1/e2 @@ -0,0 +1,6 @@ +Index of /e1/e2/ + + +

Index of /e1/e2/

+ +
NameLast modifiedSize
../

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

diff --git a/examples/rsync/fixtures/run-pages/want/staging/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/ctan.katoptra.org.directory.index.html index c4d322b..bd94927 100644 --- a/examples/rsync/fixtures/run-pages/want/staging/ctan.katoptra.org.directory.index.html +++ b/examples/rsync/fixtures/run-pages/want/staging/ctan.katoptra.org.directory.index.html @@ -3,5 +3,6 @@

Index of /

+
NameLast modifiedSize
a/
e1/
timestamp2026/08/28 06:00:0020

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

diff --git a/examples/rsync/fixtures/run-pages/want/staging/e1/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/e1/ctan.katoptra.org.directory.index.html new file mode 100644 index 0000000..aaaa22d --- /dev/null +++ b/examples/rsync/fixtures/run-pages/want/staging/e1/ctan.katoptra.org.directory.index.html @@ -0,0 +1,7 @@ +Index of /e1/ + + +

Index of /e1/

+ + +
NameLast modifiedSize
../
e2/

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

diff --git a/examples/rsync/fixtures/run-pages/want/staging/e1/e2/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/e1/e2/ctan.katoptra.org.directory.index.html new file mode 100644 index 0000000..3535035 --- /dev/null +++ b/examples/rsync/fixtures/run-pages/want/staging/e1/e2/ctan.katoptra.org.directory.index.html @@ -0,0 +1,6 @@ +Index of /e1/e2/ + + +

Index of /e1/e2/

+ +
NameLast modifiedSize
../

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

diff --git a/examples/rsync/render.txt b/examples/rsync/render.txt index 1640b37..a68dc0f 100644 --- a/examples/rsync/render.txt +++ b/examples/rsync/render.txt @@ -258,8 +258,8 @@ task: [push] xz -T0 -c /work/examples/rsync/.run/applied.new > /work/examples/rs task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/applied.new.xz s3://example/.state/applied.txt.xz task: [rebuild] mv /work/examples/rsync/.run/applied.new /work/examples/rsync/.run/applied.txt task: [rebuild] rm -f /work/examples/rsync/.run/rebuild-now -task: [reconcile] awk -F'\t' '!("" in s) { s[""] = 1; print "" } - { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/examples/rsync/.run/applied.txt | awk '$0 != "" { print substr($0, 1, length($0) - 1) }' | LC_ALL=C sort > /work/examples/rsync/.run/index-keys.txt +task: [reconcile] cat /work/examples/rsync/.run/applied.txt /work/examples/rsync/.run/upstream-dirs.txt | awk -F'\t' '!("" in s) { s[""] = 1; print "" } + { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' | awk '$0 != "" { print substr($0, 1, length($0) - 1) }' | LC_ALL=C sort > /work/examples/rsync/.run/index-keys.txt task: [reconcile] cut -f1 /work/examples/rsync/.run/bucket.txt | grep -vE '^\.state/' | awk -v i="/example.katoptra.org.directory.index.html" '{ s = "/" $0 } substr(s, length(s) - length(i) + 1) != i' | LC_ALL=C comm -23 - /work/examples/rsync/.run/index-keys.txt | LC_ALL=C comm -23 - /work/examples/rsync/.run/paths.txt > /work/examples/rsync/.run/orphans.txt task: [remove] rm -f /work/examples/rsync/.run/del-*; split -l 1000 -a 4 /work/examples/rsync/.run/orphans.txt /work/examples/rsync/.run/del- task: [remove] for f in /work/examples/rsync/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"") @@ -282,10 +282,14 @@ case $rc in esac task: [pages] mkdir -p /work/examples/rsync/staging /work/examples/rsync/.run/slash && find /work/examples/rsync/staging /work/examples/rsync/.run/slash -mindepth 1 -delete && : > /work/examples/rsync/.run/index-real.txt -task: [pages] LC_ALL=C comm -3 /work/examples/rsync/.run/indexed.txt /work/examples/rsync/.run/applied.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }' | awk -F'\t' '!("" in s) { s[""] = 1; print "" } +task: [pages] { cat /work/examples/rsync/.run/applied.txt; awk '{ print $0 "\t\t" }' /work/examples/rsync/.run/upstream-dirs.txt; } | LC_ALL=C sort > /work/examples/rsync/.run/page-state.txt +task: [pages] printf '%s\n' 2 example.katoptra.org.directory.index.html 'index.html index.htm' 'A mirror of rsync.example.org/pub/. Report a problem.' '' '' example.katoptra.org | sha256sum | awk '{ print "#pages " substr($1, 1, 16) }' > /work/examples/rsync/.run/index-stamp.txt +task: [pages] awk 'NR == 1 && /^#pages / { next } { print }' /work/examples/rsync/.run/indexed.txt > /work/examples/rsync/.run/index-old.txt +task: [pages] { LC_ALL=C comm -3 /work/examples/rsync/.run/index-old.txt /work/examples/rsync/.run/page-state.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }'; head -1 /work/examples/rsync/.run/indexed.txt | cmp -s - /work/examples/rsync/.run/index-stamp.txt || cat /work/examples/rsync/.run/page-state.txt; } | awk -F'\t' '!("" in s) { s[""] = 1; print "" } { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' | LC_ALL=C sort > /work/examples/rsync/.run/index-dirty.txt task: [pages] awk -F'\t' '!("" in s) { s[""] = 1; print "" } - { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/examples/rsync/.run/applied.txt | LC_ALL=C sort > /work/examples/rsync/.run/index-dirs.txt + { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/examples/rsync/.run/page-state.txt | LC_ALL=C sort > /work/examples/rsync/.run/index-dirs.txt +task: [pages] cat /work/examples/rsync/.run/index-stamp.txt /work/examples/rsync/.run/page-state.txt > /work/examples/rsync/.run/indexed.new task: [pages] LC_ALL=C comm -23 /work/examples/rsync/.run/index-dirty.txt /work/examples/rsync/.run/index-dirs.txt | awk -v i="example.katoptra.org.directory.index.html" '{ print $0 i } $0 != "" { print substr($0, 1, length($0) - 1) }' > /work/examples/rsync/.run/index-gone.txt task: [pages] LC_ALL=C comm -12 /work/examples/rsync/.run/index-dirty.txt /work/examples/rsync/.run/index-dirs.txt > /work/examples/rsync/.run/index-make.txt task: [pages] test ! -s /work/examples/rsync/.run/index-make.txt || awk -v s="/work/examples/rsync/staging/" '{ print s $0 }' /work/examples/rsync/.run/index-make.txt | tr '\n' '\0' | xargs -0 mkdir -p @@ -295,8 +299,8 @@ task: [pages] test ! -s /work/examples/rsync/.run/index-make.txt || awk -v r="/w task: [pages] test ! -s /work/examples/rsync/.run/index-make.txt || awk -F'\t' -v OFS='\t' 'NR == FNR { want[$0] = 1; next } { n = split($1, c, "/"); d = "" for (i = 1; i < n; i++) { if ((d in want) && !((d c[i]) in seen)) { seen[d c[i]] = 1; print d, "d", c[i], "", "" }; d = d c[i] "/" } - if (d in want) print d, "f", c[n], $2, $3 }' /work/examples/rsync/.run/index-make.txt /work/examples/rsync/.run/applied.txt \ -| LC_ALL=C sort | awk -F'\t' -v S="/work/examples/rsync/staging/" -v R="/work/examples/rsync/.run/slash/" -v I="example.katoptra.org.directory.index.html" -v F='A mirror of rsync.example.org/pub/. Report a problem.' \ + if (d in want) print d, (c[n] == "" ? "e" : "f"), c[n], $2, $3 }' /work/examples/rsync/.run/index-make.txt /work/examples/rsync/.run/page-state.txt \ +| LC_ALL=C sort -t "$(printf '\t')" -k1,1 -k2,2 -k3,3V | awk -F'\t' -v S="/work/examples/rsync/staging/" -v R="/work/examples/rsync/.run/slash/" -v I="example.katoptra.org.directory.index.html" -v F='A mirror of rsync.example.org/pub/. Report a problem.' \ -v N='index.html index.htm' -v X="/work/examples/rsync/.run/index-real.txt" 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } @@ -313,6 +317,7 @@ function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = s if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/") f = R k "/" b; printf("%s", p) > f; close(f) } } !live || $1 != cur { if (live) shut(); open($1) } + $2 == "e" { next } $2 == "d" { p = p "" esc($3) "/\n"; next } ($3 in rank) && (real == "" || rank[$3] < rank[real]) { real = $3 } { p = p "" esc($3) "" $5 "" $4 "\n" } @@ -324,7 +329,8 @@ task: [remove] rm -f /work/examples/rsync/.run/del-*; split -l 1000 -a 4 /work/e task: [remove] for f in /work/examples/rsync/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"") printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" \ > "$f.json" && aws s3api delete-objects --bucket example --delete "file://$f.json" | tee -a /work/examples/rsync/.run/publish.txt | { test "$(grep -c Errors)" = 0; }; done -task: [index] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 s3://example/.state/applied.txt.xz s3://example/.state/indexed.txt.xz +task: [push] xz -T0 -c /work/examples/rsync/.run/indexed.new > /work/examples/rsync/.run/indexed.new.xz +task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/indexed.new.xz s3://example/.state/indexed.txt.xz task: [smoke] { awk -F'\t' '!seen[$2]++' /work/examples/rsync/.run/labeled.txt 2>/dev/null; } | while IFS="$(printf '\t')" read -r k t; do u=$(printf '%s\n' "$k" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2); want=$(awk -F'\t' -v k="$k" '$1 == k { print $2 }' /work/examples/rsync/.run/upstream.txt); h=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://example.katoptra.org/$u"); got=$(printf '%s\n' "$h" | awk 'tolower($1) == "content-range:" { split($3, a, "/"); r = a[2] } From 6928ab794e6b65e2bf256efb1eb084bf858d6369 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Mon, 19 Oct 2026 18:06:54 -0700 Subject: [PATCH 3/5] feat(rsync): design C directory pages with a lib footer --- engines/rsync.yml | 66 +++++++++++++---- examples/rsync/Taskfile.yml | 66 +++++++++++++---- examples/rsync/fixtures/run-pages/applied.txt | 12 ++++ .../fixtures/run-pages/upstream-dirs.txt | 1 + .../fixtures/run-pages/want/index-make.txt | 1 + .../rsync/fixtures/run-pages/want/indexed.new | 15 +++- .../rsync/fixtures/run-pages/want/slash/1/a | 10 +-- .../rsync/fixtures/run-pages/want/slash/1/e1 | 10 +-- .../rsync/fixtures/run-pages/want/slash/2/a/b | 21 ++++-- .../fixtures/run-pages/want/slash/2/a/doc | 14 ++-- .../fixtures/run-pages/want/slash/2/a/sp ace | 13 ++-- .../fixtures/run-pages/want/slash/2/e1/e2 | 10 +-- .../want/slash/3/a/sp ace/\303\274%&" | 9 +++ .../b/ctan.katoptra.org.directory.index.html | 21 ++++-- .../a/ctan.katoptra.org.directory.index.html | 10 +-- .../ctan.katoptra.org.directory.index.html | 13 ++-- .../ctan.katoptra.org.directory.index.html" | 9 +++ .../ctan.katoptra.org.directory.index.html | 12 ++-- .../e1/ctan.katoptra.org.directory.index.html | 10 +-- .../e2/ctan.katoptra.org.directory.index.html | 10 +-- examples/rsync/render.txt | 70 ++++++++++++++----- 21 files changed, 303 insertions(+), 100 deletions(-) create mode 100644 "examples/rsync/fixtures/run-pages/want/slash/3/a/sp ace/\303\274%&" create mode 100644 "examples/rsync/fixtures/run-pages/want/staging/a/sp ace/\303\274%&/ctan.katoptra.org.directory.index.html" diff --git a/engines/rsync.yml b/engines/rsync.yml index bc59342..68268af 100644 --- a/engines/rsync.yml +++ b/engines/rsync.yml @@ -881,24 +881,62 @@ tasks: for (i = 1; i < n; i++) { if ((d in want) && !((d c[i]) in seen)) { seen[d c[i]] = 1; print d, "d", c[i], "", "" }; d = d c[i] "/" } if (d in want) print d, (c[n] == "" ? "e" : "f"), c[n], $2, $3 }' {{.RUN}}/index-make.txt {{.RUN}}/page-state.txt \ | LC_ALL=C sort -t "$(printf '\t')" -k1,1 -k2,2 -k3,3V | awk -F'\t' -v S="{{.STAGING}}/" -v R="{{.SLASH}}/" -v I="{{.INDEX}}" -v F={{shellQuote (.PAGE_FOOT | default "")}} \ - -v N={{shellQuote (.INDEX_FILES | default "")}} -v X="{{.RUN}}/index-real.txt" '{{.URLENC}} - BEGIN { n = split(N, nm, " "); for (i = n; i >= 1; i--) rank[nm[i]] = i } + -v N={{shellQuote (.INDEX_FILES | default "")}} -v X="{{.RUN}}/index-real.txt" -v U={{shellQuote (.UPSTREAM | default "")}} \ + -v C={{shellQuote (.CONTACT | default "")}} -v H="{{.HOST}}" '{{.URLENC}} + BEGIN { n = split(N, nm, " "); for (i = n; i >= 1; i--) rank[nm[i]] = i + if (C == "" && (j = index(H, ".")) > 1) C = substr(H, 1, j - 1) "@" substr(H, j + 1) + css = ":root{color-scheme:light dark;--bg:#f6f5f2;--fg:#1c1c28;--mut:#6e6e7e;--line:#e2e0da;--hov:#edebe5;--a:#3b45d9}" + css = css "@media (prefers-color-scheme:dark){:root{--bg:#121218;--fg:#f0f0f5;--mut:#8b8b9c;--line:#262630;--hov:#1a1a22;--a:#8b93ff}}" + css = css "body{margin:0 auto;max-width:60rem;padding:2rem 1rem 3rem;background:var(--bg);color:var(--fg);font:15px/1.45 system-ui,-apple-system,sans-serif}" + css = css "h1{font-size:1.35rem;font-weight:650;margin:0;overflow-wrap:anywhere}p{color:var(--mut);margin:.2rem 0 1.25rem;font-size:.9rem}" + css = css "table{width:100%;border-collapse:collapse;font:13.5px/1.4 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}" + css = css "th{font:600 .7rem/1 system-ui,sans-serif;letter-spacing:.06em;text-transform:uppercase;color:var(--mut);text-align:left;padding:0 .5rem .6rem;white-space:nowrap}" + css = css "td{padding:.4rem .5rem;border-top:1px solid var(--line);vertical-align:baseline}td:first-child{overflow-wrap:anywhere;width:100%}" + css = css "td+td{white-space:nowrap;color:var(--mut);font-variant-numeric:tabular-nums}th:last-child,td:last-child{text-align:right}" + css = css "tr:hover td{background:var(--hov)}a{color:var(--a);text-decoration:none}a:hover{text-decoration:underline}.s a{color:var(--mut)}" + css = css ".k{font-size:.72rem;margin-left:.6rem;padding:.05rem .4rem;border:1px solid var(--line);border-radius:.3rem;color:var(--mut)}" + css = css ".k:hover{color:var(--a);border-color:var(--a);text-decoration:none}" + css = css "footer{margin-top:2rem;padding-top:1rem;border-top:1px solid var(--line);color:var(--mut);font-size:.85rem;overflow-wrap:anywhere}" + css = css "footer a{color:inherit;text-decoration:underline}@media (max-width:36rem){thead{display:none}" + css = css "tr{display:grid;grid-template-columns:1fr auto;border-top:1px solid var(--line)}td{border:0;padding:.1rem .5rem}" + css = css "td:first-child{grid-column:1/-1;padding-top:.45rem}td+td{padding-bottom:.45rem;font-size:.8rem}}" } function esc(s) { gsub(/&/, "\\&", s); gsub(//, "\\>", s); gsub(/"/, "\\"", s); return s } - function open(d) { cur = d; live = 1; real = "" - p = "Index of /" esc(d) "\n" - p = p "\n" - p = p "\n" - p = p "

Index of /" esc(d) "

\n" - if (d != "") p = p "\n" } - function shut( f, b, k, e) { p = p "
NameLast modifiedSize
../
" (F == "" ? "" : "

" sprintf(F, enc(cur)) "

") "\n" - if (real == "") { f = S cur I; printf("%s", p) > f; close(f) } else print cur real "\t" cur I > X - if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/") - f = R k "/" b; printf("%s", p) > f; close(f) } } + function grp(n, s, o) { s = sprintf("%.0f", n); o = "" + while (length(s) > 3) { o = "," substr(s, length(s) - 2) o; s = substr(s, 1, length(s) - 3) } return s o } + function iec(n, u, i) { if (n < 1024) return sprintf("%.0f B", n) + split("KiB MiB GiB TiB", u, " "); n /= 1024; i = 1; while (n >= 1024 && i < 4) { n /= 1024; i++ } + return sprintf(n < 10 ? "%.1f %s" : "%.0f %s", n, u[i]) } + function ymd(t) { gsub(/\//, "-", t); return substr(t, 1, 16) } + function foot( t) { if (F != "") return "
" sprintf(F, enc(cur)) "
" + t = (U == "" ? "" : "Mirror of " esc(U) enc(cur) ".") + if (C != "") t = t (t == "" ? "" : " ") "Report a problem to " esc(C) "." + return (t == "" ? "" : "
" t "
") } + function open(d) { cur = d; live = 1; real = ""; nd = 0; nfl = 0; tot = 0; split("", has) } + function w(s) { if (real == "") printf("%s", s) > fa; if (cur != "") printf("%s", s) > fb } + function shut( i, r, x, c, e, k, b) { split("", fold); split("", links) + for (i = 1; i <= nfl; i++) { r = fname[i]; x = "" + while (match(r, /\.(sig|asc|sign|sha256|sha512)$/)) { r = substr(r, 1, RSTART - 1); if (r in has) x = r } + if (x != "") { fold[i] = 1 + links[x] = links[x] "" esc(substr(fname[i], length(x) + 1)) "" } } + c = (nd ? grp(nd) (nd == 1 ? " directory" : " directories") : "") + if (nfl) c = c (c == "" ? "" : ", ") grp(nfl) (nfl == 1 ? " file" : " files") ", " iec(tot) + if (c == "") c = "This directory is empty." + fa = S cur I; if (real != "") print cur real "\t" cur I > X + if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/"); fb = R k "/" b } + w("\nIndex of /" esc(cur) "\n") + w("\n\n

Index of /" esc(cur) "

" c "

\n") + w("\n") + if (cur != "") w("\n") + for (i = 1; i <= nd; i++) w("\n") + for (i = 1; i <= nfl; i++) if (!(i in fold)) + w("\n") + w("
NameModified (UTC)Size
../
" esc(dname[i]) "/
" esc(fname[i]) "" links[fname[i]] "" ymd(ftime[i]) "" iec(fsize[i]) "
" foot() "\n") + if (real == "") close(fa); if (cur != "") close(fb) } !live || $1 != cur { if (live) shut(); open($1) } $2 == "e" { next } - $2 == "d" { p = p "" esc($3) "/\n"; next } + $2 == "d" { dname[++nd] = $3; next } ($3 in rank) && (real == "" || rank[$3] < rank[real]) { real = $3 } - { p = p "" esc($3) "" $5 "" $4 "\n" } + { fname[++nfl] = $3; fsize[nfl] = $4; ftime[nfl] = $5; has[$3] = 1; tot += $4 } END { if (live) shut() }' smoke: diff --git a/examples/rsync/Taskfile.yml b/examples/rsync/Taskfile.yml index 3359d00..ebc069a 100644 --- a/examples/rsync/Taskfile.yml +++ b/examples/rsync/Taskfile.yml @@ -14,7 +14,7 @@ vars: LIST_FLOOR: 1 INDEX: '{{.HOST}}.directory.index.html' INDEX_FILES: index.html index.htm - PAGE_FOOT: 'A mirror of rsync.example.org/pub/. Report a problem.' + UPSTREAM: https://example.org/pub/ CANARY: pub/index.html FRESH_KEY: timestamp includes: @@ -52,11 +52,12 @@ tasks: # A local tree. Here, the daemon-only flags of the engine are a usage error. LOCAL: rsync --no-h FP: 3505F9E22FF030FE5D9C655A825C376AB4571163 - # run-pages/want/ is the page set of ctan for that state. It is the same as the - # pages that ctan.katoptra.org sends. It has the INDEX of ctan, and a last line with - # a link to the directory on ctan.org. + # run-pages/want/ is the page set of ctan for that state. It has the INDEX, the + # UPSTREAM and the HOST of ctan. Thus, the footer that the engine makes gives the + # directory on ctan.org as text, and the address ctan@katoptra.org. CTAN_INDEX: ctan.katoptra.org.directory.index.html - CTAN_FOOT: 'This directory on ctan.org. Sizes in bytes, times as CTAN lists them.' + CTAN_HOST: ctan.katoptra.org + CTAN_UP: https://ctan.org/tex-archive/ cmds: # list and normalise on a local tree: its two regular files, sorted with byte # collation, and no directory. upstream-dirs.txt gets its two directories, each with @@ -140,9 +141,10 @@ tasks: # verify rejects a container that the tlpdb does not identify, independently of its # bytes. - '! task verify-refuses 2>/dev/null' - # run-pages: the new state has a file with & < > in its name, and a directory with a - # space in its name. gone/ is not in the new state. pages makes the pages of the - # root, a/, a/b/ and a/sp ace/, each with its two keys (the root with one). It also + # run-pages: the new state has a file with & < > in its name, a directory with a + # space in its name, and a/sp ace/ü%&/, which the URLs encode. gone/ is not in the + # new state. pages makes the pages of the root, a/, a/b/, a/sp ace/ and + # a/sp ace/ü%&/, each with its two keys (the root with one). It also # gives the two keys of gone/ for removal. a/doc/ has index.htm and index.html. Thus, # its listing goes only to its key without the slash, and index-real.txt gives # index.html, the first name in INDEX_FILES, as the source of its page key. Each byte @@ -153,10 +155,38 @@ tasks: # page, and the root page and the page of e1/ show them. indexed.txt has no stamp # (the format before PAGE_FORMAT 2). Thus, pages makes all the pages again, and it # still gives the two keys of gone/ for removal. indexed.new starts with the stamp. - - {task: pages, vars: {RUN: '{{.F}}/run-pages', STAGING: '{{.F}}/run-pages/staging', INDEX: '{{.CTAN_INDEX}}', PAGE_FOOT: '{{.CTAN_FOOT}}', UPSTREAM: 'https://ctan.org/tex-archive/'}} - - test "$(tr '\n' '|' < {{.F}}/run-pages/index-make.txt)" = "|a/|a/b/|a/doc/|a/sp ace/|e1/|e1/e2/|" + - {task: pages, vars: {RUN: '{{.F}}/run-pages', STAGING: '{{.F}}/run-pages/staging', INDEX: '{{.CTAN_INDEX}}', HOST: '{{.CTAN_HOST}}', UPSTREAM: '{{.CTAN_UP}}'}} + - test "$(tr '\n' '|' < {{.F}}/run-pages/index-make.txt)" = "|a/|a/b/|a/doc/|a/sp ace/|a/sp ace/ü%&/|e1/|e1/e2/|" - test -f "{{.F}}/run-pages/staging/e1/e2/{{.CTAN_INDEX}}" && grep -q 'href="e2/"' "{{.F}}/run-pages/staging/e1/{{.CTAN_INDEX}}" - head -1 {{.F}}/run-pages/indexed.new | grep -q '^#pages [0-9a-f]\{16\}$' + # The page of a/b/: + # - The names are in version sequence: x-2.2, x-2.10, x-2.12, x-2.12.3. + # - A signature or checksum file goes on the row of its file, as a link with its + # suffix, directly after the name. y.exe.sha512.asc goes on the row of y.exe. + # - A suffix file with no file before the suffix keeps its row, with class s. + # - The size is IEC, with the exact bytes in the title. The time is UTC, to the + # minute. + # - The footer gives UPSTREAM and the directory as text, and CONTACT, which comes + # from HOST. + # The page of a/sp ace/ü%&/ escapes the name in the heading and encodes it in the URLs. + # No page has a script, an image or a link to a different site. + - | + s={{.F}}/run-pages/staging; p="$s/a/b/{{.CTAN_INDEX}}" + test "$(grep -o 'href="x-[^"]*\.tar\.gz"' "$p" | tr '\n' ' ')" = 'href="x-2.2.tar.gz" href="x-2.10.tar.gz" href="x-2.12.tar.gz" href="x-2.12.3.tar.gz" ' + grep -qF 'x-2.12.3.tar.gz.asc.sig' "$p" + grep -qF 'y.exe.sha512.sha512.asc' "$p" + test "$(grep -c '">y.exe.sha512' "$p")" -eq 0 + grep -qF 'orphan.tar.gz.sig' "$p" + grep -qF '.sig' "$p" + grep -qF '1.1 MiB' "$p" + grep -qF 'Modified (UTC)' "$p" && grep -qF '2026-08-28 06:00' "$p" + grep -qF '' "$p" + u="$s/a/sp ace/ü%&/{{.CTAN_INDEX}}" + grep -qF '

Index of /a/sp ace/ü%&/

' "$u" && grep -qF '' "$u" + grep -qF 'tex-archive/a/sp%20ace/%C3%BC%25%26/.' "$u" + grep -qF '

This directory is empty.

' "$s/e1/e2/{{.CTAN_INDEX}}" + grep -qF '

2 directories, 1 file, 20 B

' "$s/{{.CTAN_INDEX}}" + ! grep -rlE 'x' + # run-pages-big: one directory with 10,000 files, each with a .sig. The page has the + # 10,000 files, each with its signature link, in less than 10 seconds. + - | + r={{.F}}/run-pages-big; mkdir -p $r; : > $r/indexed.txt; echo big/ > $r/upstream-dirs.txt + awk 'BEGIN { for (i = 0; i < 10000; i++) { f = sprintf("big/f-%05d.tar.gz", i); print f "\t1234\t2026/08/28 06:00:00"; print f ".sig\t65\t2026/08/28 06:00:00" } }' \ + | LC_ALL=C sort > $r/applied.txt; date +%s > $r/t0 + - {task: pages, vars: {RUN: '{{.F}}/run-pages-big', STAGING: '{{.F}}/run-pages-big/staging', INDEX: '{{.CTAN_INDEX}}', HOST: '{{.CTAN_HOST}}', UPSTREAM: '{{.CTAN_UP}}'}} + - | + r={{.F}}/run-pages-big; test $(( $(date +%s) - $(cat $r/t0) )) -lt 10 + test "$(grep -c 'class="k"' "$r/staging/big/{{.CTAN_INDEX}}")" -eq 10000 # CTYPE gives the full value of the header, with its charset. - "test \"$(printf 'HTTP/2 200\\r\\nContent-Type: text/plain; charset=utf-8\\r\\n' | {{.CTYPE}})\" = 'text/plain; charset=utf-8'" # run-smoke: an hour with one change only, timestamp, and one new page only, the diff --git a/examples/rsync/fixtures/run-pages/applied.txt b/examples/rsync/fixtures/run-pages/applied.txt index 8a38edf..8a74589 100644 --- a/examples/rsync/fixtures/run-pages/applied.txt +++ b/examples/rsync/fixtures/run-pages/applied.txt @@ -1,6 +1,18 @@ +a/b/.sig 65 2026/08/28 06:00:00 a/b/f.txt 10 2026/08/27 01:00:00 a/b/new & .txt 7 2026/08/28 06:00:00 +a/b/orphan.tar.gz.sig 65 2026/08/28 06:00:00 +a/b/x-2.10.tar.gz 300 2026/08/28 06:00:00 +a/b/x-2.12.3.tar.gz 1189208 2026/08/28 06:00:00 +a/b/x-2.12.3.tar.gz.asc 488 2026/08/28 06:00:00 +a/b/x-2.12.3.tar.gz.sig 310 2026/08/28 06:00:00 +a/b/x-2.12.tar.gz 350 2026/08/28 06:00:00 +a/b/x-2.2.tar.gz 250 2026/08/28 06:00:00 +a/b/y.exe 2048 2026/08/28 06:00:00 +a/b/y.exe.sha512 128 2026/08/28 06:00:00 +a/b/y.exe.sha512.asc 488 2026/08/28 06:00:00 a/doc/index.htm 4 2026/08/28 06:00:00 a/doc/index.html 5 2026/08/28 06:00:00 a/sp ace/g.tar.gz 3 2026/08/28 06:00:00 +a/sp ace/ü%&/z.txt 5 2026/08/28 06:00:00 timestamp 20 2026/08/28 06:00:00 diff --git a/examples/rsync/fixtures/run-pages/upstream-dirs.txt b/examples/rsync/fixtures/run-pages/upstream-dirs.txt index 5911069..27c9a65 100644 --- a/examples/rsync/fixtures/run-pages/upstream-dirs.txt +++ b/examples/rsync/fixtures/run-pages/upstream-dirs.txt @@ -2,5 +2,6 @@ a/ a/b/ a/doc/ a/sp ace/ +a/sp ace/ü%&/ e1/ e1/e2/ diff --git a/examples/rsync/fixtures/run-pages/want/index-make.txt b/examples/rsync/fixtures/run-pages/want/index-make.txt index e44a1c0..20b39d2 100644 --- a/examples/rsync/fixtures/run-pages/want/index-make.txt +++ b/examples/rsync/fixtures/run-pages/want/index-make.txt @@ -3,5 +3,6 @@ a/ a/b/ a/doc/ a/sp ace/ +a/sp ace/ü%&/ e1/ e1/e2/ diff --git a/examples/rsync/fixtures/run-pages/want/indexed.new b/examples/rsync/fixtures/run-pages/want/indexed.new index a1a1e4d..a8f2b42 100644 --- a/examples/rsync/fixtures/run-pages/want/indexed.new +++ b/examples/rsync/fixtures/run-pages/want/indexed.new @@ -1,13 +1,26 @@ -#pages 4ea9d5547efb43a3 +#pages c822eeaf3ab7cc3d a/ a/b/ +a/b/.sig 65 2026/08/28 06:00:00 a/b/f.txt 10 2026/08/27 01:00:00 a/b/new & .txt 7 2026/08/28 06:00:00 +a/b/orphan.tar.gz.sig 65 2026/08/28 06:00:00 +a/b/x-2.10.tar.gz 300 2026/08/28 06:00:00 +a/b/x-2.12.3.tar.gz 1189208 2026/08/28 06:00:00 +a/b/x-2.12.3.tar.gz.asc 488 2026/08/28 06:00:00 +a/b/x-2.12.3.tar.gz.sig 310 2026/08/28 06:00:00 +a/b/x-2.12.tar.gz 350 2026/08/28 06:00:00 +a/b/x-2.2.tar.gz 250 2026/08/28 06:00:00 +a/b/y.exe 2048 2026/08/28 06:00:00 +a/b/y.exe.sha512 128 2026/08/28 06:00:00 +a/b/y.exe.sha512.asc 488 2026/08/28 06:00:00 a/doc/ a/doc/index.htm 4 2026/08/28 06:00:00 a/doc/index.html 5 2026/08/28 06:00:00 a/sp ace/ a/sp ace/g.tar.gz 3 2026/08/28 06:00:00 +a/sp ace/ü%&/ +a/sp ace/ü%&/z.txt 5 2026/08/28 06:00:00 e1/ e1/e2/ timestamp 20 2026/08/28 06:00:00 diff --git a/examples/rsync/fixtures/run-pages/want/slash/1/a b/examples/rsync/fixtures/run-pages/want/slash/1/a index 640dd06..710e3b0 100644 --- a/examples/rsync/fixtures/run-pages/want/slash/1/a +++ b/examples/rsync/fixtures/run-pages/want/slash/1/a @@ -1,9 +1,11 @@ -Index of /a/ + +Index of /a/ - -

Index of /a/

+ +

Index of /a/

3 directories

+
NameLast modifiedSize
-
NameModified (UTC)Size
../
b/
doc/
sp ace/

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

+ diff --git a/examples/rsync/fixtures/run-pages/want/slash/1/e1 b/examples/rsync/fixtures/run-pages/want/slash/1/e1 index aaaa22d..592de76 100644 --- a/examples/rsync/fixtures/run-pages/want/slash/1/e1 +++ b/examples/rsync/fixtures/run-pages/want/slash/1/e1 @@ -1,7 +1,9 @@ -Index of /e1/ + +Index of /e1/ - -

Index of /e1/

+ +

Index of /e1/

1 directory

+
NameLast modifiedSize
-
NameModified (UTC)Size
../
e2/

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

+ diff --git a/examples/rsync/fixtures/run-pages/want/slash/2/a/b b/examples/rsync/fixtures/run-pages/want/slash/2/a/b index ce432b2..9e9c1d0 100644 --- a/examples/rsync/fixtures/run-pages/want/slash/2/a/b +++ b/examples/rsync/fixtures/run-pages/want/slash/2/a/b @@ -1,8 +1,17 @@ -Index of /a/b/ + +Index of /a/b/ - -

Index of /a/b/

+ +

Index of /a/b/

13 files, 1.1 MiB

+
NameLast modifiedSize
- - -
NameModified (UTC)Size
../
f.txt2026/08/27 01:00:0010
new & <odd>.txt2026/08/28 06:00:007

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

+.sig2026-08-28 06:0065 B +f.txt2026-08-27 01:0010 B +new & <odd>.txt2026-08-28 06:007 B +orphan.tar.gz.sig2026-08-28 06:0065 B +x-2.2.tar.gz2026-08-28 06:00250 B +x-2.10.tar.gz2026-08-28 06:00300 B +x-2.12.tar.gz2026-08-28 06:00350 B +x-2.12.3.tar.gz.asc.sig2026-08-28 06:001.1 MiB +y.exe.sha512.sha512.asc2026-08-28 06:002.0 KiB + diff --git a/examples/rsync/fixtures/run-pages/want/slash/2/a/doc b/examples/rsync/fixtures/run-pages/want/slash/2/a/doc index 1146f63..2e78ae8 100644 --- a/examples/rsync/fixtures/run-pages/want/slash/2/a/doc +++ b/examples/rsync/fixtures/run-pages/want/slash/2/a/doc @@ -1,8 +1,10 @@ -Index of /a/doc/ + +Index of /a/doc/ - -

Index of /a/doc/

+ +

Index of /a/doc/

2 files, 9 B

+
NameLast modifiedSize
- - -
NameModified (UTC)Size
../
index.htm2026/08/28 06:00:004
index.html2026/08/28 06:00:005

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

+index.htm2026-08-28 06:004 B +index.html2026-08-28 06:005 B +
Mirror of https://ctan.org/tex-archive/a/doc/. Report a problem to ctan@katoptra.org.
diff --git a/examples/rsync/fixtures/run-pages/want/slash/2/a/sp ace b/examples/rsync/fixtures/run-pages/want/slash/2/a/sp ace index ae7c38b..ccfab96 100644 --- a/examples/rsync/fixtures/run-pages/want/slash/2/a/sp ace +++ b/examples/rsync/fixtures/run-pages/want/slash/2/a/sp ace @@ -1,7 +1,10 @@ -Index of /a/sp ace/ + +Index of /a/sp ace/ - -

Index of /a/sp ace/

+ +

Index of /a/sp ace/

1 directory, 1 file, 3 B

+
NameLast modifiedSize
- -
NameModified (UTC)Size
../
g.tar.gz2026/08/28 06:00:003

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

+ü%&/ +g.tar.gz2026-08-28 06:003 B +
Mirror of https://ctan.org/tex-archive/a/sp%20ace/. Report a problem to ctan@katoptra.org.
diff --git a/examples/rsync/fixtures/run-pages/want/slash/2/e1/e2 b/examples/rsync/fixtures/run-pages/want/slash/2/e1/e2 index 3535035..b76d6d7 100644 --- a/examples/rsync/fixtures/run-pages/want/slash/2/e1/e2 +++ b/examples/rsync/fixtures/run-pages/want/slash/2/e1/e2 @@ -1,6 +1,8 @@ -Index of /e1/e2/ + +Index of /e1/e2/ - -

Index of /e1/e2/

+ +

Index of /e1/e2/

This directory is empty.

+
NameLast modifiedSize
-
NameModified (UTC)Size
../

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

+
Mirror of https://ctan.org/tex-archive/e1/e2/. Report a problem to ctan@katoptra.org.
diff --git "a/examples/rsync/fixtures/run-pages/want/slash/3/a/sp ace/\303\274%&" "b/examples/rsync/fixtures/run-pages/want/slash/3/a/sp ace/\303\274%&" new file mode 100644 index 0000000..1569f10 --- /dev/null +++ "b/examples/rsync/fixtures/run-pages/want/slash/3/a/sp ace/\303\274%&" @@ -0,0 +1,9 @@ + +Index of /a/sp ace/ü%&/ + + +

Index of /a/sp ace/ü%&/

1 file, 5 B

+ + + +
NameModified (UTC)Size
../
z.txt2026-08-28 06:005 B
Mirror of https://ctan.org/tex-archive/a/sp%20ace/%C3%BC%25%26/. Report a problem to ctan@katoptra.org.
diff --git a/examples/rsync/fixtures/run-pages/want/staging/a/b/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/a/b/ctan.katoptra.org.directory.index.html index ce432b2..9e9c1d0 100644 --- a/examples/rsync/fixtures/run-pages/want/staging/a/b/ctan.katoptra.org.directory.index.html +++ b/examples/rsync/fixtures/run-pages/want/staging/a/b/ctan.katoptra.org.directory.index.html @@ -1,8 +1,17 @@ -Index of /a/b/ + +Index of /a/b/ - -

Index of /a/b/

+ +

Index of /a/b/

13 files, 1.1 MiB

+
NameLast modifiedSize
- - -
NameModified (UTC)Size
../
f.txt2026/08/27 01:00:0010
new & <odd>.txt2026/08/28 06:00:007

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

+.sig2026-08-28 06:0065 B +f.txt2026-08-27 01:0010 B +new & <odd>.txt2026-08-28 06:007 B +orphan.tar.gz.sig2026-08-28 06:0065 B +x-2.2.tar.gz2026-08-28 06:00250 B +x-2.10.tar.gz2026-08-28 06:00300 B +x-2.12.tar.gz2026-08-28 06:00350 B +x-2.12.3.tar.gz.asc.sig2026-08-28 06:001.1 MiB +y.exe.sha512.sha512.asc2026-08-28 06:002.0 KiB + diff --git a/examples/rsync/fixtures/run-pages/want/staging/a/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/a/ctan.katoptra.org.directory.index.html index 640dd06..710e3b0 100644 --- a/examples/rsync/fixtures/run-pages/want/staging/a/ctan.katoptra.org.directory.index.html +++ b/examples/rsync/fixtures/run-pages/want/staging/a/ctan.katoptra.org.directory.index.html @@ -1,9 +1,11 @@ -Index of /a/ + +Index of /a/ - -

Index of /a/

+ +

Index of /a/

3 directories

+
NameLast modifiedSize
-
NameModified (UTC)Size
../
b/
doc/
sp ace/

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

+ diff --git a/examples/rsync/fixtures/run-pages/want/staging/a/sp ace/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/a/sp ace/ctan.katoptra.org.directory.index.html index ae7c38b..ccfab96 100644 --- a/examples/rsync/fixtures/run-pages/want/staging/a/sp ace/ctan.katoptra.org.directory.index.html +++ b/examples/rsync/fixtures/run-pages/want/staging/a/sp ace/ctan.katoptra.org.directory.index.html @@ -1,7 +1,10 @@ -Index of /a/sp ace/ + +Index of /a/sp ace/ - -

Index of /a/sp ace/

+ +

Index of /a/sp ace/

1 directory, 1 file, 3 B

+
NameLast modifiedSize
- -
NameModified (UTC)Size
../
g.tar.gz2026/08/28 06:00:003

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

+ü%&/ +g.tar.gz2026-08-28 06:003 B +
Mirror of https://ctan.org/tex-archive/a/sp%20ace/. Report a problem to ctan@katoptra.org.
diff --git "a/examples/rsync/fixtures/run-pages/want/staging/a/sp ace/\303\274%&/ctan.katoptra.org.directory.index.html" "b/examples/rsync/fixtures/run-pages/want/staging/a/sp ace/\303\274%&/ctan.katoptra.org.directory.index.html" new file mode 100644 index 0000000..1569f10 --- /dev/null +++ "b/examples/rsync/fixtures/run-pages/want/staging/a/sp ace/\303\274%&/ctan.katoptra.org.directory.index.html" @@ -0,0 +1,9 @@ + +Index of /a/sp ace/ü%&/ + + +

Index of /a/sp ace/ü%&/

1 file, 5 B

+ + + +
NameModified (UTC)Size
../
z.txt2026-08-28 06:005 B
Mirror of https://ctan.org/tex-archive/a/sp%20ace/%C3%BC%25%26/. Report a problem to ctan@katoptra.org.
diff --git a/examples/rsync/fixtures/run-pages/want/staging/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/ctan.katoptra.org.directory.index.html index bd94927..2d47cc2 100644 --- a/examples/rsync/fixtures/run-pages/want/staging/ctan.katoptra.org.directory.index.html +++ b/examples/rsync/fixtures/run-pages/want/staging/ctan.katoptra.org.directory.index.html @@ -1,8 +1,10 @@ -Index of / + +Index of / - -

Index of /

+ +

Index of /

2 directories, 1 file, 20 B

+
NameLast modifiedSize
- -
NameModified (UTC)Size
a/
e1/
timestamp2026/08/28 06:00:0020

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

+timestamp2026-08-28 06:0020 B + diff --git a/examples/rsync/fixtures/run-pages/want/staging/e1/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/e1/ctan.katoptra.org.directory.index.html index aaaa22d..592de76 100644 --- a/examples/rsync/fixtures/run-pages/want/staging/e1/ctan.katoptra.org.directory.index.html +++ b/examples/rsync/fixtures/run-pages/want/staging/e1/ctan.katoptra.org.directory.index.html @@ -1,7 +1,9 @@ -Index of /e1/ + +Index of /e1/ - -

Index of /e1/

+ +

Index of /e1/

1 directory

+
NameLast modifiedSize
-
NameModified (UTC)Size
../
e2/

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

+ diff --git a/examples/rsync/fixtures/run-pages/want/staging/e1/e2/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/e1/e2/ctan.katoptra.org.directory.index.html index 3535035..b76d6d7 100644 --- a/examples/rsync/fixtures/run-pages/want/staging/e1/e2/ctan.katoptra.org.directory.index.html +++ b/examples/rsync/fixtures/run-pages/want/staging/e1/e2/ctan.katoptra.org.directory.index.html @@ -1,6 +1,8 @@ -Index of /e1/e2/ + +Index of /e1/e2/ - -

Index of /e1/e2/

+ +

Index of /e1/e2/

This directory is empty.

+
NameLast modifiedSize
-
NameModified (UTC)Size
../

This directory on ctan.org. Sizes in bytes, times as CTAN lists them.

+
Mirror of https://ctan.org/tex-archive/e1/e2/. Report a problem to ctan@katoptra.org.
diff --git a/examples/rsync/render.txt b/examples/rsync/render.txt index a68dc0f..b9445a7 100644 --- a/examples/rsync/render.txt +++ b/examples/rsync/render.txt @@ -283,7 +283,7 @@ esac task: [pages] mkdir -p /work/examples/rsync/staging /work/examples/rsync/.run/slash && find /work/examples/rsync/staging /work/examples/rsync/.run/slash -mindepth 1 -delete && : > /work/examples/rsync/.run/index-real.txt task: [pages] { cat /work/examples/rsync/.run/applied.txt; awk '{ print $0 "\t\t" }' /work/examples/rsync/.run/upstream-dirs.txt; } | LC_ALL=C sort > /work/examples/rsync/.run/page-state.txt -task: [pages] printf '%s\n' 2 example.katoptra.org.directory.index.html 'index.html index.htm' 'A mirror of rsync.example.org/pub/. Report a problem.' '' '' example.katoptra.org | sha256sum | awk '{ print "#pages " substr($1, 1, 16) }' > /work/examples/rsync/.run/index-stamp.txt +task: [pages] printf '%s\n' 2 example.katoptra.org.directory.index.html 'index.html index.htm' '' https://example.org/pub/ '' example.katoptra.org | sha256sum | awk '{ print "#pages " substr($1, 1, 16) }' > /work/examples/rsync/.run/index-stamp.txt task: [pages] awk 'NR == 1 && /^#pages / { next } { print }' /work/examples/rsync/.run/indexed.txt > /work/examples/rsync/.run/index-old.txt task: [pages] { LC_ALL=C comm -3 /work/examples/rsync/.run/index-old.txt /work/examples/rsync/.run/page-state.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }'; head -1 /work/examples/rsync/.run/indexed.txt | cmp -s - /work/examples/rsync/.run/index-stamp.txt || cat /work/examples/rsync/.run/page-state.txt; } | awk -F'\t' '!("" in s) { s[""] = 1; print "" } { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' | LC_ALL=C sort > /work/examples/rsync/.run/index-dirty.txt @@ -300,27 +300,65 @@ task: [pages] test ! -s /work/examples/rsync/.run/index-make.txt || awk -F'\t' - { n = split($1, c, "/"); d = "" for (i = 1; i < n; i++) { if ((d in want) && !((d c[i]) in seen)) { seen[d c[i]] = 1; print d, "d", c[i], "", "" }; d = d c[i] "/" } if (d in want) print d, (c[n] == "" ? "e" : "f"), c[n], $2, $3 }' /work/examples/rsync/.run/index-make.txt /work/examples/rsync/.run/page-state.txt \ -| LC_ALL=C sort -t "$(printf '\t')" -k1,1 -k2,2 -k3,3V | awk -F'\t' -v S="/work/examples/rsync/staging/" -v R="/work/examples/rsync/.run/slash/" -v I="example.katoptra.org.directory.index.html" -v F='A mirror of rsync.example.org/pub/. Report a problem.' \ - -v N='index.html index.htm' -v X="/work/examples/rsync/.run/index-real.txt" 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } +| LC_ALL=C sort -t "$(printf '\t')" -k1,1 -k2,2 -k3,3V | awk -F'\t' -v S="/work/examples/rsync/staging/" -v R="/work/examples/rsync/.run/slash/" -v I="example.katoptra.org.directory.index.html" -v F='' \ + -v N='index.html index.htm' -v X="/work/examples/rsync/.run/index-real.txt" -v U=https://example.org/pub/ \ + -v C='' -v H="example.katoptra.org" 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } - BEGIN { n = split(N, nm, " "); for (i = n; i >= 1; i--) rank[nm[i]] = i } + BEGIN { n = split(N, nm, " "); for (i = n; i >= 1; i--) rank[nm[i]] = i + if (C == "" && (j = index(H, ".")) > 1) C = substr(H, 1, j - 1) "@" substr(H, j + 1) + css = ":root{color-scheme:light dark;--bg:#f6f5f2;--fg:#1c1c28;--mut:#6e6e7e;--line:#e2e0da;--hov:#edebe5;--a:#3b45d9}" + css = css "@media (prefers-color-scheme:dark){:root{--bg:#121218;--fg:#f0f0f5;--mut:#8b8b9c;--line:#262630;--hov:#1a1a22;--a:#8b93ff}}" + css = css "body{margin:0 auto;max-width:60rem;padding:2rem 1rem 3rem;background:var(--bg);color:var(--fg);font:15px/1.45 system-ui,-apple-system,sans-serif}" + css = css "h1{font-size:1.35rem;font-weight:650;margin:0;overflow-wrap:anywhere}p{color:var(--mut);margin:.2rem 0 1.25rem;font-size:.9rem}" + css = css "table{width:100%;border-collapse:collapse;font:13.5px/1.4 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}" + css = css "th{font:600 .7rem/1 system-ui,sans-serif;letter-spacing:.06em;text-transform:uppercase;color:var(--mut);text-align:left;padding:0 .5rem .6rem;white-space:nowrap}" + css = css "td{padding:.4rem .5rem;border-top:1px solid var(--line);vertical-align:baseline}td:first-child{overflow-wrap:anywhere;width:100%}" + css = css "td+td{white-space:nowrap;color:var(--mut);font-variant-numeric:tabular-nums}th:last-child,td:last-child{text-align:right}" + css = css "tr:hover td{background:var(--hov)}a{color:var(--a);text-decoration:none}a:hover{text-decoration:underline}.s a{color:var(--mut)}" + css = css ".k{font-size:.72rem;margin-left:.6rem;padding:.05rem .4rem;border:1px solid var(--line);border-radius:.3rem;color:var(--mut)}" + css = css ".k:hover{color:var(--a);border-color:var(--a);text-decoration:none}" + css = css "footer{margin-top:2rem;padding-top:1rem;border-top:1px solid var(--line);color:var(--mut);font-size:.85rem;overflow-wrap:anywhere}" + css = css "footer a{color:inherit;text-decoration:underline}@media (max-width:36rem){thead{display:none}" + css = css "tr{display:grid;grid-template-columns:1fr auto;border-top:1px solid var(--line)}td{border:0;padding:.1rem .5rem}" + css = css "td:first-child{grid-column:1/-1;padding-top:.45rem}td+td{padding-bottom:.45rem;font-size:.8rem}}" } function esc(s) { gsub(/&/, "\\&", s); gsub(//, "\\>", s); gsub(/"/, "\\"", s); return s } - function open(d) { cur = d; live = 1; real = "" - p = "Index of /" esc(d) "\n" - p = p "\n" - p = p "\n" - p = p "

Index of /" esc(d) "

\n" - if (d != "") p = p "\n" } - function shut( f, b, k, e) { p = p "
NameLast modifiedSize
../
" (F == "" ? "" : "

" sprintf(F, enc(cur)) "

") "\n" - if (real == "") { f = S cur I; printf("%s", p) > f; close(f) } else print cur real "\t" cur I > X - if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/") - f = R k "/" b; printf("%s", p) > f; close(f) } } + function grp(n, s, o) { s = sprintf("%.0f", n); o = "" + while (length(s) > 3) { o = "," substr(s, length(s) - 2) o; s = substr(s, 1, length(s) - 3) } return s o } + function iec(n, u, i) { if (n < 1024) return sprintf("%.0f B", n) + split("KiB MiB GiB TiB", u, " "); n /= 1024; i = 1; while (n >= 1024 && i < 4) { n /= 1024; i++ } + return sprintf(n < 10 ? "%.1f %s" : "%.0f %s", n, u[i]) } + function ymd(t) { gsub(/\//, "-", t); return substr(t, 1, 16) } + function foot( t) { if (F != "") return "
" sprintf(F, enc(cur)) "
" + t = (U == "" ? "" : "Mirror of " esc(U) enc(cur) ".") + if (C != "") t = t (t == "" ? "" : " ") "Report a problem to " esc(C) "." + return (t == "" ? "" : "
" t "
") } + function open(d) { cur = d; live = 1; real = ""; nd = 0; nfl = 0; tot = 0; split("", has) } + function w(s) { if (real == "") printf("%s", s) > fa; if (cur != "") printf("%s", s) > fb } + function shut( i, r, x, c, e, k, b) { split("", fold); split("", links) + for (i = 1; i <= nfl; i++) { r = fname[i]; x = "" + while (match(r, /\.(sig|asc|sign|sha256|sha512)$/)) { r = substr(r, 1, RSTART - 1); if (r in has) x = r } + if (x != "") { fold[i] = 1 + links[x] = links[x] "" esc(substr(fname[i], length(x) + 1)) "" } } + c = (nd ? grp(nd) (nd == 1 ? " directory" : " directories") : "") + if (nfl) c = c (c == "" ? "" : ", ") grp(nfl) (nfl == 1 ? " file" : " files") ", " iec(tot) + if (c == "") c = "This directory is empty." + fa = S cur I; if (real != "") print cur real "\t" cur I > X + if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/"); fb = R k "/" b } + w("\nIndex of /" esc(cur) "\n") + w("\n\n

Index of /" esc(cur) "

" c "

\n") + w("\n") + if (cur != "") w("\n") + for (i = 1; i <= nd; i++) w("\n") + for (i = 1; i <= nfl; i++) if (!(i in fold)) + w("\n") + w("
NameModified (UTC)Size
../
" esc(dname[i]) "/
" esc(fname[i]) "" links[fname[i]] "" ymd(ftime[i]) "" iec(fsize[i]) "
" foot() "\n") + if (real == "") close(fa); if (cur != "") close(fb) } !live || $1 != cur { if (live) shut(); open($1) } $2 == "e" { next } - $2 == "d" { p = p "" esc($3) "/\n"; next } + $2 == "d" { dname[++nd] = $3; next } ($3 in rank) && (real == "" || rank[$3] < rank[real]) { real = $3 } - { p = p "" esc($3) "" $5 "" $4 "\n" } + { fname[++nfl] = $3; fsize[nfl] = $4; ftime[nfl] = $5; has[$3] = 1; tot += $4 } END { if (live) shut() }' task: [index] test ! -s /work/examples/rsync/.run/index-make.txt || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' /work/examples/rsync/staging/ s3://example/ task: [index] for d in /work/examples/rsync/.run/slash/*/; do test -d "$d" || continue; aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' "$d" s3://example/; done From 5e13501d24801dba791c6f3cc95c29b332da6971 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Mon, 19 Oct 2026 18:09:12 -0700 Subject: [PATCH 4/5] docs(rsync): directory pages, UPSTREAM and CONTACT --- CLAUDE.md | 10 ++++++- README.md | 55 ++++++++++++++++++++++++++++--------- engines/rsync.yml | 19 ++++++++----- examples/rsync/Taskfile.yml | 4 +-- 4 files changed, 65 insertions(+), 23 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 13b9588..ded1f39 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -112,6 +112,13 @@ Read that section before you write. - Give `aws s3api copy-object --copy-source` the raw key. The CLI encodes it. A key that is encoded first is encoded two times (`a%20b` goes out as `a%2520b`), and the copy then reads a different key (verified with `aws --debug`, aws-cli 2.36.24). +- Increase `PAGE_FORMAT` in `engines/rsync.yml` at each change to the HTML or the style of + a directory page. If you do not, each mirror keeps the pages of the directories that do + not change, and they show the old format. The stamp in `.state/indexed.txt.xz` includes + `PAGE_FORMAT`. A new value makes each mirror write all of its pages again, one time. +- The page awk of `pages` writes each page directly to its two files. Do not make the page + with a string that gets a row at a time. That method copies the full string for each + row. The `archive/` directory of TeX Live has approximately 34k rows. - In a check, give `RUN` to a verb as a call var, not as `RUN=...` on the command line. The toolbox sets `RUN` as a global var, and go-task uses the command-line value in some runs only (verified, 3.53.1). For this cause, `offline` starts `copy-cli`, which gives `RUN` @@ -134,7 +141,8 @@ The rsync check runs on `examples/rsync/fixtures/`: - The list diff on `run-root` and `run-empty` - The exit codes of `retry` - `pages` on `run-pages`. Its `want/` is the page set of ctan, and the result must agree - with it byte for byte. + with it byte for byte. Then the stamp on `run-pages-stamp`, and the time of a page with + 10,000 files on `run-pages-big`. - The page read-back of `smoke` on `run-smoke` - `prepare` and `verify` on `tree/`, a signed subtree. A test key, which the example pins, signs its tlpdb. diff --git a/README.md b/README.md index 052711c..64e38c7 100644 --- a/README.md +++ b/README.md @@ -349,7 +349,7 @@ The verbs with a broken line are hooks. These hooks do no work until a mirror se | `checkpoint` | Runs `merge` to add the uploaded files to the state. Then it uploads the state as one PutObject, and it deletes the files in staging | | `delete` | Removes the keys in `deleted.txt` from the bucket, 1,000 in each call, and removes them from the state. It waits for the run that uploads the last batch | | `reconcile` | If `due` wrote `.run/reconcile`, it runs `rebuild`. Then it deletes each key that is not a path of upstream, a key in `OWN`, or a directory of the state. Then it runs `reconciled` | -| `index` | A hook. If `INDEX` has a value: it runs `pages`, uploads the two sets of keys, runs `index-copy`, which copies the index files of `.run/index-real.txt` to their page keys, and removes the keys of the directories that became empty. Then it moves `.state/indexed.txt.xz` to the new state. A mirror with a different landing page overrides `index` | +| `index` | A hook. If `INDEX` has a value: it runs `pages`, uploads the two sets of keys, runs `index-copy`, which copies the index files of `.run/index-real.txt` to their page keys, and removes the keys of the directories that upstream removed. Then it writes the stamp and the page state to `.state/indexed.txt.xz`. A mirror with a different landing page overrides `index` | | `pages` | Writes a page for each directory that the run changed, from the state, into staging. The keys with no slash at the end go in one tree for each depth. A directory with a file in `INDEX_FILES` gets only the key with no slash, and a line in `.run/index-real.txt` | | `smoke` | Reads a sample back from `HOST`. The list below the table gives each item of the sample. Then it runs `fresh` and `smoke-mirror` | | `fresh` | If `FRESH_KEY` has a value: it reads the clock of upstream in that file, from `HOST`. If the clock is more than `FRESH_HOURS` before the time of the run, it stops the run | @@ -484,8 +484,9 @@ continue to get the last good copy. After the engine uploads the batch, `smoke` #### Directory pages R2 serves no directory listings. If `INDEX` has a value, `index` writes a page for each -directory that a run changed. It writes the pages from the state, not from upstream, and -each page has two keys: +directory that a run changed. It writes the pages from the page state: the files of the +state and the directories of the upstream listing. Thus, a directory with no file also gets +a page. Each page has two keys: - `/`: a Transform Rule of the zone serves this key for `/dir/`. - ``, with no slash at the end: this key serves `/dir`. A mirror on a file system @@ -498,6 +499,29 @@ depth, and `index` uploads one tree at a time. `index` gives these keys `--content-type 'text/html; charset=utf-8'`. Without it, a key with no suffix gets `binary/octet-stream`, and a browser downloads the page and does not show it. +Each page shows these items: + +- The path, and a line with the number of directories and files and the total size of the + files. A directory with no item shows "This directory is empty." +- A table with the name, the time and the size of each item. The subdirectories are first. + The names are in version sequence (`sort -V`). Thus, `x-2.9` is before `x-2.10`. +- The time is the mtime of upstream, in UTC, to the minute. The size is in B, KiB, MiB, GiB + or TiB. The title of the cell gives the exact number of bytes. +- A signature file or a checksum file (`.sig`, `.asc`, `.sign`, `.sha256` or `.sha512`) + is a link on the row of its file. The link text is the suffix, directly after the name. + Thus, a search of the page for the full name finds it. A file with such a suffix and no + such file has a row of its own. +- The footer. + +The style of the page is in the page. It has a light and a dark color scheme, and a layout +for a narrow screen. The page has no script, no image and no link to a different site, +other than the `mailto:` address in the footer. + +The engine makes the footer from `UPSTREAM` and `CONTACT`: "Mirror of +``. Report a problem to ``." The address of upstream is text, not +a link. GNU tells its mirrors to put no link in a header or a footer, other than a link for +problem reports. If `PAGE_FOOT` has a value, the footer is that HTML. + `INDEX_FILES` gives the names of the index files, in sequence, for example `index.html index.htm`. A directory can contain a file with one of these names. Then: @@ -514,11 +538,13 @@ it. No page goes into the state. `merge` adds to the state only the files that are in staging and also in the batch, and a page is in no batch. -`.state/indexed.txt.xz` records the state that the pages showed last. If a run stops before -it moves that file, the next run writes the same pages again. If you delete the file, the -next run writes all the pages again, one time. Thus, the pages of the directories that did -not change also get a change to the HTML of the page or to `PAGE_FOOT`. `reconcile` does -not delete the two keys. +`.state/indexed.txt.xz` records the page state that the pages showed last, after a stamp +line. The stamp is a hash of `PAGE_FORMAT` and of the vars that change a page: `INDEX`, +`INDEX_FILES`, `PAGE_FOOT`, `UPSTREAM`, `CONTACT` and `HOST`. When the stamp does not +agree, the run writes all the pages again, one time. Thus, a change to one of these vars, +or a release with a new `PAGE_FORMAT`, gets to each page. If a run stops before it writes +that file, the next run writes the same pages again. `reconcile` does not delete the two +keys of a directory of the page state. The rsync mirrors of katoptra use one `INDEX`, `katoptra.org.directory.index.html`. Thus, the Transform Rules of the zone for their hosts use one rewrite: @@ -585,9 +611,11 @@ inline default, and a mirror sets only the vars that are different: | `TL`, `TL_KEY` | Empty | A signed TeX Live subtree, and the fingerprint of the key that signs it. If they are empty, the engine does no signature checks | | `FILTER` | Empty | rsync filter arguments that make the listing smaller, for a mirror of a subtree | | `OWN` | Empty | Keys at the root of the bucket that the mirror writes, not upstream. A space is between two keys. `reconcile` does not delete them | -| `INDEX` | Empty | The key suffix of the directory pages. If it has a value, `index` writes the pages. `reconcile` does not delete the pages, or a key with the name of a directory of the state | +| `INDEX` | Empty | The key suffix of the directory pages. If it has a value, `index` writes the pages. `reconcile` does not delete the pages, or a key with the name of a directory of the page state | | `INDEX_FILES` | Empty | The names of the index files, in sequence. A space is between two names. The page key of a directory with such a file is a copy of the first one. Refer to [Directory pages](#directory-pages) | -| `PAGE_FOOT` | Empty | The HTML at the end of each directory page. `%s` is the encoded path of the directory, and `%%` gives one `%` | +| `UPSTREAM` | Empty | The URL of the upstream tree, with a slash at the end. The footer of each directory page gives it as text, with the path of the directory. If it is empty, the footer does not give upstream | +| `CONTACT` | From `HOST` | The e-mail address for problem reports, in the footer of each directory page. The default is the first label of `HOST`, `@`, and the other labels: `gnu.katoptra.org` gives `gnu@katoptra.org` | +| `PAGE_FOOT` | Empty | HTML for the footer of each directory page, in place of the footer from `UPSTREAM` and `CONTACT`. `%s` is the encoded path of the directory, and `%%` gives one `%` | | `CANARY` | Empty | A file that `smoke` reads from the domain as a Perl client, and compares with the bucket, byte for byte. All files can be the canary. A file with `http://` links (not `https://`) or `mailto:` addresses also finds the HTML rewriters of the zone. The path must have no character that a URL encodes | | `FRESH_KEY` | Empty | A file in which upstream records its clock, as Unix time or as `YYYY-MM-DD-HH-MM` (the format of CTAN). If it has a value, `fresh` stops the run when the clock is more than `FRESH_HOURS` before the time of the run | | `FRESH_HOURS` | 24 | Hours. This value is less than 28 hours: at 28 hours, the mirror checks of GNU and CTAN show a problem | @@ -667,7 +695,8 @@ These procedures repair other problems: reconcile finds that the key is missing, and the subsequent run fetches it again. Do not edit the state manually to get the key back before the next reconcile. An error in the state can make the engine delete or fetch the incorrect files. -- **Write all directory pages again.** Use `aws s3 rm s3:///.state/indexed.txt.xz`. +- **Write all directory pages again.** A change to a var of the stamp does this at the next + run. To do it with no change, use `aws s3 rm s3:///.state/indexed.txt.xz`. The next run writes the page of each directory at its two keys. This procedure is safe: it does not change the files in the bucket. - **A directory URL gives 404.** The page is at its key, with or without the zone rule. Use @@ -905,7 +934,7 @@ engine. |---|---|---| | rsync | Each path of upstream, at the root | The mirror. A public domain serves the bucket | | rsync | `.state/applied.txt.xz` | The state: the keys that the bucket contains, with the length and the mtime of upstream | -| rsync, with `INDEX` | `.state/indexed.txt.xz`, `/`, `` | The state that the directory pages showed last, and the pages with their two keys. With `INDEX_FILES`, `/` can be a copy of an index file | +| rsync, with `INDEX` | `.state/indexed.txt.xz`, `/`, `` | The stamp and the page state that the directory pages showed last, and the pages with their two keys. With `INDEX_FILES`, `/` can be a copy of an index file | | rsync, a key of the mirror | `index.html` | The landing page of tlnet. `reconcile` does not delete it, because it is in `OWN` | | proton | `.state/session.tar.age` | The CLI session, encrypted. This is the only key | | A mirror with its pipeline | `.state/state.sqlite.xz.age`, `.state/history/-