services/nomad/build/build-rsyncd: reorganise sync architecture

now packages get synced to an incoming directory. a script runs after
each rsync to move and register the packages into the repo

Author: classabbyamp

services/nomad/build/build-rsyncd.nomad

@@ -19,6 +19,18 @@ job "build-rsyncd" {
       read_only = false
     }
 
+    volume "root_mirror" {
+      type = "host"
+      source = "root_mirror"
+      read_only = false
+    }
+
+    volume "incoming_pkgs" {
+      type = "host"
+      source = "incoming_pkgs"
+      read_only = false
+    }
+
     service {
       provider = "nomad"
       name = "build-rsyncd"
@@ -29,8 +41,7 @@ job "build-rsyncd" {
       driver = "docker"
 
       config {
-        image = "ghcr.io/void-linux/infra-rsync:20240709R1"
-        volumes = [ "local/buildsync.conf:/etc/rsyncd.conf.d/buildsync.conf" ]
+        image = "ghcr.io/void-linux/infra-rsync:20251102R1"
       }
 
       resources {
@@ -43,17 +54,26 @@ job "build-rsyncd" {
         destination = "/hostdir"
       }
 
+      volume_mount {
+        volume = "root_mirror"
+        destination = "/mirror"
+      }
+
+      volume_mount {
+        volume = "incoming_pkgs"
+        destination = "/incoming"
+      }
+
       template {
-        data = file("xbps-clean-sigs")
-        destination = "local/xbps-clean-sigs"
+        data = file("rsync-post-xfer")
+        destination = "local/rsync-post-xfer"
         perms = "0755"
       }
 
       template {
         data = <<EOF
 {{- with nomadVar "nomad/jobs/buildsync" }}
-buildsync-aarch64:{{ .aarch64_password }}
-buildsync-musl:{{ .musl_password }}
+buildsync:{{ .password }}
 {{- end }}
 EOF
         destination = "secrets/buildsync.secrets"
@@ -75,21 +95,30 @@ incoming chmod = D0755,F0644
 [sources]
 path = /hostdir/sources
 filter = - by_sha256/ - .* - *.part
-auth users = buildsync-*:rw
-
-[aarch64]
-path = /hostdir/binpkgs/aarch64
-auth users = buildsync-aarch64:rw
-filter = + */ + *-repodata + otime + *.xbps - *.sig - *.sig2 - *-repodata.* - *-stagedata.* - *.x86_64* - x86_64*-repodata - .*
-post-xfer exec = /local/xbps-clean-sigs
-
-[musl]
-path = /hostdir/binpkgs/musl
-auth users = buildsync-musl:rw
-filter = + */ + *-repodata + otime + *.xbps - *.sig - *.sig2 - *-repodata.* - *-stagedata.* - .*
-post-xfer exec = /local/xbps-clean-sigs
+auth users = buildsync:rw
+
+&merge /local/rsyncd.conf.d
+EOF
+        destination = "local/rsyncd.conf.d/buildsync.conf"
+      }
+
+      dynamic "template" {
+        for_each = [
+          "x86_64", "i686", "armv7l", "armv6l",
+          "x86_64-musl", "armv7l-musl", "armv6l-musl",
+          "aarch64", "aarch64-musl",
+        ]
+
+        content {
+          data = <<EOF
+[incoming-${template.value}]
+path = /incoming/${template.value}
+auth users = buildsync:rw
+filter = + */ + *.${template.value}.xbps - *.sig - *.sig2 - *-repodata* - .*
+post-xfer exec = /local/rsync-post-xfer
 EOF
-        destination = "local/buildsync.conf"
+          destination = "local/rsyncd.conf.d/${template.value}.conf.inc"
+        }
       }
     }
   }

services/nomad/build/rsync-post-xfer

@@ -0,0 +1,48 @@
+#!/bin/sh
+
+set -e
+
+TGT_BASE="/mirror/current"
+
+process_repo() {
+    repo="$1"
+    if [ ! -d "$RSYNC_MODULE_PATH/$repo" ] || [ ! -d "$TGT_BASE/$TGT/$repo" ]; then
+        return
+    fi
+    # add to repodata
+    find "$RSYNC_MODULE_PATH"/"$repo" -maxdepth 1 -type f -printf "$TGT_BASE/$TGT/$repo/%f\0" | \
+        xargs -r0 xbps-rindex -a
+    # clean old packages from repodata
+    xbps-rindex -c "$TGT_BASE/$TGT/$repo"
+    # remove old packages
+    xbps-rindex -r "$TGT_BASE/$TGT/$repo"
+    # Remove signatures that don't have a corresponding package
+    find "$TGT_BASE/$TGT/$repo" -maxdepth 1 \( -name '*.xbps.sig' -o -name '*.xbps.sig2' \) \
+        -exec sh -c 'for x in "$@"; do [ -e "${x%.sig*}" ] || rm -- $x; done' _ {} +
+}
+
+export XBPS_TARGET_ARCH="${RSYNC_MODULE_NAME#*-}"
+
+case "$XBPS_TARGET_ARCH" in
+    aarch64*) TGT="aarch64" ;;
+    *-musl) TGT="musl" ;;
+    *) TGT="" ;;
+esac
+
+# copy files to repo
+rsync -va "${RSYNC_MODULE_PATH:?}"/ "$TGT_BASE/$TGT/"
+
+for repo in / /bootstrap /debug /nonfree; do
+    process_repo "$repo"
+done
+
+if [ "$XBPS_TARGET_ARCH" = 'i686' ]; then
+    for repo in /multilib /multilib/bootstrap /multilib/nonfree; do
+        process_repo "$repo"
+    done
+fi
+
+# clean up incoming
+rm -r "${RSYNC_MODULE_PATH:?}"/*
+
+exit 0