services/nomad/build/buildbot: native aarch64 worker, use remote repo (shadow) only

Author: classabbyamp

services/nomad/build/buildbot.cfg

@@ -10,7 +10,7 @@ from pathlib import Path
 from twisted.internet import defer
 from twisted.python import log
 
-from buildbot.process.results import SUCCESS, SKIPPED
+from buildbot.process.results import SUCCESS
 from buildbot.plugins import util, secrets, reporters, worker, schedulers
 from buildbot.plugins import steps
 
@@ -129,13 +129,19 @@ c['schedulers'].append(schedulers.ForceScheduler(
 
 # ###### BUILDERS
 
-distdir = lambda f: util.Interpolate(f'/workdir/%(prop:name)s/void-packages/{f}')
-bulkdir = lambda f: util.Interpolate(f'/workdir/%(prop:name)s/xbps-bulk/{f}')
+def distdir(f):
+    return util.Interpolate(f'/workdir/%(prop:name)s/void-packages/{f}')
+
+def bulkdir(f):
+    return util.Interpolate(f'/workdir/%(prop:name)s/xbps-bulk/{f}')
+
 hostdir = '/hostdir'
-buildroot = lambda: util.Interpolate(f'/buildroots/%(prop:name)s/buildroot')
-builddir = lambda: util.Interpolate('builddir-%(prop:buildnumber)s')
-do_sync = lambda: util.Interpolate('%(prop:sync)s') == "True"
-hide_skipped = lambda results, _: results == SKIPPED
+
+def buildroot():
+    return util.Interpolate('/buildroots/%(prop:name)s/buildroot')
+
+def builddir():
+    return util.Interpolate('builddir-%(prop:buildnumber)s')
 
 
 factory = util.BuildFactory()
@@ -198,7 +204,7 @@ def make_xbps_bulk_cmd(props):
         '-h', hostdir,
         '-d', distdir(''),
         '-m', buildroot(),
-        '-t',
+        '-t', '-s', '-S',
     ]
     if props.getProperty('cross') == 'True':
         command += ['-a', props.getProperty('target')]
@@ -229,32 +235,21 @@ def build_packages(props):
     return cmds
 
 
-@util.renderer
-def make_prune_cmd(props):
-    return ['bash', '-c',
-            util.Interpolate(f"""
-export XBPS_TARGET_ARCH="%(prop:target)s"
-for repo in / /debug /nonfree /bootstrap; do
-    xbps-rindex -r "{hostdir}/binpkgs/$repo"
-done
-if [ "$XBPS_TARGET_ARCH" = i686 ]; then
-    for repo in /multilib /multilib/nonfree /multilib/bootstrap; do
-        XBPS_TARGET_ARCH=x86_64 xbps-rindex -r "{hostdir}/binpkgs/$repo"
-    done
-fi
-""")]
-
 @util.renderer
 def make_rsync_cmd(props):
-    return ['bash', '-c',
+    return ['bash', '-e', '-c',
             util.Interpolate("""
-rsync -vurk --delete-after --delay-updates \
---filter='+ */' --filter='+ %(prop:target)s-repodata' --filter='+ *.%(prop:target)s.xbps' \
---filter='+ otime' --filter='- .*' --filter='- *' \
+echo "=> Syncing packages to the shadow repository..."
+rsync -vurk --delete-after \
+--filter='+ */' --filter='+ *.%(prop:target)s.xbps' \
+--filter='- .*' --filter='- *' \
 --password-file=/secrets/rsync/password /hostdir/binpkgs/ \
 {{ range nomadService 1 $allocID "build-rsyncd" -}}
-rsync://buildsync-%(prop:worker)s@{{ .Address }}:{{ .Port }}/%(prop:worker)s
-{{ end -}}""")]
+rsync://buildsync@{{ .Address }}:{{ .Port }}/incoming-%(prop:target)s
+{{ end -}}
+echo "=> Cleaning built packages..."
+find /hostdir/binpkgs -name '*.xbps' -o -name '*-repodata' -print -delete
+""")]
 
 
 factory.addStep(GitWithDiff(
@@ -350,18 +345,6 @@ factory.addStep(steps.ShellSequence(
     timeout=14400,
 ))
 
-factory.addStep(steps.ShellCommand(
-    command=make_prune_cmd,
-    name='prune_packages',
-    description='removing obsolete packages',
-    descriptionDone='removed obsolete packages',
-    haltOnFailure=True,
-    logEnviron=False,
-    usePTY=True,
-    workdir='.',
-    timeout=14400,
-))
-
 factory.addStep(steps.ShellCommand(
     command=make_rsync_cmd,
     name='sync_packages',
@@ -371,8 +354,6 @@ factory.addStep(steps.ShellCommand(
     logEnviron=False,
     usePTY=True,
     workdir='.',
-    doStepIf=do_sync(),
-    hideStepIf=hide_skipped,
     timeout=14400,
     decodeRC={
         0: SUCCESS,
@@ -393,8 +374,7 @@ for b in builders:
         'target': targetarch,
         'cross': str(hostarch != targetarch),
         'worker': b['worker'],
-        'sync': str(b['sync']),
-        'bootstrap_args': b.get('bootstrap_args', '-N'),
+        'bootstrap_args': b.get('bootstrap_args', ''),
     }
 
     c['builders'].append(util.BuilderConfig(

services/nomad/build/buildbot.nomad

@@ -107,15 +107,15 @@ EOF
             { name = "aarch64", max-builds = 2 },
           ],
           builders = [
-            { name = "x86_64",       host = "x86_64",                               worker = "glibc",   sync = false                     },
-            { name = "i686",         host = "i686",                                 worker = "glibc",   sync = false                     },
-            { name = "armv7l",       host = "x86_64",      target = "armv7l",       worker = "glibc",   sync = false                     },
-            { name = "armv6l",       host = "x86_64",      target = "armv6l",       worker = "glibc",   sync = false                     },
-            { name = "x86_64-musl",  host = "x86_64-musl",                          worker = "musl",    sync = true                      },
-            { name = "armv7l-musl",  host = "x86_64-musl", target = "armv7l-musl",  worker = "musl",    sync = true                      },
-            { name = "armv6l-musl",  host = "x86_64-musl", target = "armv6l-musl",  worker = "musl",    sync = true                      },
-            { name = "aarch64",      host = "x86_64",      target = "aarch64",      worker = "aarch64", sync = true, bootstrap_args = "" },
-            { name = "aarch64-musl", host = "x86_64-musl", target = "aarch64-musl", worker = "aarch64", sync = true, bootstrap_args = "" },
+            { name = "x86_64",       host = "x86_64",                               worker = "glibc",  },
+            { name = "i686",         host = "i686",                                 worker = "glibc",  },
+            { name = "armv7l",       host = "x86_64",      target = "armv7l",       worker = "glibc",  },
+            { name = "armv6l",       host = "x86_64",      target = "armv6l",       worker = "glibc",  },
+            { name = "x86_64-musl",  host = "x86_64-musl",                          worker = "musl",   },
+            { name = "armv7l-musl",  host = "x86_64-musl", target = "armv7l-musl",  worker = "musl",   },
+            { name = "armv6l-musl",  host = "x86_64-musl", target = "armv6l-musl",  worker = "musl",   },
+            { name = "aarch64",      host = "aarch64",                              worker = "aarch64" },
+            { name = "aarch64-musl", host = "aarch64-musl",                         worker = "aarch64" },
           ],
         })
         destination = "local/workers.json"