services/nomad/build/buildbot: collect diff better to avoid skipping new packages

for some reason, when collapsing build requests, `self.build.allFiles()`
only includes the changes from the `HEAD` commit, which is especially
egregious when we jump to the `HEAD` commit (not the one that's supposed
to be built according to buildbot). By doing the diff ourselves, we can
avoid that pain and get the true list of packages that have changed from
the previous build to now.

This will still miss builds if there was a failure, though. We'd need a
better build orchestrator to fix that, I think.

Author: classabbyamp

services/nomad/build/buildbot.cfg

@@ -8,6 +8,7 @@ import shlex
 from pathlib import Path
 
 from twisted.internet import defer
+from twisted.python import log
 
 from buildbot.process.results import SUCCESS, SKIPPED
 from buildbot.plugins import util, secrets, reporters, worker, schedulers
@@ -133,6 +134,37 @@ hide_skipped = lambda results, _: results == SKIPPED
 factory = util.BuildFactory()
 
 
+class GitWithDiff(steps.Git):
+    """git fetch and checkout, with diff"""
+    @defer.inlineCallbacks
+    def mode_incremental(self):
+        action = yield self._sourcedirIsUpdatable()
+        # if not updatable, do a full checkout
+        if action == "clobber":
+            yield self.clobber()
+            return
+        elif action == "clone":
+            log.msg("No git repo present, making full clone")
+            yield self._fullCloneOrFallback(shallowClone=self.shallow)
+            pkgs = yield self._get_changed_pkgs("HEAD~")
+            self.updateSourceProperty("packages_changed", " ".join(pkgs))
+            return
+
+        old_ref = yield self._dovccmd(["rev-parse", "HEAD"], collectStdout=True)
+        yield self._fetchOrFallback()
+        pkgs = yield self._get_changed_pkgs(old_ref.strip())
+        self.updateSourceProperty("packages_changed", " ".join(pkgs))
+
+    @defer.inlineCallbacks
+    def _get_changed_pkgs(self, old_ref: str):
+        diff = yield self._dovccmd(["diff", "--name-only", old_ref, "HEAD"], collectStdout=True)
+        pkgs = []
+        for pth in diff.splitlines():
+            if pth and pth.startswith("srcpkgs/") and pth.endswith("/template"):
+                pkgs.append(pth.split("/")[-2])
+        return pkgs
+
+
 @util.renderer
 def make_xbps_src_cmd(props, cmd):
     command = [
@@ -166,21 +198,11 @@ def make_xbps_bulk_cmd(props):
     else:
         command += ['-a', 'native-' + props.getProperty('host')]
     command += ['--']
+    command += props.getProperty('packages_changed').split()
 
     return command
 
 
-class ShellCommandWithChanges(steps.ShellCommand):
-    @defer.inlineCallbacks
-    def run(self):
-        cmd = yield self.makeRemoteShellCommand()
-        pkgs = [f.split("/")[-2] for f in self.build.allFiles()
-            if f.startswith("srcpkgs/") and f.endswith("/template")]
-        cmd.command += pkgs
-        yield self.runCommand(cmd)
-        return cmd.results()
-
-
 @util.renderer
 def build_packages(props):
     cmds = []
@@ -226,7 +248,7 @@ rsync://buildsync-%(prop:worker)s@{{ .Address }}:{{ .Port }}/%(prop:worker)s
 {{ end -}}""")]
 
 
-factory.addStep(steps.Git(
+factory.addStep(GitWithDiff(
     repourl='https://github.com/void-linux/void-packages.git',
     mode='incremental',
     workdir=distdir(''),
@@ -274,7 +296,7 @@ factory.addStep(steps.ShellCommand(
     workdir='.',
 ))
 
-factory.addStep(ShellCommandWithChanges(
+factory.addStep(steps.ShellCommand(
     command=make_xbps_bulk_cmd,
     name='find_packages',
     description='finding packages to build',