feat(outprint): allow several git sources for packages

charts/outprint/Chart.yaml

@@ -15,7 +15,7 @@ sources:
   - https://github.com/taskmedia/helm
 icon: https://media.task.media/images/logo.png
 type: application
-version: 1.2.0
+version: 1.3.0
 # outprint release tag without the leading v
 appVersion: "1.0.1"
 
@@ -32,6 +32,8 @@ annotations:
     - name: outprint-injector
       image: ghcr.io/taskmedia/outprint-injector:1.0.1
   artifacthub.io/changes: |
+    - kind: added
+      description: packages.sources takes several Git repositories whose packages are merged into one revision
     - kind: added
       description: packages.git, packages.image and packages.existingClaim mount your own Template and Configuration packages
     - kind: added

charts/outprint/README.md

@@ -69,11 +69,27 @@ serviceConfig:
     configuration: acme
 ```
 
-With `git`, a Job (at install and on a change of `url`, `ref`, `commit` or `path`) and a CronJob (every
-`packages.git.interval`) resolve `ref` to a commit. On a change they clone it (Git LFS included), run
+With `git`, a Job (at install and on a change of a source) and a CronJob (every `packages.git.interval`)
+resolve each `ref` to a commit. On a change they clone the commits (Git LFS included), run
 `outprint validate` against the service configuration and switch over only if it passes; then they restart
-Gateway and Worker. A broken revision never replaces a working one. `packages.git.commit` refuses any other
-commit. Only the Job has network access; the Worker keeps its NetworkPolicy. The revisions live on a
+Gateway and Worker. A broken revision never replaces a working one. `commit` refuses any other commit.
+
+For packages from more than one repository, list them under `packages.sources` (the shorthand
+`packages.git` counts as a source named `default`):
+
+```yaml
+packages:
+  sources:
+    - name: letters
+      git: {url: git@github.com:example/letters.git, ref: v1.0.0, path: outprint, sshKeySecret: letters-key}
+    - name: reports
+      git: {url: https://github.com/example/reports.git, ref: main}
+```
+
+Each source brings `templates/<name>` and/or `configurations/<name>`. The packages of all sources are merged
+into one revision that is validated and switched as a whole; a package name that two sources provide stops
+the sync and keeps the running revision. `interval`, `image`, `resources`, `persistence` and `keepRevisions`
+under `packages.git` apply to all sources. Only the Job has network access; the Worker keeps its NetworkPolicy. The revisions live on a
 `ReadWriteOnce` PVC, so on a multi-node cluster pin Gateway, Worker and the Job to one node or use a
 ReadWriteMany storage class. When a change adds a Binding that the running revision does not have yet, the
 Gateway and the Worker restart until the Job has switched over.

charts/outprint/files/packages-sync.sh

@@ -1,71 +1,134 @@
-# Resolve GIT_REF to a commit and, if it differs from the active one, check it
-# out (Git LFS included) as /packages/rev-<commit>. Leaves /packages/.pending
-# for the validate and activate steps. Never touches the active revision.
+# Resolve the ref of every source in /scripts/sources.json to a commit. If the
+# set of commits differs from the active revision, check every source out (Git
+# LFS included), merge the packages into /packages/rev-<id> and leave the id in
+# /packages/.pending for the validate and activate steps. Never touches the
+# active revision. Any failure (a ref that does not resolve, a pinned commit
+# that differs, a package name that two sources provide) stops here.
 set -eu
 
 PK=/packages
+SOURCES=/scripts/sources.json
 rm -f "$PK/.pending"
 
 apk add --no-cache git-lfs openssh-client >/dev/null
 
-if [ -f /ssh/ssh-privatekey ]; then
-  install -m 600 /ssh/ssh-privatekey /tmp/id
+count="$(jq length "$SOURCES")"
+field() { jq -r ".[$1].$2" "$SOURCES"; }
+
+# Sets GIT_SSH_COMMAND for the key of source $1, or clears it.
+use_key() {
+  unset GIT_SSH_COMMAND || true
+  dir="/ssh/$1"
+  [ -f "$dir/ssh-privatekey" ] || return 0
+  install -m 600 "$dir/ssh-privatekey" "/tmp/id-$1"
   check=accept-new
-  if [ -f /ssh/known_hosts ]; then
-    cp /ssh/known_hosts /tmp/known_hosts
+  if [ -f "$dir/known_hosts" ]; then
+    cp "$dir/known_hosts" "/tmp/known_hosts-$1"
     check=yes
   fi
-  export GIT_SSH_COMMAND="ssh -i /tmp/id -o IdentitiesOnly=yes -o StrictHostKeyChecking=$check -o UserKnownHostsFile=/tmp/known_hosts"
-fi
+  export GIT_SSH_COMMAND="ssh -i /tmp/id-$1 -o IdentitiesOnly=yes -o StrictHostKeyChecking=$check -o UserKnownHostsFile=/tmp/known_hosts-$1"
+}
 
-if printf '%s' "$GIT_REF" | grep -Eq '^[0-9a-f]{40}$'; then
-  sha="$GIT_REF"
-else
-  sha=""
-  for candidate in "refs/tags/$GIT_REF^{}" "refs/tags/$GIT_REF" "refs/heads/$GIT_REF"; do
-    sha="$(git ls-remote "$GIT_URL" "$candidate" | awk 'NR==1 {print $1}')"
-    [ -n "$sha" ] && break
-  done
-fi
-if [ -z "$sha" ]; then
-  echo "cannot resolve ref '$GIT_REF' in $GIT_URL" >&2
-  exit 1
-fi
-echo "ref '$GIT_REF' resolves to $sha"
+# --- resolve every ref -------------------------------------------------
+resolved=""
+i=0
+while [ "$i" -lt "$count" ]; do
+  name="$(field "$i" name)"
+  url="$(field "$i" url)"
+  ref="$(field "$i" ref)"
+  commit="$(field "$i" commit)"
+  use_key "$name"
+
+  if printf '%s' "$ref" | grep -Eq '^[0-9a-f]{40}$'; then
+    sha="$ref"
+  else
+    sha=""
+    for candidate in "refs/tags/$ref^{}" "refs/tags/$ref" "refs/heads/$ref"; do
+      sha="$(git ls-remote "$url" "$candidate" | awk 'NR==1 {print $1}')"
+      [ -n "$sha" ] && break
+    done
+  fi
+  if [ -z "$sha" ]; then
+    echo "$name: cannot resolve ref '$ref' in $url" >&2
+    exit 1
+  fi
+  echo "$name: ref '$ref' resolves to $sha"
 
-if [ -n "$GIT_COMMIT" ]; then
-  case "$sha" in
-    "$GIT_COMMIT"*) ;;
-    *) echo "ref resolves to $sha, packages.git.commit expects $GIT_COMMIT: keeping the active revision" >&2; exit 1 ;;
-  esac
+  if [ -n "$commit" ]; then
+    case "$sha" in
+      "$commit"*) ;;
+      *) echo "$name: ref resolves to $sha, packages commit expects $commit: keeping the active revision" >&2; exit 1 ;;
+    esac
+  fi
+  resolved="$resolved$name=$sha
+"
+  i=$((i + 1))
+done
+
+# One source: the commit is the id. Several: a hash over all name=commit pairs.
+if [ "$count" -eq 1 ]; then
+  id="${resolved#*=}"
+  id="${id%?}"
+else
+  id="$(printf '%s' "$resolved" | sha256sum | cut -c1-16)"
 fi
 
-if [ "$sha" = "$(cat "$PK/.revision" 2>/dev/null || true)" ] && [ -d "$PK/current/templates" ]; then
-  echo "revision $sha is already active"
+if [ "$id" = "$(cat "$PK/.revision" 2>/dev/null || true)" ] && [ -d "$PK/current/templates" ]; then
+  echo "revision $id is already active"
   exit 0
 fi
 
-rm -rf "$PK/.work" "$PK/rev-$sha"
-mkdir "$PK/.work"
-cd "$PK/.work"
-git init -q .
-git remote add origin "$GIT_URL"
-git lfs install --local >/dev/null
-git fetch -q --depth 1 origin "$sha"
-git checkout -q FETCH_HEAD
-git lfs pull
+# --- check out and merge -----------------------------------------------
+rev="$PK/rev-$id"
+rm -rf "$PK/.work" "$rev"
+mkdir -p "$PK/.work" "$rev/templates" "$rev/configurations"
+: >"$PK/.work/owners"
 
-src="$PK/.work${PKG_PATH:+/$PKG_PATH}"
-for dir in templates configurations; do
-  if [ ! -d "$src/$dir" ]; then
-    echo "$src/$dir not found: set packages.git.path to the folder with templates/ and configurations/" >&2
+i=0
+while [ "$i" -lt "$count" ]; do
+  name="$(field "$i" name)"
+  url="$(field "$i" url)"
+  path="$(field "$i" path)"
+  sha="$(printf '%s' "$resolved" | sed -n "$((i + 1))p" | cut -d= -f2)"
+  use_key "$name"
+
+  mkdir "$PK/.work/$name"
+  (
+    cd "$PK/.work/$name"
+    git init -q .
+    git remote add origin "$url"
+    git lfs install --local >/dev/null
+    git fetch -q --depth 1 origin "$sha"
+    git checkout -q FETCH_HEAD
+    git lfs pull
+  )
+
+  src="$PK/.work/$name${path:+/$path}"
+  found=false
+  for kind in templates configurations; do
+    [ -d "$src/$kind" ] || continue
+    for pkg in "$src/$kind"/*/; do
+      [ -d "$pkg" ] || continue
+      found=true
+      pkgname="$(basename "$pkg")"
+      if [ -e "$rev/$kind/$pkgname" ]; then
+        owner="$(grep "^$kind/$pkgname " "$PK/.work/owners" | cut -d' ' -f2)"
+        echo "$kind/$pkgname is provided by the sources '$owner' and '$name'; package names must be unique" >&2
+        exit 1
+      fi
+      cp -R "$pkg" "$rev/$kind/$pkgname"
+      echo "$kind/$pkgname $name" >>"$PK/.work/owners"
+    done
+  done
+  if [ "$found" = false ]; then
+    echo "$name: no package found in $src: set the path of the source to the folder with templates/ and/or configurations/" >&2
     exit 1
   fi
+  i=$((i + 1))
 done
-mkdir "$PK/rev-$sha"
-cp -R "$src/templates" "$src/configurations" "$PK/rev-$sha/"
-cd "$PK"
+
+printf '%s' "$resolved" >"$rev/.sources"
 rm -rf "$PK/.work"
-chmod -R a+rX "$PK/rev-$sha"
-printf '%s\n' "$sha" >"$PK/.pending"
+chmod -R a+rX "$rev"
+printf '%s\n' "$id" >"$PK/.pending"
 chmod a+r "$PK/.pending"

charts/outprint/templates/_helpers.tpl

@@ -58,14 +58,42 @@ seccompProfile:
   type: RuntimeDefault
 {{- end -}}
 
+{{/*
+The Git sources as JSON: the shorthand packages.git (name "default") followed by
+packages.sources. Checks names and URLs.
+*/}}
+{{- define "outprint.packagesSources" -}}
+{{- $out := list -}}
+{{- $names := dict -}}
+{{- if .Values.packages.git.url -}}
+{{- $g := .Values.packages.git -}}
+{{- $out = append $out (dict "name" "default" "url" $g.url "ref" $g.ref "commit" $g.commit "path" (trimAll "/" $g.path) "sshKeySecret" $g.sshKeySecret) -}}
+{{- $_ := set $names "default" true -}}
+{{- end -}}
+{{- range .Values.packages.sources -}}
+{{- $name := required "packages.sources[].name is required" .name -}}
+{{- if not (regexMatch "^[a-z0-9][a-z0-9-]*$" $name) -}}
+{{- fail (printf "packages.sources: name %q must match ^[a-z0-9][a-z0-9-]*$" $name) -}}
+{{- end -}}
+{{- if hasKey $names $name -}}
+{{- fail (printf "packages.sources: name %q is used twice (the shorthand packages.git is named \"default\")" $name) -}}
+{{- end -}}
+{{- $_ := set $names $name true -}}
+{{- $g := required (printf "packages.sources[%s].git is required" $name) .git -}}
+{{- $url := required (printf "packages.sources[%s].git.url is required" $name) $g.url -}}
+{{- $out = append $out (dict "name" $name "url" $url "ref" (default "main" $g.ref) "commit" (default "" $g.commit) "path" (trimAll "/" (default "" $g.path)) "sshKeySecret" (default "" $g.sshKeySecret)) -}}
+{{- end -}}
+{{- toJson $out -}}
+{{- end -}}
+
 {{/* The packages source in use: git, image, existingClaim or empty. At most one may be set. */}}
 {{- define "outprint.packagesSource" -}}
 {{- $sources := list -}}
-{{- if .Values.packages.git.url }}{{ $sources = append $sources "git" }}{{ end -}}
+{{- if gt (len (fromJsonArray (include "outprint.packagesSources" .))) 0 }}{{ $sources = append $sources "git" }}{{ end -}}
 {{- if .Values.packages.image.reference }}{{ $sources = append $sources "image" }}{{ end -}}
 {{- if .Values.packages.existingClaim }}{{ $sources = append $sources "existingClaim" }}{{ end -}}
 {{- if gt (len $sources) 1 -}}
-{{- fail (printf "packages: set only one source, got %s" (join ", " $sources)) -}}
+{{- fail (printf "packages: set only one kind of source, got %s" (join ", " $sources)) -}}
 {{- end -}}
 {{- first (append $sources "") -}}
 {{- end -}}

charts/outprint/templates/_packages-sync.tpl

@@ -1,5 +1,6 @@
 {{- define "outprint.packagesSyncPod" }}
 {{- $git := .Values.packages.git }}
+{{- $sources := fromJsonArray (include "outprint.packagesSources" .) }}
 {{- $name := printf "%s-packages" (include "outprint.fullname" .) }}
 serviceAccountName: {{ $name }}
 restartPolicy: Never
@@ -15,11 +16,6 @@ initContainers:
     image: {{ printf "%s:%s" $git.image.repository $git.image.tag | quote }}
     imagePullPolicy: {{ $git.image.pullPolicy }}
     command: ["sh", "/scripts/sync.sh"]
-    env:
-      - { name: GIT_URL, value: {{ $git.url | quote }} }
-      - { name: GIT_REF, value: {{ $git.ref | quote }} }
-      - { name: GIT_COMMIT, value: {{ $git.commit | quote }} }
-      - { name: PKG_PATH, value: {{ trimAll "/" $git.path | quote }} }
     securityContext:
       runAsUser: 0
       allowPrivilegeEscalation: false
@@ -31,8 +27,10 @@ initContainers:
     volumeMounts:
       - { name: packages, mountPath: /packages }
       - { name: scripts, mountPath: /scripts, readOnly: true }
-      {{- if $git.sshKeySecret }}
-      - { name: ssh, mountPath: /ssh, readOnly: true }
+      {{- range $sources }}
+      {{- if .sshKeySecret }}
+      - { name: {{ printf "ssh-%s" .name }}, mountPath: {{ printf "/ssh/%s" .name | quote }}, readOnly: true }
+      {{- end }}
       {{- end }}
   - name: validate
     image: {{ include "outprint.image" (dict "repository" .Values.image.repository "tag" .Values.image.tag "appVersion" .Chart.AppVersion) | quote }}
@@ -87,10 +85,12 @@ volumes:
       name: {{ include "outprint.fullname" . }}-service
   - name: tmp
     emptyDir: { sizeLimit: 64Mi }
-  {{- if $git.sshKeySecret }}
-  - name: ssh
+  {{- range $sources }}
+  {{- if .sshKeySecret }}
+  - name: {{ printf "ssh-%s" .name }}
     secret:
-      secretName: {{ $git.sshKeySecret }}
+      secretName: {{ .sshKeySecret }}
       defaultMode: 0400
   {{- end }}
+  {{- end }}
 {{- end }}

charts/outprint/templates/packages-git.yaml

@@ -69,6 +69,8 @@ metadata:
     {{- include "outprint.labels" . | nindent 4 }}
     app.kubernetes.io/component: packages
 data:
+  sources.json: |
+    {{- include "outprint.packagesSources" . | nindent 4 }}
   sync.sh: |
     {{- .Files.Get "files/packages-sync.sh" | nindent 4 }}
   validate.sh: |
@@ -100,13 +102,13 @@ spec:
         spec:
           {{- include "outprint.packagesSyncPod" . | nindent 10 }}
 ---
-# First sync (and a sync after a change of url, ref, commit or path) without
-# waiting for the next CronJob tick. The name carries a hash of those values,
-# so a change creates a fresh Job.
+# First sync (and a sync after a change of the sources) without waiting for
+# the next CronJob tick. The name carries a hash of the sources, so a change
+# creates a fresh Job.
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ printf "%s-sync-%s" $name (printf "%s|%s|%s|%s" $git.url $git.ref $git.commit $git.path | sha256sum | trunc 8) }}
+  name: {{ printf "%s-sync-%s" $name (include "outprint.packagesSources" . | sha256sum | trunc 8) }}
   labels:
     {{- include "outprint.labels" . | nindent 4 }}
     app.kubernetes.io/component: packages