From 8728ff6cfd072b6f837e04509beed9ebaf75f4b9 Mon Sep 17 00:00:00 2001 From: Marco Lecheler Date: Sun, 8 Nov 2026 14:50:04 +0100 Subject: [PATCH] feat(outprint): allow several git sources for packages --- charts/outprint/Chart.yaml | 4 +- charts/outprint/README.md | 24 ++- charts/outprint/files/packages-sync.sh | 159 +++++++++++++------ charts/outprint/templates/_helpers.tpl | 32 +++- charts/outprint/templates/_packages-sync.tpl | 20 +-- charts/outprint/templates/packages-git.yaml | 10 +- charts/outprint/values.yaml | 34 +++- 7 files changed, 209 insertions(+), 74 deletions(-) diff --git a/charts/outprint/Chart.yaml b/charts/outprint/Chart.yaml index d8902ac..11df6cf 100644 --- a/charts/outprint/Chart.yaml +++ b/charts/outprint/Chart.yaml @@ -15,7 +15,7 @@ sources: - https://github.com/taskmedia/helm icon: https://media.task.media/images/logo.png type: application -version: 1.2.0 +version: 1.3.0 # outprint release tag without the leading v appVersion: "1.0.1" @@ -32,6 +32,8 @@ annotations: - name: outprint-injector image: ghcr.io/taskmedia/outprint-injector:1.0.1 artifacthub.io/changes: | + - kind: added + description: packages.sources takes several Git repositories whose packages are merged into one revision - kind: added description: packages.git, packages.image and packages.existingClaim mount your own Template and Configuration packages - kind: added diff --git a/charts/outprint/README.md b/charts/outprint/README.md index fbc080d..31818b4 100644 --- a/charts/outprint/README.md +++ b/charts/outprint/README.md @@ -69,11 +69,27 @@ serviceConfig: configuration: acme ``` -With `git`, a Job (at install and on a change of `url`, `ref`, `commit` or `path`) and a CronJob (every -`packages.git.interval`) resolve `ref` to a commit. On a change they clone it (Git LFS included), run +With `git`, a Job (at install and on a change of a source) and a CronJob (every `packages.git.interval`) +resolve each `ref` to a commit. On a change they clone the commits (Git LFS included), run `outprint validate` against the service configuration and switch over only if it passes; then they restart -Gateway and Worker. A broken revision never replaces a working one. `packages.git.commit` refuses any other -commit. Only the Job has network access; the Worker keeps its NetworkPolicy. The revisions live on a +Gateway and Worker. A broken revision never replaces a working one. `commit` refuses any other commit. + +For packages from more than one repository, list them under `packages.sources` (the shorthand +`packages.git` counts as a source named `default`): + +```yaml +packages: + sources: + - name: letters + git: {url: git@github.com:example/letters.git, ref: v1.0.0, path: outprint, sshKeySecret: letters-key} + - name: reports + git: {url: https://github.com/example/reports.git, ref: main} +``` + +Each source brings `templates/` and/or `configurations/`. The packages of all sources are merged +into one revision that is validated and switched as a whole; a package name that two sources provide stops +the sync and keeps the running revision. `interval`, `image`, `resources`, `persistence` and `keepRevisions` +under `packages.git` apply to all sources. Only the Job has network access; the Worker keeps its NetworkPolicy. The revisions live on a `ReadWriteOnce` PVC, so on a multi-node cluster pin Gateway, Worker and the Job to one node or use a ReadWriteMany storage class. When a change adds a Binding that the running revision does not have yet, the Gateway and the Worker restart until the Job has switched over. diff --git a/charts/outprint/files/packages-sync.sh b/charts/outprint/files/packages-sync.sh index 79969f5..88dd8dc 100644 --- a/charts/outprint/files/packages-sync.sh +++ b/charts/outprint/files/packages-sync.sh @@ -1,71 +1,134 @@ -# Resolve GIT_REF to a commit and, if it differs from the active one, check it -# out (Git LFS included) as /packages/rev-. Leaves /packages/.pending -# for the validate and activate steps. Never touches the active revision. +# Resolve the ref of every source in /scripts/sources.json to a commit. If the +# set of commits differs from the active revision, check every source out (Git +# LFS included), merge the packages into /packages/rev- and leave the id in +# /packages/.pending for the validate and activate steps. Never touches the +# active revision. Any failure (a ref that does not resolve, a pinned commit +# that differs, a package name that two sources provide) stops here. set -eu PK=/packages +SOURCES=/scripts/sources.json rm -f "$PK/.pending" apk add --no-cache git-lfs openssh-client >/dev/null -if [ -f /ssh/ssh-privatekey ]; then - install -m 600 /ssh/ssh-privatekey /tmp/id +count="$(jq length "$SOURCES")" +field() { jq -r ".[$1].$2" "$SOURCES"; } + +# Sets GIT_SSH_COMMAND for the key of source $1, or clears it. +use_key() { + unset GIT_SSH_COMMAND || true + dir="/ssh/$1" + [ -f "$dir/ssh-privatekey" ] || return 0 + install -m 600 "$dir/ssh-privatekey" "/tmp/id-$1" check=accept-new - if [ -f /ssh/known_hosts ]; then - cp /ssh/known_hosts /tmp/known_hosts + if [ -f "$dir/known_hosts" ]; then + cp "$dir/known_hosts" "/tmp/known_hosts-$1" check=yes fi - export GIT_SSH_COMMAND="ssh -i /tmp/id -o IdentitiesOnly=yes -o StrictHostKeyChecking=$check -o UserKnownHostsFile=/tmp/known_hosts" -fi + export GIT_SSH_COMMAND="ssh -i /tmp/id-$1 -o IdentitiesOnly=yes -o StrictHostKeyChecking=$check -o UserKnownHostsFile=/tmp/known_hosts-$1" +} -if printf '%s' "$GIT_REF" | grep -Eq '^[0-9a-f]{40}$'; then - sha="$GIT_REF" -else - sha="" - for candidate in "refs/tags/$GIT_REF^{}" "refs/tags/$GIT_REF" "refs/heads/$GIT_REF"; do - sha="$(git ls-remote "$GIT_URL" "$candidate" | awk 'NR==1 {print $1}')" - [ -n "$sha" ] && break - done -fi -if [ -z "$sha" ]; then - echo "cannot resolve ref '$GIT_REF' in $GIT_URL" >&2 - exit 1 -fi -echo "ref '$GIT_REF' resolves to $sha" +# --- resolve every ref ------------------------------------------------- +resolved="" +i=0 +while [ "$i" -lt "$count" ]; do + name="$(field "$i" name)" + url="$(field "$i" url)" + ref="$(field "$i" ref)" + commit="$(field "$i" commit)" + use_key "$name" + + if printf '%s' "$ref" | grep -Eq '^[0-9a-f]{40}$'; then + sha="$ref" + else + sha="" + for candidate in "refs/tags/$ref^{}" "refs/tags/$ref" "refs/heads/$ref"; do + sha="$(git ls-remote "$url" "$candidate" | awk 'NR==1 {print $1}')" + [ -n "$sha" ] && break + done + fi + if [ -z "$sha" ]; then + echo "$name: cannot resolve ref '$ref' in $url" >&2 + exit 1 + fi + echo "$name: ref '$ref' resolves to $sha" -if [ -n "$GIT_COMMIT" ]; then - case "$sha" in - "$GIT_COMMIT"*) ;; - *) echo "ref resolves to $sha, packages.git.commit expects $GIT_COMMIT: keeping the active revision" >&2; exit 1 ;; - esac + if [ -n "$commit" ]; then + case "$sha" in + "$commit"*) ;; + *) echo "$name: ref resolves to $sha, packages commit expects $commit: keeping the active revision" >&2; exit 1 ;; + esac + fi + resolved="$resolved$name=$sha +" + i=$((i + 1)) +done + +# One source: the commit is the id. Several: a hash over all name=commit pairs. +if [ "$count" -eq 1 ]; then + id="${resolved#*=}" + id="${id%?}" +else + id="$(printf '%s' "$resolved" | sha256sum | cut -c1-16)" fi -if [ "$sha" = "$(cat "$PK/.revision" 2>/dev/null || true)" ] && [ -d "$PK/current/templates" ]; then - echo "revision $sha is already active" +if [ "$id" = "$(cat "$PK/.revision" 2>/dev/null || true)" ] && [ -d "$PK/current/templates" ]; then + echo "revision $id is already active" exit 0 fi -rm -rf "$PK/.work" "$PK/rev-$sha" -mkdir "$PK/.work" -cd "$PK/.work" -git init -q . -git remote add origin "$GIT_URL" -git lfs install --local >/dev/null -git fetch -q --depth 1 origin "$sha" -git checkout -q FETCH_HEAD -git lfs pull +# --- check out and merge ----------------------------------------------- +rev="$PK/rev-$id" +rm -rf "$PK/.work" "$rev" +mkdir -p "$PK/.work" "$rev/templates" "$rev/configurations" +: >"$PK/.work/owners" -src="$PK/.work${PKG_PATH:+/$PKG_PATH}" -for dir in templates configurations; do - if [ ! -d "$src/$dir" ]; then - echo "$src/$dir not found: set packages.git.path to the folder with templates/ and configurations/" >&2 +i=0 +while [ "$i" -lt "$count" ]; do + name="$(field "$i" name)" + url="$(field "$i" url)" + path="$(field "$i" path)" + sha="$(printf '%s' "$resolved" | sed -n "$((i + 1))p" | cut -d= -f2)" + use_key "$name" + + mkdir "$PK/.work/$name" + ( + cd "$PK/.work/$name" + git init -q . + git remote add origin "$url" + git lfs install --local >/dev/null + git fetch -q --depth 1 origin "$sha" + git checkout -q FETCH_HEAD + git lfs pull + ) + + src="$PK/.work/$name${path:+/$path}" + found=false + for kind in templates configurations; do + [ -d "$src/$kind" ] || continue + for pkg in "$src/$kind"/*/; do + [ -d "$pkg" ] || continue + found=true + pkgname="$(basename "$pkg")" + if [ -e "$rev/$kind/$pkgname" ]; then + owner="$(grep "^$kind/$pkgname " "$PK/.work/owners" | cut -d' ' -f2)" + echo "$kind/$pkgname is provided by the sources '$owner' and '$name'; package names must be unique" >&2 + exit 1 + fi + cp -R "$pkg" "$rev/$kind/$pkgname" + echo "$kind/$pkgname $name" >>"$PK/.work/owners" + done + done + if [ "$found" = false ]; then + echo "$name: no package found in $src: set the path of the source to the folder with templates/ and/or configurations/" >&2 exit 1 fi + i=$((i + 1)) done -mkdir "$PK/rev-$sha" -cp -R "$src/templates" "$src/configurations" "$PK/rev-$sha/" -cd "$PK" + +printf '%s' "$resolved" >"$rev/.sources" rm -rf "$PK/.work" -chmod -R a+rX "$PK/rev-$sha" -printf '%s\n' "$sha" >"$PK/.pending" +chmod -R a+rX "$rev" +printf '%s\n' "$id" >"$PK/.pending" chmod a+r "$PK/.pending" diff --git a/charts/outprint/templates/_helpers.tpl b/charts/outprint/templates/_helpers.tpl index 50769e5..5dca3a5 100644 --- a/charts/outprint/templates/_helpers.tpl +++ b/charts/outprint/templates/_helpers.tpl @@ -58,14 +58,42 @@ seccompProfile: type: RuntimeDefault {{- end -}} +{{/* +The Git sources as JSON: the shorthand packages.git (name "default") followed by +packages.sources. Checks names and URLs. +*/}} +{{- define "outprint.packagesSources" -}} +{{- $out := list -}} +{{- $names := dict -}} +{{- if .Values.packages.git.url -}} +{{- $g := .Values.packages.git -}} +{{- $out = append $out (dict "name" "default" "url" $g.url "ref" $g.ref "commit" $g.commit "path" (trimAll "/" $g.path) "sshKeySecret" $g.sshKeySecret) -}} +{{- $_ := set $names "default" true -}} +{{- end -}} +{{- range .Values.packages.sources -}} +{{- $name := required "packages.sources[].name is required" .name -}} +{{- if not (regexMatch "^[a-z0-9][a-z0-9-]*$" $name) -}} +{{- fail (printf "packages.sources: name %q must match ^[a-z0-9][a-z0-9-]*$" $name) -}} +{{- end -}} +{{- if hasKey $names $name -}} +{{- fail (printf "packages.sources: name %q is used twice (the shorthand packages.git is named \"default\")" $name) -}} +{{- end -}} +{{- $_ := set $names $name true -}} +{{- $g := required (printf "packages.sources[%s].git is required" $name) .git -}} +{{- $url := required (printf "packages.sources[%s].git.url is required" $name) $g.url -}} +{{- $out = append $out (dict "name" $name "url" $url "ref" (default "main" $g.ref) "commit" (default "" $g.commit) "path" (trimAll "/" (default "" $g.path)) "sshKeySecret" (default "" $g.sshKeySecret)) -}} +{{- end -}} +{{- toJson $out -}} +{{- end -}} + {{/* The packages source in use: git, image, existingClaim or empty. At most one may be set. */}} {{- define "outprint.packagesSource" -}} {{- $sources := list -}} -{{- if .Values.packages.git.url }}{{ $sources = append $sources "git" }}{{ end -}} +{{- if gt (len (fromJsonArray (include "outprint.packagesSources" .))) 0 }}{{ $sources = append $sources "git" }}{{ end -}} {{- if .Values.packages.image.reference }}{{ $sources = append $sources "image" }}{{ end -}} {{- if .Values.packages.existingClaim }}{{ $sources = append $sources "existingClaim" }}{{ end -}} {{- if gt (len $sources) 1 -}} -{{- fail (printf "packages: set only one source, got %s" (join ", " $sources)) -}} +{{- fail (printf "packages: set only one kind of source, got %s" (join ", " $sources)) -}} {{- end -}} {{- first (append $sources "") -}} {{- end -}} diff --git a/charts/outprint/templates/_packages-sync.tpl b/charts/outprint/templates/_packages-sync.tpl index 6899b50..446b284 100644 --- a/charts/outprint/templates/_packages-sync.tpl +++ b/charts/outprint/templates/_packages-sync.tpl @@ -1,5 +1,6 @@ {{- define "outprint.packagesSyncPod" }} {{- $git := .Values.packages.git }} +{{- $sources := fromJsonArray (include "outprint.packagesSources" .) }} {{- $name := printf "%s-packages" (include "outprint.fullname" .) }} serviceAccountName: {{ $name }} restartPolicy: Never @@ -15,11 +16,6 @@ initContainers: image: {{ printf "%s:%s" $git.image.repository $git.image.tag | quote }} imagePullPolicy: {{ $git.image.pullPolicy }} command: ["sh", "/scripts/sync.sh"] - env: - - { name: GIT_URL, value: {{ $git.url | quote }} } - - { name: GIT_REF, value: {{ $git.ref | quote }} } - - { name: GIT_COMMIT, value: {{ $git.commit | quote }} } - - { name: PKG_PATH, value: {{ trimAll "/" $git.path | quote }} } securityContext: runAsUser: 0 allowPrivilegeEscalation: false @@ -31,8 +27,10 @@ initContainers: volumeMounts: - { name: packages, mountPath: /packages } - { name: scripts, mountPath: /scripts, readOnly: true } - {{- if $git.sshKeySecret }} - - { name: ssh, mountPath: /ssh, readOnly: true } + {{- range $sources }} + {{- if .sshKeySecret }} + - { name: {{ printf "ssh-%s" .name }}, mountPath: {{ printf "/ssh/%s" .name | quote }}, readOnly: true } + {{- end }} {{- end }} - name: validate image: {{ include "outprint.image" (dict "repository" .Values.image.repository "tag" .Values.image.tag "appVersion" .Chart.AppVersion) | quote }} @@ -87,10 +85,12 @@ volumes: name: {{ include "outprint.fullname" . }}-service - name: tmp emptyDir: { sizeLimit: 64Mi } - {{- if $git.sshKeySecret }} - - name: ssh + {{- range $sources }} + {{- if .sshKeySecret }} + - name: {{ printf "ssh-%s" .name }} secret: - secretName: {{ $git.sshKeySecret }} + secretName: {{ .sshKeySecret }} defaultMode: 0400 {{- end }} + {{- end }} {{- end }} diff --git a/charts/outprint/templates/packages-git.yaml b/charts/outprint/templates/packages-git.yaml index 9a3470d..1202845 100644 --- a/charts/outprint/templates/packages-git.yaml +++ b/charts/outprint/templates/packages-git.yaml @@ -69,6 +69,8 @@ metadata: {{- include "outprint.labels" . | nindent 4 }} app.kubernetes.io/component: packages data: + sources.json: | + {{- include "outprint.packagesSources" . | nindent 4 }} sync.sh: | {{- .Files.Get "files/packages-sync.sh" | nindent 4 }} validate.sh: | @@ -100,13 +102,13 @@ spec: spec: {{- include "outprint.packagesSyncPod" . | nindent 10 }} --- -# First sync (and a sync after a change of url, ref, commit or path) without -# waiting for the next CronJob tick. The name carries a hash of those values, -# so a change creates a fresh Job. +# First sync (and a sync after a change of the sources) without waiting for +# the next CronJob tick. The name carries a hash of the sources, so a change +# creates a fresh Job. apiVersion: batch/v1 kind: Job metadata: - name: {{ printf "%s-sync-%s" $name (printf "%s|%s|%s|%s" $git.url $git.ref $git.commit $git.path | sha256sum | trunc 8) }} + name: {{ printf "%s-sync-%s" $name (include "outprint.packagesSources" . | sha256sum | trunc 8) }} labels: {{- include "outprint.labels" . | nindent 4 }} app.kubernetes.io/component: packages diff --git a/charts/outprint/values.yaml b/charts/outprint/values.yaml index 157888e..73186cb 100644 --- a/charts/outprint/values.yaml +++ b/charts/outprint/values.yaml @@ -56,11 +56,16 @@ workerToken: # through OUTPRINT_TEMPLATES_ROOT / OUTPRINT_CONFIGURATIONS_ROOT. Allow the # pairs you want in serviceConfig.bindings. packages: - # Packages in a Git repository. A CronJob (and a hook Job on install and - # upgrade) resolves `ref` to a commit; on a change it clones the commit - # (Git LFS included), runs `outprint validate`, switches over only if the - # check passes, then restarts Gateway and Worker. Only this Job has network; - # the Worker keeps its NetworkPolicy. + # Packages in Git repositories. A CronJob (and a Job on install and on a + # change of the sources) resolves each `ref` to a commit; on a change it + # clones the commits (Git LFS included), merges their packages into one + # revision, runs `outprint validate`, switches over only if the check passes, + # then restarts Gateway and Worker. Only this Job has network; the Worker + # keeps its NetworkPolicy. + # + # `git` below is the shorthand for one repository. For more than one, list + # them under `sources` (see there). The settings `interval`, `image`, + # `resources`, `persistence` and `keepRevisions` of `git` apply to all sources. git: # SSH or HTTPS URL. Empty = off. url: "" @@ -91,6 +96,25 @@ packages: accessMode: ReadWriteOnce # Keep this many old revisions besides the active one. keepRevisions: 2 + # Several Git repositories. The shorthand `git` above, if its `url` is set, + # counts as a source named `default`. Each source brings Template and + # Configuration packages (folders `templates/` and + # `configurations/`, at least one of the two). A package name must be + # unique across all sources; a clash stops the sync and keeps the running + # revision. Names of sources: lower case letters, digits and `-`. + # + # sources: + # - name: letters + # git: + # url: git@github.com:example/letters.git + # ref: v1.0.0 + # path: outprint + # sshKeySecret: letters-deploy-key + # - name: reports + # git: + # url: https://github.com/example/reports.git + # ref: main + sources: [] # Packages in an OCI image (Kubernetes image volume, Kubernetes 1.35+ or the # ImageVolume feature gate). The image holds templates/ and configurations/. image: