From 893b088446cbc6155a7dabf1d36c28992c65eb3c Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sat, 17 Oct 2026 19:22:40 +0200 Subject: [PATCH] fix(chart): a failed pg_dump fails the kates backup job, and backup.enabled without the bundled database fails rendering The kates chart's backup CronJob dumps the bundled PostgreSQL only, and rendered only when backup.enabled and postgresql.enabled were both set. values-prod.yaml set backup.enabled (retention 30, a 50Gi PVC) with externalDatabase, since productionMode refuses the bundled database, so production rendered no CronJob and no PVC while NOTES printed a backup schedule. The README's production example did the same. Where the job did run, as on values-staging.yaml, its script piped pg_dump into gzip without pipefail. A dump that failed (wrong password, database down) still left a valid, empty archive as the newest file, the job printed "Backup complete" and passed, and the prune then deleted the real backups older than the retention. Each OnFailure retry added another empty archive. backup.enabled without postgresql.enabled now fails rendering and says to back the external database up with its own tooling. values-prod.yaml and the README example no longer set it. The script runs with set -eo pipefail and dumps to a .partial file, renamed only once the dump succeeds; each run clears partials a failed try left. A failed dump now fails the job before the prune and leaves no archive a restore would pick. A helm-lint step in ci.yml renders the prod overlay and asserts the combination is refused with this message. Chart 0.11.0: values that rendered before now fail on upgrade. --- .github/workflows/ci.yml | 25 ++++++++++++++++++++++ README.md | 2 +- charts/kates/Chart.yaml | 12 ++++++----- charts/kates/README.md | 3 +-- charts/kates/templates/backup-cronjob.yaml | 13 +++++++++-- charts/kates/values-prod.yaml | 11 ++++------ charts/kates/values.yaml | 4 +++- docs/book/appendix-d-versions.md | 2 +- 8 files changed, 53 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dacc670e..9231d7fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,6 +191,31 @@ jobs: - name: Template validation run: helm template kates charts/kates/ > /dev/null + # The backup CronJob dumps only the bundled PostgreSQL. values-prod.yaml + # set backup.enabled with externalDatabase and rendered no backup, + # silently. Assert the prod overlay renders and that the combination is + # refused. + - name: Assert backup needs the bundled database + run: | + helm template kates charts/kates/ \ + -f charts/kates/values-prod.yaml \ + --set externalDatabase.existingSecret=ci > /dev/null + if out=$(helm template kates charts/kates/ \ + --set backup.enabled=true \ + --set postgresql.enabled=false \ + --set externalDatabase.enabled=true \ + --set externalDatabase.host=db \ + --set externalDatabase.password=x 2>&1); then + echo "::error::backup.enabled with postgresql.enabled=false rendered; it must fail" + exit 1 + fi + # Any other failure would pass the check above without testing it. + if ! grep -q 'backup.enabled: the backup CronJob dumps the bundled PostgreSQL' <<< "$out"; then + echo "::error::the render failed, but not on the backup check: $out" + exit 1 + fi + echo "OK: backup without the bundled database is refused" + # The Strimzi version is declared in five places. If values.yaml's # strimziVersion drifts from the Chart.yaml dependency version, the CRD # hook applies CRDs for a different operator than the one installed — diff --git a/README.md b/README.md index 75cd1c36..7811f203 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ When you install charts with Helm directly: | [`kafka-ui`](charts/kafka-ui/) | 0.3.0 | v1.5.0 | A Helm chart for deploying Kafka UI (Kafbat) with Strimzi SCRAM-SHA-512 authentication | | [`kates-chaos`](charts/kates-chaos/) | 2.2.1 | 3.28.0 | Kates Chaos Engineering — wraps the LitmusChaos execution plane (operator, exporter, CRDs) with Kafka-specific RBAC, experiment/engine templating, and monitoring | | [`kates-platform`](charts/kates-platform/) | 0.7.0 | 1.0.0 | Umbrella chart for the full Kates platform — Kafka, Kates, and supporting infrastructure | -| [`kates`](charts/kates/) | 0.10.11 | 1.25.1 | Kates — Kafka Advanced Testing & Engineering Suite | +| [`kates`](charts/kates/) | 0.11.0 | 1.25.1 | Kates — Kafka Advanced Testing & Engineering Suite | | [`legacy-kafka`](charts/legacy-kafka/) | 0.2.0 | 3.9.1 | A deliberately old Kafka (2.x on ZooKeeper, or 3.x on KRaft) to migrate FROM — the replication source that the pinned Strimzi release cannot deploy | | [`mirror-maker2`](charts/mirror-maker2/) | 0.11.4 | 4.3.1 | Strimzi KafkaMirrorMaker2 — cross-cluster replication, DR, and cross-version migration (2.x/3.x → 4.x) | | [`monitoring`](charts/monitoring/) | 1.6.0 | 82.4.3 | Kates Monitoring — wraps kube-prometheus-stack with Kates-specific dashboards and configuration | diff --git a/charts/kates/Chart.yaml b/charts/kates/Chart.yaml index caf1ec4d..ffb23c05 100644 --- a/charts/kates/Chart.yaml +++ b/charts/kates/Chart.yaml @@ -2,11 +2,11 @@ apiVersion: v2 name: kates description: Kates — Kafka Advanced Testing & Engineering Suite type: application -# Patch: the test run cleanup CronJob deletes runs at last, through the -# API's DELETE /api/tests with the apiKey Secret's key, and cleanup.statuses -# picks which finished runs it deletes. appVersion is unchanged; the job -# takes an image newer than it. -version: 0.10.11 +# Minor: backup.enabled without the bundled PostgreSQL now fails rendering, +# where it rendered no backup and said nothing, and values-prod.yaml drops its +# backup block. A failed pg_dump fails the backup job before the prune and +# leaves no archive. appVersion is unchanged. +version: 0.11.0 # The non-blocking health endpoints this chart's probes now point at come from # a BUILD-TIME property (quarkus.messaging.health.enabled=false), so they only # exist in the image built from this branch. Keep appVersion and @@ -30,6 +30,8 @@ annotations: artifacthub.io/category: integration-delivery artifacthub.io/license: Apache-2.0 artifacthub.io/changes: | + - kind: fixed + description: "ACTION REQUIRED for values that set backup.enabled with externalDatabase — those backups never ran. The backup CronJob dumps the bundled PostgreSQL only and rendered only with postgresql.enabled, so values-prod.yaml's backup block (and the README's production example) rendered no CronJob and no PVC, while NOTES printed a backup schedule. backup.enabled without postgresql.enabled now fails rendering; back the external database up with its own tooling and set backup.enabled=false. values-prod.yaml no longer sets it. A pg_dump that failed also left a valid, empty gzip archive, the job passed, and the prune then deleted the real backups. The script runs with set -eo pipefail and dumps to a .partial file it renames only once the dump succeeds, so a failed dump fails the job before the prune and leaves no archive behind" - kind: changed description: "appVersion 1.25.1 and the image tag move to 1.25.1, the release of the deployment hardening in #368: engine timeout and reaper tunables wired through values and the configmap, stranded runs recovered more robustly, and network policy updates. Templates are otherwise unchanged." - kind: fixed diff --git a/charts/kates/README.md b/charts/kates/README.md index e7907b62..1b8aeaac 100644 --- a/charts/kates/README.md +++ b/charts/kates/README.md @@ -194,7 +194,7 @@ that — `image.tag` does. | Parameter | Default | Description | |-----------|---------|-------------| -| `backup.enabled` | `false` | Enable PostgreSQL backup CronJob | +| `backup.enabled` | `false` | Enable PostgreSQL backup CronJob (bundled PostgreSQL only; rendering fails with `postgresql.enabled=false`) | | `backup.schedule` | `0 2 * * *` | Backup cron schedule | | `backup.retention` | `7` | Days to keep backups | | `backup.persistence.enabled` | `false` | Use PVC instead of emptyDir | @@ -256,7 +256,6 @@ helm install kates ./charts/kates \ --set networkPolicy.enabled=true \ --set metrics.serviceMonitor.enabled=true \ --set metrics.prometheusRule.enabled=true \ - --set backup.enabled=true \ --set cleanup.enabled=true ``` diff --git a/charts/kates/templates/backup-cronjob.yaml b/charts/kates/templates/backup-cronjob.yaml index 425bc8ba..d2e2716a 100644 --- a/charts/kates/templates/backup-cronjob.yaml +++ b/charts/kates/templates/backup-cronjob.yaml @@ -1,4 +1,5 @@ -{{- if and .Values.backup.enabled .Values.postgresql.enabled }} +{{- if .Values.backup.enabled }} +{{- if not .Values.postgresql.enabled }}{{- fail "backup.enabled: the backup CronJob dumps the bundled PostgreSQL (postgresql.enabled) only, so with externalDatabase no backup would run. Back the external database up with its own tooling and set backup.enabled=false." }}{{- end }} apiVersion: batch/v1 kind: CronJob metadata: @@ -31,14 +32,22 @@ spec: - sh - -c - | + # Without pipefail a failed pg_dump still left gzip's valid, + # empty archive, the job passed, and the prune below deleted + # the real backups. The dump is written as .partial and named + # only once it succeeds, so a failed try (OnFailure retries in + # the same pod) leaves no archive a restore would pick. + set -eo pipefail TIMESTAMP=$(date +%Y%m%d-%H%M%S) FILENAME="/backups/kates-${TIMESTAMP}.sql.gz" + rm -f /backups/kates-*.sql.gz.partial echo "Starting backup to ${FILENAME}..." pg_dump -h {{ include "kates.postgresql.fullname" . }} \ -U "${PGUSER}" \ -d "${PGDATABASE}" \ --no-password \ - | gzip > "${FILENAME}" + | gzip > "${FILENAME}.partial" + mv "${FILENAME}.partial" "${FILENAME}" echo "Backup complete: $(du -sh ${FILENAME})" echo "Pruning backups older than {{ .Values.backup.retention }} days..." find /backups -name "kates-*.sql.gz" -mtime +{{ .Values.backup.retention }} -delete diff --git a/charts/kates/values-prod.yaml b/charts/kates/values-prod.yaml index 10413b6d..7a20dd40 100644 --- a/charts/kates/values-prod.yaml +++ b/charts/kates/values-prod.yaml @@ -73,13 +73,10 @@ externalDatabase: # your secrets operator: # existingSecret: "kates-db-credentials" -backup: - enabled: true - schedule: "0 1 * * *" - retention: 30 - persistence: - enabled: true - size: 50Gi +# No backup block: the chart's backup dumps only the bundled PostgreSQL, which +# productionMode refuses. Back the external database up with its own tooling +# (CloudNativePG backups, a managed service's point-in-time recovery); see the +# externalDatabase comment in values.yaml. cleanup: enabled: true diff --git a/charts/kates/values.yaml b/charts/kates/values.yaml index 1554ec55..a7deb651 100644 --- a/charts/kates/values.yaml +++ b/charts/kates/values.yaml @@ -674,7 +674,9 @@ tests: # -- PostgreSQL backup CronJob backup: - # -- Enable backup CronJob + # -- Enable backup CronJob. It dumps the bundled PostgreSQL only, so + # rendering fails with postgresql.enabled=false: back an external database + # up with its own tooling. enabled: false # -- Cron schedule expression schedule: "0 2 * * *" diff --git a/docs/book/appendix-d-versions.md b/docs/book/appendix-d-versions.md index 37f3f255..3d70f79c 100644 --- a/docs/book/appendix-d-versions.md +++ b/docs/book/appendix-d-versions.md @@ -22,7 +22,7 @@ Every version the platform pins, in one place. This table is generated from the | `kafka-ui` chart | 0.3.0 (app v1.5.0) | `charts/kafka-ui/Chart.yaml` | | `kates-chaos` chart | 2.2.1 (app 3.28.0) | `charts/kates-chaos/Chart.yaml` | | `kates-platform` chart | 0.7.0 (app 1.0.0) | `charts/kates-platform/Chart.yaml` | -| `kates` chart | 0.10.11 (app 1.25.1) | `charts/kates/Chart.yaml` | +| `kates` chart | 0.11.0 (app 1.25.1) | `charts/kates/Chart.yaml` | | `legacy-kafka` chart | 0.2.0 (app 3.9.1) | `charts/legacy-kafka/Chart.yaml` | | `mirror-maker2` chart | 0.11.4 (app 4.3.1) | `charts/mirror-maker2/Chart.yaml` | | `monitoring` chart | 1.6.0 (app 82.4.3) | `charts/monitoring/Chart.yaml` |