Merge pull request #573 from ValgulNecron/fix/018-backup-job-limits-values

feat(operator,chart): make backup/restore Job retry and deadline limits configurable

Author: ValgulNecron

CHANGELOG.md

@@ -49,6 +49,7 @@ reaches `1.0.0`. Pre-1.0 minor versions may contain breaking changes.
 
 ### Added
 
+- **Configurable backup/restore Job limits:** the restic backup and restore Jobs' retry limit and deadline are now Helm values (`operator.backupJobBackoffLimit`, default `2`; `operator.backupJobActiveDeadlineSeconds`, default `86400`), passed to the operator as `--backup-job-backoff-limit` / `--backup-job-active-deadline-seconds`.
 - **Multicluster gateways and unified server management:** an optional
   private mTLS agent gateway (with its own Helm deployment) lets one dashboard
   and API manage authorized game servers across independent clusters.

charts/gameplane/templates/operator.yaml

@@ -282,6 +282,31 @@ spec:
             {{- if .Values.operator.resticImage }}
             - --restic-image={{ .Values.operator.resticImage }}
             {{- end }}
+            {{- /* Omit the flags when the keys are absent/null (e.g. --reuse-values
+                   from a release predating them) so the operator's defaults
+                   (2 / 86400) apply. Not `| default`: an explicit 0 backoff is
+                   valid and must be honoured; a 0 deadline falls back. */}}
+            {{- /* Validate before `| int64`: it turns any non-integer (e.g. "abc",
+                   1.5) into 0 instead of failing. YAML numbers arrive as
+                   float64 (1000000 prints as 1e+06), so check by kind. */}}
+            {{- $backoff := .Values.operator.backupJobBackoffLimit }}
+            {{- if not (kindIs "invalid" $backoff) }}
+            {{- $whole := or (and (kindIs "float64" $backoff) (eq ($backoff | int64 | float64) $backoff)) (kindIs "int" $backoff) (kindIs "int64" $backoff) (and (kindIs "string" $backoff) (regexMatch "^[0-9]+$" $backoff)) }}
+            {{- if or (not $whole) (lt ($backoff | int64) 0) }}
+            {{- fail "operator.backupJobBackoffLimit must be an integer >= 0" }}
+            {{- end }}
+            - --backup-job-backoff-limit={{ $backoff | int64 }}
+            {{- end }}
+            {{- $deadline := .Values.operator.backupJobActiveDeadlineSeconds }}
+            {{- if $deadline }}
+            {{- $whole := or (and (kindIs "float64" $deadline) (eq ($deadline | int64 | float64) $deadline)) (kindIs "int" $deadline) (kindIs "int64" $deadline) (and (kindIs "string" $deadline) (regexMatch "^[0-9]+$" $deadline)) }}
+            {{- if or (not $whole) (lt ($deadline | int64) 0) }}
+            {{- fail "operator.backupJobActiveDeadlineSeconds must be an integer > 0" }}
+            {{- end }}
+            {{- if gt ($deadline | int64) 0 }}
+            - --backup-job-active-deadline-seconds={{ $deadline | int64 }}
+            {{- end }}
+            {{- end }}
             - --agent-ca-bundle=/etc/gameplane/agent-ca/ca.crt
             - --agent-client-cert=/etc/gameplane/agent-client/tls.crt
             - --agent-client-key=/etc/gameplane/agent-client/tls.key

charts/gameplane/values.yaml

@@ -94,6 +94,12 @@ operator:
   # private registry mirror for air-gapped installs (Docker Hub blocked).
   configInitImage: busybox:1.38.0
   resticImage: restic/restic:0.19.1
+  # Bounds for the restic backup and restore Jobs the operator creates.
+  # backupJobBackoffLimit: retries before a Job is marked Failed (>= 0).
+  # backupJobActiveDeadlineSeconds: wall-clock cap per Job (> 0), so a stuck
+  # pod cannot hold a Backup/Restore in flight forever. Defaults: 2 and 86400 (24h).
+  backupJobBackoffLimit: 2
+  backupJobActiveDeadlineSeconds: 86400
   # Mount a directory of module bundles into the operator so local-type
   # ModuleSources can index it (homelab: drop module dirs on disk and
   # they appear in the catalog). Provide exactly one of existingClaim

docs/install.md

@@ -113,6 +113,7 @@ Top-level knobs (see `values.yaml` for the full list):
   upstream pins; retag them to a private registry mirror for air-gapped clusters
   where Docker Hub is unreachable. They map to the operator's
   `--config-init-image` / `--restic-image` flags, mirroring `operator.agentImage`
+- `operator.backupJobBackoffLimit` / `operator.backupJobActiveDeadlineSeconds` — retry limit (default `2`, must be >= 0) and wall-clock deadline in seconds (default `86400`, must be > 0) for the restic backup/restore Jobs. They map to the operator's `--backup-job-backoff-limit` / `--backup-job-active-deadline-seconds` flags. Negative or non-integer values fail the render; an unset key (e.g. `helm upgrade --reuse-values` from an older release) or a `0` deadline uses the default, while an explicit `0` backoff is honoured.
 - `operator.gameDataStorage.storageClassName` — install-time default storage class
   for game server data volumes (unreleased; ships in the next release) (default `""`). Empty string uses the cluster's
   default StorageClass. Applies to all GameServers where neither the GameTemplate

operator/cmd/main.go

@@ -148,6 +148,8 @@ func main() {
 		captureSidecarImage              string
 		captureDefaultMaxDurationSeconds int64
 		captureDefaultMaxSizeBytes       int64
+		backupJobBackoffLimit            int64
+		backupJobActiveDeadlineSeconds   int64
 	)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "Address the metrics endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "Address the probe endpoint binds to.")
@@ -167,6 +169,10 @@ func main() {
 	flag.StringVar(&resticImage, "restic-image", controller.DefaultResticImage,
 		"Image for the restic backup/restore Jobs. "+
 			"Point at a private registry mirror for air-gapped installs.")
+	flag.Int64Var(&backupJobBackoffLimit, "backup-job-backoff-limit", int64(controller.DefaultBackupJobBackoffLimit),
+		"BackoffLimit for the restic backup and restore Jobs (retries before the Job is marked Failed). Must be >= 0. Defaults to 2.")
+	flag.Int64Var(&backupJobActiveDeadlineSeconds, "backup-job-active-deadline-seconds", controller.DefaultBackupJobActiveDeadlineSeconds,
+		"ActiveDeadlineSeconds for the restic backup and restore Jobs. Must be > 0. Defaults to 86400 (24 hours).")
 	flag.StringVar(&sentinelImage, "sentinel-image", controller.DefaultSentinelImage,
 		"Image for the wake sentinel pod that holds advertised ports while a server is asleep. "+
 			"Point at a private registry mirror for air-gapped installs.")
@@ -257,6 +263,16 @@ func main() {
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 	setupLog := ctrl.Log.WithName("setup")
 
+	backupJobBackoff32, err := boundedInt32(backupJobBackoffLimit) // rejects <0 and >MaxInt32
+	if err != nil {
+		setupLog.Error(err, "invalid --backup-job-backoff-limit value (must be >= 0)", "value", backupJobBackoffLimit)
+		os.Exit(1)
+	}
+	if backupJobActiveDeadlineSeconds <= 0 {
+		setupLog.Error(nil, "invalid --backup-job-active-deadline-seconds value (must be > 0)", "value", backupJobActiveDeadlineSeconds)
+		os.Exit(1)
+	}
+
 	// Validate capture retention flags while they are still the int64 type
 	// flag.Int64Var naturally produces. The minimum of 60 seconds aligns with
 	// the CRD's own validation. The maximum of 604800 (7 days) is the
@@ -398,12 +414,14 @@ func main() {
 	}
 
 	if err := (&controller.BackupReconciler{
-		Client:        mgr.GetClient(),
-		Scheme:        mgr.GetScheme(),
-		Clientset:     kubernetes.NewForConfigOrDie(mgr.GetConfig()),
-		AgentClient:   agentClient,
-		ResticImage:   resticImage,
-		EventRecorder: mgr.GetEventRecorder("backup-controller"),
+		Client:                   mgr.GetClient(),
+		Scheme:                   mgr.GetScheme(),
+		Clientset:                kubernetes.NewForConfigOrDie(mgr.GetConfig()),
+		AgentClient:              agentClient,
+		ResticImage:              resticImage,
+		JobBackoffLimit:          &backupJobBackoff32,
+		JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds,
+		EventRecorder:            mgr.GetEventRecorder("backup-controller"),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to set up controller", "controller", "Backup")
 		os.Exit(1)
@@ -416,10 +434,12 @@ func main() {
 		os.Exit(1)
 	}
 	if err := (&controller.RestoreReconciler{
-		Client:      mgr.GetClient(),
-		APIReader:   mgr.GetAPIReader(),
-		Scheme:      mgr.GetScheme(),
-		ResticImage: resticImage,
+		Client:                   mgr.GetClient(),
+		APIReader:                mgr.GetAPIReader(),
+		Scheme:                   mgr.GetScheme(),
+		ResticImage:              resticImage,
+		JobBackoffLimit:          &backupJobBackoff32,
+		JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to set up controller", "controller", "Restore")
 		os.Exit(1)

operator/internal/controller/backup_controller.go

@@ -50,6 +50,29 @@ const (
 // or the pod is restarted.
 const maxUnquiesceFinalizeRetry = 10 * time.Minute
 
+// Default bounds for the restic backup and restore Jobs (spec 018 OD-027).
+// Operators override them via --backup-job-backoff-limit /
+// --backup-job-active-deadline-seconds (Helm operator.backupJobBackoffLimit /
+// operator.backupJobActiveDeadlineSeconds).
+const (
+	DefaultBackupJobBackoffLimit          int32 = 2
+	DefaultBackupJobActiveDeadlineSeconds int64 = 86400 // 24 hours
+)
+
+// resolveJobLimits returns the Job BackoffLimit and ActiveDeadlineSeconds to
+// use: a nil backoff selects the default (0 is a valid explicit value, so
+// nil, not zero, means unset); a non-positive deadline selects the default.
+func resolveJobLimits(backoff *int32, deadline int64) (int32, int64) {
+	b, d := DefaultBackupJobBackoffLimit, DefaultBackupJobActiveDeadlineSeconds
+	if backoff != nil && *backoff >= 0 {
+		b = *backoff
+	}
+	if deadline > 0 {
+		d = deadline
+	}
+	return b, d
+}
+
 // backupRestoreJobLabel/backupRestoreJobValue mark Backup and Restore Job
 // pods so the chart's allow-backup-restore-egress NetworkPolicy (F-215)
 // can select them. Without this label the Job pod carries only the Job
@@ -94,6 +117,11 @@ type BackupReconciler struct {
 	// operator flag so air-gapped installs can point it at a private
 	// registry mirror. Empty falls back to DefaultResticImage.
 	ResticImage string
+	// JobBackoffLimit / JobActiveDeadlineSeconds bound the restic backup Job.
+	// Set from operator flags; nil / zero fall back to
+	// DefaultBackupJobBackoffLimit / DefaultBackupJobActiveDeadlineSeconds.
+	JobBackoffLimit          *int32
+	JobActiveDeadlineSeconds int64
 	// SnapshotScrapeGracePeriod bounds how long after the restic Job
 	// completes the reconciler keeps retrying to read the snapshot id from
 	// the pod logs before failing the Backup. Zero uses
@@ -319,8 +347,7 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr
 		return ctrl.Result{}, err
 	}
 
-	backoff := int32(2)
-	deadline := int64(86400) // 24 hours
+	backoff, deadline := resolveJobLimits(r.JobBackoffLimit, r.JobActiveDeadlineSeconds)
 	job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace}}
 	if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, job, func() error {
 		if job.CreationTimestamp.IsZero() {

operator/internal/controller/job_limits_test.go

@@ -0,0 +1,27 @@
+package controller
+
+import "testing"
+
+func TestResolveJobLimits(t *testing.T) {
+	zero, five, neg := int32(0), int32(5), int32(-1)
+	cases := []struct {
+		name  string
+		b     *int32
+		d     int64
+		wantB int32
+		wantD int64
+	}{
+		{"unset uses defaults", nil, 0, 2, 86400},
+		{"override both", &five, 3600, 5, 3600},
+		{"explicit zero backoff honoured", &zero, 0, 0, 86400},
+		{"negative backoff and deadline fall back", &neg, -5, 2, 86400},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			b, d := resolveJobLimits(tc.b, tc.d)
+			if b != tc.wantB || d != tc.wantD {
+				t.Errorf("got (%d,%d), want (%d,%d)", b, d, tc.wantB, tc.wantD)
+			}
+		})
+	}
+}

operator/internal/controller/restore_controller.go

@@ -36,6 +36,11 @@ type RestoreReconciler struct {
 	// operator flag so air-gapped installs can point it at a private
 	// registry mirror. Empty falls back to DefaultResticImage.
 	ResticImage string
+	// JobBackoffLimit / JobActiveDeadlineSeconds bound the restic restore Job.
+	// Set from operator flags; nil / zero fall back to
+	// DefaultBackupJobBackoffLimit / DefaultBackupJobActiveDeadlineSeconds.
+	JobBackoffLimit          *int32
+	JobActiveDeadlineSeconds int64
 }
 
 // apiReader returns the uncached reader for live-consistency checks,
@@ -158,8 +163,7 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 
 	// src (the source Backup, with RepoRef for the Job env) was resolved
 	// above before the volume-snapshot branch.
-	backoff := int32(2)
-	deadline := int64(86400) // 24 hours
+	backoff, deadline := resolveJobLimits(r.JobBackoffLimit, r.JobActiveDeadlineSeconds)
 	job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "restore-" + rs.Name, Namespace: rs.Namespace}}
 	_, err := controllerutil.CreateOrUpdate(ctx, r.Client, job, func() error {
 		if job.CreationTimestamp.IsZero() {