Make Brainstore rollout controls configurable

braintrust/README.md

@@ -289,6 +289,85 @@ Pools use fixed replica counts by default (`api.replicas` and
 `api.workloadIsolation.<pool>.replicas`). On GKE, enable `api.autoscaling` to
 let each pool scale independently instead.
 
+## Brainstore Rollout Controls
+
+Brainstore readers, fast readers, and writers have independently configurable
+Deployment strategies and readiness dwell times. These controls let operators
+limit how many replacement pods start together and require replacements to
+remain Ready before an upgrade continues. This can reduce simultaneous cache
+warm-up, object-storage, scheduling, and compaction pressure in cache-heavy or
+high-throughput deployments.
+
+The defaults preserve the rollout behavior from earlier chart versions:
+
+- `strategy.type: RollingUpdate`
+- `strategy.rollingUpdate.maxSurge: 100%`
+- `strategy.rollingUpdate.maxUnavailable: 0`
+- `minReadySeconds: 0`
+- `progressDeadlineSeconds: 600`
+
+Upgrading the chart without overriding these values does not change rollout
+pacing. This feature also does not change the pod template, so it does not
+restart Brainstore pods by itself. Custom settings take effect the next time a
+pod-template change triggers a rollout.
+
+Cache-heavy or high-throughput deployments can use a longer readiness dwell:
+
+```yaml
+brainstore:
+  writer:
+    minReadySeconds: 300
+    progressDeadlineSeconds: 900
+    strategy:
+      type: RollingUpdate
+      rollingUpdate:
+        maxSurge: 1
+        maxUnavailable: 0
+  reader:
+    minReadySeconds: 120
+    progressDeadlineSeconds: 600
+    strategy:
+      type: RollingUpdate
+      rollingUpdate:
+        maxSurge: 1
+        maxUnavailable: 0
+  fastreader:
+    minReadySeconds: 120
+    progressDeadlineSeconds: 600
+    strategy:
+      type: RollingUpdate
+      rollingUpdate:
+        maxSurge: 1
+        maxUnavailable: 0
+```
+
+The same `strategy`, `minReadySeconds`, and `progressDeadlineSeconds` settings
+are available under each Brainstore role. `progressDeadlineSeconds` must be
+greater than `minReadySeconds`; the chart rejects an invalid pairing. Keep
+enough deadline margin for scheduling, image pulls, startup, and the readiness
+dwell.
+
+A longer dwell reduces rollout pressure but does not prove that a pod's local
+cache is fully warm; monitor workload health until the rollout has converged.
+`maxUnavailable: 0` also requires enough cluster capacity for the configured
+surge.
+
+Slow rollouts extend the period during which old and new Brainstore versions
+run together. Follow version-specific upgrade guidance and do not change
+`brainstoreWalFooterVersion` in the same upgrade as an image version bump,
+except where the documented data plane 2.0 upgrade sequence explicitly permits
+it. See the [data plane 2.0 upgrade guide](https://www.braintrust.dev/docs/admin/self-hosting/upgrade/v2).
+
+Setting `strategy.type: Recreate` stops all pods in that Brainstore role before
+creating replacements. This causes a complete role outage and, with the default
+single writer, pauses background processing until the replacement becomes
+Ready.
+
+These settings pace Deployment-managed rollouts only. The chart does not
+currently create PodDisruptionBudgets for Brainstore, so these controls do not
+limit voluntary disruptions such as node drains or protect against involuntary
+pod or node failures.
+
 ## Testing
 
 This Helm chart includes comprehensive automated unit tests.

braintrust/templates/_helpers.tpl

@@ -9,6 +9,17 @@ Get the namespace to use for resources
 {{- end -}}
 {{- end -}}
 
+{{/*
+Validate Brainstore Deployment rollout timing for one role.
+*/}}
+{{- define "braintrust.brainstoreRollout.validate" -}}
+{{- $minReadySeconds := int .config.minReadySeconds -}}
+{{- $progressDeadlineSeconds := int .config.progressDeadlineSeconds -}}
+{{- if le $progressDeadlineSeconds $minReadySeconds -}}
+{{- fail (printf "brainstore.%s.progressDeadlineSeconds (%d) must be greater than brainstore.%s.minReadySeconds (%d)" .role $progressDeadlineSeconds .role $minReadySeconds) -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Static fast reader query sources used by API.
 */}}

braintrust/templates/brainstore-fastreader-deployment.yaml

@@ -1,3 +1,4 @@
+{{- include "braintrust.brainstoreRollout.validate" (dict "role" "fastreader" "config" .Values.brainstore.fastreader) }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -13,11 +14,14 @@ metadata:
   {{- end }}
 spec:
   replicas: {{ .Values.brainstore.fastreader.replicas }}
+  minReadySeconds: {{ .Values.brainstore.fastreader.minReadySeconds }}
+  progressDeadlineSeconds: {{ .Values.brainstore.fastreader.progressDeadlineSeconds }}
   strategy:
-    type: RollingUpdate
+    type: {{ .Values.brainstore.fastreader.strategy.type }}
+    {{- if eq .Values.brainstore.fastreader.strategy.type "RollingUpdate" }}
     rollingUpdate:
-      maxSurge: 100%
-      maxUnavailable: 0
+      {{- toYaml .Values.brainstore.fastreader.strategy.rollingUpdate | nindent 6 }}
+    {{- end }}
   selector:
     matchLabels:
       app: {{ .Values.brainstore.fastreader.name }}

braintrust/templates/brainstore-reader-deployment.yaml

@@ -1,3 +1,4 @@
+{{- include "braintrust.brainstoreRollout.validate" (dict "role" "reader" "config" .Values.brainstore.reader) }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -13,11 +14,14 @@ metadata:
   {{- end }}
 spec:
   replicas: {{ .Values.brainstore.reader.replicas }}
+  minReadySeconds: {{ .Values.brainstore.reader.minReadySeconds }}
+  progressDeadlineSeconds: {{ .Values.brainstore.reader.progressDeadlineSeconds }}
   strategy:
-    type: RollingUpdate
+    type: {{ .Values.brainstore.reader.strategy.type }}
+    {{- if eq .Values.brainstore.reader.strategy.type "RollingUpdate" }}
     rollingUpdate:
-      maxSurge: 100%
-      maxUnavailable: 0
+      {{- toYaml .Values.brainstore.reader.strategy.rollingUpdate | nindent 6 }}
+    {{- end }}
   selector:
     matchLabels:
       app: {{ .Values.brainstore.reader.name }}

braintrust/templates/brainstore-writer-deployment.yaml

@@ -1,3 +1,4 @@
+{{- include "braintrust.brainstoreRollout.validate" (dict "role" "writer" "config" .Values.brainstore.writer) }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -13,11 +14,14 @@ metadata:
   {{- end }}
 spec:
   replicas: {{ .Values.brainstore.writer.replicas }}
+  minReadySeconds: {{ .Values.brainstore.writer.minReadySeconds }}
+  progressDeadlineSeconds: {{ .Values.brainstore.writer.progressDeadlineSeconds }}
   strategy:
-    type: RollingUpdate
+    type: {{ .Values.brainstore.writer.strategy.type }}
+    {{- if eq .Values.brainstore.writer.strategy.type "RollingUpdate" }}
     rollingUpdate:
-      maxSurge: 100%
-      maxUnavailable: 0
+      {{- toYaml .Values.brainstore.writer.strategy.rollingUpdate | nindent 6 }}
+    {{- end }}
   selector:
     matchLabels:
       app: {{ .Values.brainstore.writer.name }}

braintrust/tests/brainstore-fastreader_test.yaml

@@ -18,6 +18,21 @@ tests:
       - equal:
           path: spec.replicas
           value: 1
+      - equal:
+          path: spec.minReadySeconds
+          value: 0
+      - equal:
+          path: spec.progressDeadlineSeconds
+          value: 600
+      - equal:
+          path: spec.strategy.type
+          value: RollingUpdate
+      - equal:
+          path: spec.strategy.rollingUpdate.maxSurge
+          value: 100%
+      - equal:
+          path: spec.strategy.rollingUpdate.maxUnavailable
+          value: 0
       - equal:
           path: spec.template.spec.containers[0].name
           value: brainstore-fastreader
@@ -28,6 +43,44 @@ tests:
           path: spec.template.spec.containers[0].envFrom[0].configMapRef.name
           value: brainstore-fastreader
 
+  - it: should render custom Brainstore Fast Reader rollout controls
+    template: brainstore-fastreader-deployment.yaml
+    values:
+      - __fixtures__/base-values.yaml
+    set:
+      brainstore.fastreader.minReadySeconds: 120
+      brainstore.fastreader.progressDeadlineSeconds: 900
+      brainstore.fastreader.strategy.rollingUpdate.maxSurge: 2
+      brainstore.fastreader.strategy.rollingUpdate.maxUnavailable: 1
+    release:
+      namespace: "braintrust"
+    asserts:
+      - equal:
+          path: spec.minReadySeconds
+          value: 120
+      - equal:
+          path: spec.progressDeadlineSeconds
+          value: 900
+      - equal:
+          path: spec.strategy.rollingUpdate.maxSurge
+          value: 2
+      - equal:
+          path: spec.strategy.rollingUpdate.maxUnavailable
+          value: 1
+
+  - it: should reject a progress deadline that does not exceed the readiness dwell
+    template: brainstore-fastreader-deployment.yaml
+    values:
+      - __fixtures__/base-values.yaml
+    set:
+      brainstore.fastreader.minReadySeconds: 600
+      brainstore.fastreader.progressDeadlineSeconds: 600
+    release:
+      namespace: "braintrust"
+    asserts:
+      - failedTemplate:
+          errorMessage: "brainstore.fastreader.progressDeadlineSeconds (600) must be greater than brainstore.fastreader.minReadySeconds (600)"
+
   - it: should include extraEnvVars when provided
     template: brainstore-fastreader-deployment.yaml
     values:

braintrust/tests/brainstore-reader_test.yaml

@@ -18,13 +18,66 @@ tests:
       - equal:
           path: spec.replicas
           value: 1
+      - equal:
+          path: spec.minReadySeconds
+          value: 0
+      - equal:
+          path: spec.progressDeadlineSeconds
+          value: 600
+      - equal:
+          path: spec.strategy.type
+          value: RollingUpdate
+      - equal:
+          path: spec.strategy.rollingUpdate.maxSurge
+          value: 100%
+      - equal:
+          path: spec.strategy.rollingUpdate.maxUnavailable
+          value: 0
       - equal:
           path: spec.template.spec.containers[0].name
           value: brainstore-reader
       - equal:
           path: spec.template.spec.containers[0].image
           value: test/brainstore:v1.0.0
 
+  - it: should render custom Brainstore Reader rollout controls
+    template: brainstore-reader-deployment.yaml
+    values:
+      - __fixtures__/base-values.yaml
+    set:
+      brainstore.reader.minReadySeconds: 120
+      brainstore.reader.progressDeadlineSeconds: 900
+      brainstore.reader.strategy.rollingUpdate.maxSurge: 2
+      brainstore.reader.strategy.rollingUpdate.maxUnavailable: 1
+    release:
+      namespace: "braintrust"
+    asserts:
+      - equal:
+          path: spec.minReadySeconds
+          value: 120
+      - equal:
+          path: spec.progressDeadlineSeconds
+          value: 900
+      - equal:
+          path: spec.strategy.rollingUpdate.maxSurge
+          value: 2
+      - equal:
+          path: spec.strategy.rollingUpdate.maxUnavailable
+          value: 1
+
+  - it: should reject a progress deadline that does not exceed the readiness dwell
+    template: brainstore-reader-deployment.yaml
+    values:
+      - __fixtures__/base-values.yaml
+    set:
+      brainstore.reader.minReadySeconds: 600
+      brainstore.reader.progressDeadlineSeconds: 600
+    release:
+      namespace: "braintrust"
+    asserts:
+      - failedTemplate:
+          errorMessage: "brainstore.reader.progressDeadlineSeconds (600) must be greater than brainstore.reader.minReadySeconds (600)"
+
   - it: should include azure workload identity label when cloud is azure
     template: brainstore-reader-deployment.yaml
     values:

braintrust/tests/brainstore-writer_test.yaml

@@ -18,13 +18,81 @@ tests:
       - equal:
           path: spec.replicas
           value: 1
+      - equal:
+          path: spec.minReadySeconds
+          value: 0
+      - equal:
+          path: spec.progressDeadlineSeconds
+          value: 600
+      - equal:
+          path: spec.strategy.type
+          value: RollingUpdate
+      - equal:
+          path: spec.strategy.rollingUpdate.maxSurge
+          value: 100%
+      - equal:
+          path: spec.strategy.rollingUpdate.maxUnavailable
+          value: 0
       - equal:
           path: spec.template.spec.containers[0].name
           value: brainstore-writer
       - equal:
           path: spec.template.spec.containers[0].image
           value: test/brainstore:v1.0.0
 
+  - it: should render custom Brainstore Writer rollout controls
+    template: brainstore-writer-deployment.yaml
+    values:
+      - __fixtures__/base-values.yaml
+    set:
+      brainstore.writer.minReadySeconds: 300
+      brainstore.writer.progressDeadlineSeconds: 900
+      brainstore.writer.strategy.rollingUpdate.maxSurge: 2
+      brainstore.writer.strategy.rollingUpdate.maxUnavailable: 1
+    release:
+      namespace: "braintrust"
+    asserts:
+      - equal:
+          path: spec.minReadySeconds
+          value: 300
+      - equal:
+          path: spec.progressDeadlineSeconds
+          value: 900
+      - equal:
+          path: spec.strategy.rollingUpdate.maxSurge
+          value: 2
+      - equal:
+          path: spec.strategy.rollingUpdate.maxUnavailable
+          value: 1
+
+  - it: should reject a progress deadline that does not exceed the readiness dwell
+    template: brainstore-writer-deployment.yaml
+    values:
+      - __fixtures__/base-values.yaml
+    set:
+      brainstore.writer.minReadySeconds: 600
+      brainstore.writer.progressDeadlineSeconds: 600
+    release:
+      namespace: "braintrust"
+    asserts:
+      - failedTemplate:
+          errorMessage: "brainstore.writer.progressDeadlineSeconds (600) must be greater than brainstore.writer.minReadySeconds (600)"
+
+  - it: should omit rollingUpdate settings for Recreate strategy
+    template: brainstore-writer-deployment.yaml
+    values:
+      - __fixtures__/base-values.yaml
+    set:
+      brainstore.writer.strategy.type: Recreate
+    release:
+      namespace: "braintrust"
+    asserts:
+      - equal:
+          path: spec.strategy.type
+          value: Recreate
+      - isNull:
+          path: spec.strategy.rollingUpdate
+
   - it: should include azure workload identity label when cloud is azure
     template: brainstore-writer-deployment.yaml
     values: