diff --git a/braintrust/README.md b/braintrust/README.md index dd2baac..9fa8f31 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -289,6 +289,85 @@ Pools use fixed replica counts by default (`api.replicas` and `api.workloadIsolation..replicas`). On GKE, enable `api.autoscaling` to let each pool scale independently instead. +## Brainstore Rollout Controls + +Brainstore readers, fast readers, and writers have independently configurable +Deployment strategies and readiness dwell times. These controls let operators +limit how many replacement pods start together and require replacements to +remain Ready before an upgrade continues. This can reduce simultaneous cache +warm-up, object-storage, scheduling, and compaction pressure in cache-heavy or +high-throughput deployments. + +The defaults preserve the rollout behavior from earlier chart versions: + +- `strategy.type: RollingUpdate` +- `strategy.rollingUpdate.maxSurge: 100%` +- `strategy.rollingUpdate.maxUnavailable: 0` +- `minReadySeconds: 0` +- `progressDeadlineSeconds: 600` + +Upgrading the chart without overriding these values does not change rollout +pacing. This feature also does not change the pod template, so it does not +restart Brainstore pods by itself. Custom settings take effect the next time a +pod-template change triggers a rollout. + +Cache-heavy or high-throughput deployments can use a longer readiness dwell: + +```yaml +brainstore: + writer: + minReadySeconds: 300 + progressDeadlineSeconds: 900 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + reader: + minReadySeconds: 120 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + fastreader: + minReadySeconds: 120 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 +``` + +The same `strategy`, `minReadySeconds`, and `progressDeadlineSeconds` settings +are available under each Brainstore role. `progressDeadlineSeconds` must be +greater than `minReadySeconds`; the chart rejects an invalid pairing. Keep +enough deadline margin for scheduling, image pulls, startup, and the readiness +dwell. + +A longer dwell reduces rollout pressure but does not prove that a pod's local +cache is fully warm; monitor workload health until the rollout has converged. +`maxUnavailable: 0` also requires enough cluster capacity for the configured +surge. + +Slow rollouts extend the period during which old and new Brainstore versions +run together. Follow version-specific upgrade guidance and do not change +`brainstoreWalFooterVersion` in the same upgrade as an image version bump, +except where the documented data plane 2.0 upgrade sequence explicitly permits +it. See the [data plane 2.0 upgrade guide](https://www.braintrust.dev/docs/admin/self-hosting/upgrade/v2). + +Setting `strategy.type: Recreate` stops all pods in that Brainstore role before +creating replacements. This causes a complete role outage and, with the default +single writer, pauses background processing until the replacement becomes +Ready. + +These settings pace Deployment-managed rollouts only. The chart does not +currently create PodDisruptionBudgets for Brainstore, so these controls do not +limit voluntary disruptions such as node drains or protect against involuntary +pod or node failures. + ## Testing This Helm chart includes comprehensive automated unit tests. diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 31e0c0a..1045963 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -9,6 +9,17 @@ Get the namespace to use for resources {{- end -}} {{- end -}} +{{/* +Validate Brainstore Deployment rollout timing for one role. +*/}} +{{- define "braintrust.brainstoreRollout.validate" -}} +{{- $minReadySeconds := int .config.minReadySeconds -}} +{{- $progressDeadlineSeconds := int .config.progressDeadlineSeconds -}} +{{- if le $progressDeadlineSeconds $minReadySeconds -}} +{{- fail (printf "brainstore.%s.progressDeadlineSeconds (%d) must be greater than brainstore.%s.minReadySeconds (%d)" .role $progressDeadlineSeconds .role $minReadySeconds) -}} +{{- end -}} +{{- end -}} + {{/* Static fast reader query sources used by API. */}} diff --git a/braintrust/templates/brainstore-fastreader-deployment.yaml b/braintrust/templates/brainstore-fastreader-deployment.yaml index 2270917..c845db9 100644 --- a/braintrust/templates/brainstore-fastreader-deployment.yaml +++ b/braintrust/templates/brainstore-fastreader-deployment.yaml @@ -1,3 +1,4 @@ +{{- include "braintrust.brainstoreRollout.validate" (dict "role" "fastreader" "config" .Values.brainstore.fastreader) }} apiVersion: apps/v1 kind: Deployment metadata: @@ -13,11 +14,14 @@ metadata: {{- end }} spec: replicas: {{ .Values.brainstore.fastreader.replicas }} + minReadySeconds: {{ .Values.brainstore.fastreader.minReadySeconds }} + progressDeadlineSeconds: {{ .Values.brainstore.fastreader.progressDeadlineSeconds }} strategy: - type: RollingUpdate + type: {{ .Values.brainstore.fastreader.strategy.type }} + {{- if eq .Values.brainstore.fastreader.strategy.type "RollingUpdate" }} rollingUpdate: - maxSurge: 100% - maxUnavailable: 0 + {{- toYaml .Values.brainstore.fastreader.strategy.rollingUpdate | nindent 6 }} + {{- end }} selector: matchLabels: app: {{ .Values.brainstore.fastreader.name }} diff --git a/braintrust/templates/brainstore-reader-deployment.yaml b/braintrust/templates/brainstore-reader-deployment.yaml index bab062a..c74ee27 100644 --- a/braintrust/templates/brainstore-reader-deployment.yaml +++ b/braintrust/templates/brainstore-reader-deployment.yaml @@ -1,3 +1,4 @@ +{{- include "braintrust.brainstoreRollout.validate" (dict "role" "reader" "config" .Values.brainstore.reader) }} apiVersion: apps/v1 kind: Deployment metadata: @@ -13,11 +14,14 @@ metadata: {{- end }} spec: replicas: {{ .Values.brainstore.reader.replicas }} + minReadySeconds: {{ .Values.brainstore.reader.minReadySeconds }} + progressDeadlineSeconds: {{ .Values.brainstore.reader.progressDeadlineSeconds }} strategy: - type: RollingUpdate + type: {{ .Values.brainstore.reader.strategy.type }} + {{- if eq .Values.brainstore.reader.strategy.type "RollingUpdate" }} rollingUpdate: - maxSurge: 100% - maxUnavailable: 0 + {{- toYaml .Values.brainstore.reader.strategy.rollingUpdate | nindent 6 }} + {{- end }} selector: matchLabels: app: {{ .Values.brainstore.reader.name }} diff --git a/braintrust/templates/brainstore-writer-deployment.yaml b/braintrust/templates/brainstore-writer-deployment.yaml index 3ffc61b..db2b59d 100644 --- a/braintrust/templates/brainstore-writer-deployment.yaml +++ b/braintrust/templates/brainstore-writer-deployment.yaml @@ -1,3 +1,4 @@ +{{- include "braintrust.brainstoreRollout.validate" (dict "role" "writer" "config" .Values.brainstore.writer) }} apiVersion: apps/v1 kind: Deployment metadata: @@ -13,11 +14,14 @@ metadata: {{- end }} spec: replicas: {{ .Values.brainstore.writer.replicas }} + minReadySeconds: {{ .Values.brainstore.writer.minReadySeconds }} + progressDeadlineSeconds: {{ .Values.brainstore.writer.progressDeadlineSeconds }} strategy: - type: RollingUpdate + type: {{ .Values.brainstore.writer.strategy.type }} + {{- if eq .Values.brainstore.writer.strategy.type "RollingUpdate" }} rollingUpdate: - maxSurge: 100% - maxUnavailable: 0 + {{- toYaml .Values.brainstore.writer.strategy.rollingUpdate | nindent 6 }} + {{- end }} selector: matchLabels: app: {{ .Values.brainstore.writer.name }} diff --git a/braintrust/tests/brainstore-fastreader_test.yaml b/braintrust/tests/brainstore-fastreader_test.yaml index 9293725..ceecf7b 100644 --- a/braintrust/tests/brainstore-fastreader_test.yaml +++ b/braintrust/tests/brainstore-fastreader_test.yaml @@ -18,6 +18,21 @@ tests: - equal: path: spec.replicas value: 1 + - equal: + path: spec.minReadySeconds + value: 0 + - equal: + path: spec.progressDeadlineSeconds + value: 600 + - equal: + path: spec.strategy.type + value: RollingUpdate + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 100% + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 - equal: path: spec.template.spec.containers[0].name value: brainstore-fastreader @@ -28,6 +43,44 @@ tests: path: spec.template.spec.containers[0].envFrom[0].configMapRef.name value: brainstore-fastreader + - it: should render custom Brainstore Fast Reader rollout controls + template: brainstore-fastreader-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.fastreader.minReadySeconds: 120 + brainstore.fastreader.progressDeadlineSeconds: 900 + brainstore.fastreader.strategy.rollingUpdate.maxSurge: 2 + brainstore.fastreader.strategy.rollingUpdate.maxUnavailable: 1 + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.minReadySeconds + value: 120 + - equal: + path: spec.progressDeadlineSeconds + value: 900 + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 2 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 1 + + - it: should reject a progress deadline that does not exceed the readiness dwell + template: brainstore-fastreader-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.fastreader.minReadySeconds: 600 + brainstore.fastreader.progressDeadlineSeconds: 600 + release: + namespace: "braintrust" + asserts: + - failedTemplate: + errorMessage: "brainstore.fastreader.progressDeadlineSeconds (600) must be greater than brainstore.fastreader.minReadySeconds (600)" + - it: should include extraEnvVars when provided template: brainstore-fastreader-deployment.yaml values: diff --git a/braintrust/tests/brainstore-reader_test.yaml b/braintrust/tests/brainstore-reader_test.yaml index dd5e99e..9439e9a 100644 --- a/braintrust/tests/brainstore-reader_test.yaml +++ b/braintrust/tests/brainstore-reader_test.yaml @@ -18,6 +18,21 @@ tests: - equal: path: spec.replicas value: 1 + - equal: + path: spec.minReadySeconds + value: 0 + - equal: + path: spec.progressDeadlineSeconds + value: 600 + - equal: + path: spec.strategy.type + value: RollingUpdate + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 100% + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 - equal: path: spec.template.spec.containers[0].name value: brainstore-reader @@ -25,6 +40,44 @@ tests: path: spec.template.spec.containers[0].image value: test/brainstore:v1.0.0 + - it: should render custom Brainstore Reader rollout controls + template: brainstore-reader-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.reader.minReadySeconds: 120 + brainstore.reader.progressDeadlineSeconds: 900 + brainstore.reader.strategy.rollingUpdate.maxSurge: 2 + brainstore.reader.strategy.rollingUpdate.maxUnavailable: 1 + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.minReadySeconds + value: 120 + - equal: + path: spec.progressDeadlineSeconds + value: 900 + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 2 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 1 + + - it: should reject a progress deadline that does not exceed the readiness dwell + template: brainstore-reader-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.reader.minReadySeconds: 600 + brainstore.reader.progressDeadlineSeconds: 600 + release: + namespace: "braintrust" + asserts: + - failedTemplate: + errorMessage: "brainstore.reader.progressDeadlineSeconds (600) must be greater than brainstore.reader.minReadySeconds (600)" + - it: should include azure workload identity label when cloud is azure template: brainstore-reader-deployment.yaml values: diff --git a/braintrust/tests/brainstore-writer_test.yaml b/braintrust/tests/brainstore-writer_test.yaml index a2564b5..a3e8ced 100644 --- a/braintrust/tests/brainstore-writer_test.yaml +++ b/braintrust/tests/brainstore-writer_test.yaml @@ -18,6 +18,21 @@ tests: - equal: path: spec.replicas value: 1 + - equal: + path: spec.minReadySeconds + value: 0 + - equal: + path: spec.progressDeadlineSeconds + value: 600 + - equal: + path: spec.strategy.type + value: RollingUpdate + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 100% + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 - equal: path: spec.template.spec.containers[0].name value: brainstore-writer @@ -25,6 +40,59 @@ tests: path: spec.template.spec.containers[0].image value: test/brainstore:v1.0.0 + - it: should render custom Brainstore Writer rollout controls + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.minReadySeconds: 300 + brainstore.writer.progressDeadlineSeconds: 900 + brainstore.writer.strategy.rollingUpdate.maxSurge: 2 + brainstore.writer.strategy.rollingUpdate.maxUnavailable: 1 + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.minReadySeconds + value: 300 + - equal: + path: spec.progressDeadlineSeconds + value: 900 + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 2 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 1 + + - it: should reject a progress deadline that does not exceed the readiness dwell + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.minReadySeconds: 600 + brainstore.writer.progressDeadlineSeconds: 600 + release: + namespace: "braintrust" + asserts: + - failedTemplate: + errorMessage: "brainstore.writer.progressDeadlineSeconds (600) must be greater than brainstore.writer.minReadySeconds (600)" + + - it: should omit rollingUpdate settings for Recreate strategy + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.strategy.type: Recreate + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.strategy.type + value: Recreate + - isNull: + path: spec.strategy.rollingUpdate + - it: should include azure workload identity label when cloud is azure template: brainstore-writer-deployment.yaml values: diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 9e6e706..d702372 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -403,6 +403,15 @@ brainstore: service: {} pod: {} replicas: 2 + # Configure rollout pacing for this Brainstore role. Defaults preserve the + # rollout behavior from earlier chart versions. + minReadySeconds: 0 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 service: name: "" type: ClusterIP @@ -454,6 +463,13 @@ brainstore: service: {} pod: {} replicas: 2 + minReadySeconds: 0 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 service: name: "" type: ClusterIP @@ -505,6 +521,13 @@ brainstore: service: {} pod: {} replicas: 1 + minReadySeconds: 0 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 service: name: "" type: ClusterIP