diff --git a/braintrust/README.md b/braintrust/README.md index dd2baac..9fa8f31 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -289,6 +289,85 @@ Pools use fixed replica counts by default (`api.replicas` and `api.workloadIsolation..replicas`). On GKE, enable `api.autoscaling` to let each pool scale independently instead. +## Brainstore Rollout Controls + +Brainstore readers, fast readers, and writers have independently configurable +Deployment strategies and readiness dwell times. These controls let operators +limit how many replacement pods start together and require replacements to +remain Ready before an upgrade continues. This can reduce simultaneous cache +warm-up, object-storage, scheduling, and compaction pressure in cache-heavy or +high-throughput deployments. + +The defaults preserve the rollout behavior from earlier chart versions: + +- `strategy.type: RollingUpdate` +- `strategy.rollingUpdate.maxSurge: 100%` +- `strategy.rollingUpdate.maxUnavailable: 0` +- `minReadySeconds: 0` +- `progressDeadlineSeconds: 600` + +Upgrading the chart without overriding these values does not change rollout +pacing. This feature also does not change the pod template, so it does not +restart Brainstore pods by itself. Custom settings take effect the next time a +pod-template change triggers a rollout. + +Cache-heavy or high-throughput deployments can use a longer readiness dwell: + +```yaml +brainstore: + writer: + minReadySeconds: 300 + progressDeadlineSeconds: 900 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + reader: + minReadySeconds: 120 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + fastreader: + minReadySeconds: 120 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 +``` + +The same `strategy`, `minReadySeconds`, and `progressDeadlineSeconds` settings +are available under each Brainstore role. `progressDeadlineSeconds` must be +greater than `minReadySeconds`; the chart rejects an invalid pairing. Keep +enough deadline margin for scheduling, image pulls, startup, and the readiness +dwell. + +A longer dwell reduces rollout pressure but does not prove that a pod's local +cache is fully warm; monitor workload health until the rollout has converged. +`maxUnavailable: 0` also requires enough cluster capacity for the configured +surge. + +Slow rollouts extend the period during which old and new Brainstore versions +run together. Follow version-specific upgrade guidance and do not change +`brainstoreWalFooterVersion` in the same upgrade as an image version bump, +except where the documented data plane 2.0 upgrade sequence explicitly permits +it. See the [data plane 2.0 upgrade guide](https://www.braintrust.dev/docs/admin/self-hosting/upgrade/v2). + +Setting `strategy.type: Recreate` stops all pods in that Brainstore role before +creating replacements. This causes a complete role outage and, with the default +single writer, pauses background processing until the replacement becomes +Ready. + +These settings pace Deployment-managed rollouts only. The chart does not +currently create PodDisruptionBudgets for Brainstore, so these controls do not +limit voluntary disruptions such as node drains or protect against involuntary +pod or node failures. + ## Testing This Helm chart includes comprehensive automated unit tests. diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 31e0c0a..228f136 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -9,6 +9,37 @@ Get the namespace to use for resources {{- end -}} {{- end -}} +{{/* +Build backward-compatible Deployment rollout settings for one workload. +Template-level defaults are required because `helm upgrade --reuse-values` from +a chart version that predates these settings does not merge in new chart values. +*/}} +{{- define "braintrust.deploymentRollout.config" -}} +{{- $defaultStrategy := dict + "type" "RollingUpdate" + "rollingUpdate" (dict "maxSurge" "100%" "maxUnavailable" 0) +-}} +{{- $strategy := mergeOverwrite (deepCopy $defaultStrategy) (deepCopy (.config.strategy | default dict)) -}} +{{- $rollout := dict + "minReadySeconds" (int (.config.minReadySeconds | default 0)) + "progressDeadlineSeconds" (int (.config.progressDeadlineSeconds | default 600)) + "strategy" $strategy +-}} +{{- toYaml $rollout -}} +{{- end -}} + +{{/* +Validate Deployment rollout timing for one workload. +*/}} +{{- define "braintrust.deploymentRollout.validate" -}} +{{- $path := required "deployment rollout configuration path is required" .path -}} +{{- $minReadySeconds := int .config.minReadySeconds -}} +{{- $progressDeadlineSeconds := int .config.progressDeadlineSeconds -}} +{{- if le $progressDeadlineSeconds $minReadySeconds -}} +{{- fail (printf "%s.progressDeadlineSeconds (%d) must be greater than %s.minReadySeconds (%d)" $path $progressDeadlineSeconds $path $minReadySeconds) -}} +{{- end -}} +{{- end -}} + {{/* Static fast reader query sources used by API. */}} diff --git a/braintrust/templates/brainstore-fastreader-deployment.yaml b/braintrust/templates/brainstore-fastreader-deployment.yaml index 2270917..6ca1d5e 100644 --- a/braintrust/templates/brainstore-fastreader-deployment.yaml +++ b/braintrust/templates/brainstore-fastreader-deployment.yaml @@ -1,3 +1,5 @@ +{{- $rollout := include "braintrust.deploymentRollout.config" (dict "config" .Values.brainstore.fastreader) | fromYaml -}} +{{- include "braintrust.deploymentRollout.validate" (dict "path" "brainstore.fastreader" "config" $rollout) }} apiVersion: apps/v1 kind: Deployment metadata: @@ -13,11 +15,14 @@ metadata: {{- end }} spec: replicas: {{ .Values.brainstore.fastreader.replicas }} + minReadySeconds: {{ $rollout.minReadySeconds }} + progressDeadlineSeconds: {{ $rollout.progressDeadlineSeconds }} strategy: - type: RollingUpdate + type: {{ $rollout.strategy.type }} + {{- if eq $rollout.strategy.type "RollingUpdate" }} rollingUpdate: - maxSurge: 100% - maxUnavailable: 0 + {{- toYaml $rollout.strategy.rollingUpdate | nindent 6 }} + {{- end }} selector: matchLabels: app: {{ .Values.brainstore.fastreader.name }} diff --git a/braintrust/templates/brainstore-reader-deployment.yaml b/braintrust/templates/brainstore-reader-deployment.yaml index bab062a..f0342a7 100644 --- a/braintrust/templates/brainstore-reader-deployment.yaml +++ b/braintrust/templates/brainstore-reader-deployment.yaml @@ -1,3 +1,5 @@ +{{- $rollout := include "braintrust.deploymentRollout.config" (dict "config" .Values.brainstore.reader) | fromYaml -}} +{{- include "braintrust.deploymentRollout.validate" (dict "path" "brainstore.reader" "config" $rollout) }} apiVersion: apps/v1 kind: Deployment metadata: @@ -13,11 +15,14 @@ metadata: {{- end }} spec: replicas: {{ .Values.brainstore.reader.replicas }} + minReadySeconds: {{ $rollout.minReadySeconds }} + progressDeadlineSeconds: {{ $rollout.progressDeadlineSeconds }} strategy: - type: RollingUpdate + type: {{ $rollout.strategy.type }} + {{- if eq $rollout.strategy.type "RollingUpdate" }} rollingUpdate: - maxSurge: 100% - maxUnavailable: 0 + {{- toYaml $rollout.strategy.rollingUpdate | nindent 6 }} + {{- end }} selector: matchLabels: app: {{ .Values.brainstore.reader.name }} diff --git a/braintrust/templates/brainstore-writer-deployment.yaml b/braintrust/templates/brainstore-writer-deployment.yaml index 3ffc61b..f42b692 100644 --- a/braintrust/templates/brainstore-writer-deployment.yaml +++ b/braintrust/templates/brainstore-writer-deployment.yaml @@ -1,3 +1,5 @@ +{{- $rollout := include "braintrust.deploymentRollout.config" (dict "config" .Values.brainstore.writer) | fromYaml -}} +{{- include "braintrust.deploymentRollout.validate" (dict "path" "brainstore.writer" "config" $rollout) }} apiVersion: apps/v1 kind: Deployment metadata: @@ -13,11 +15,14 @@ metadata: {{- end }} spec: replicas: {{ .Values.brainstore.writer.replicas }} + minReadySeconds: {{ $rollout.minReadySeconds }} + progressDeadlineSeconds: {{ $rollout.progressDeadlineSeconds }} strategy: - type: RollingUpdate + type: {{ $rollout.strategy.type }} + {{- if eq $rollout.strategy.type "RollingUpdate" }} rollingUpdate: - maxSurge: 100% - maxUnavailable: 0 + {{- toYaml $rollout.strategy.rollingUpdate | nindent 6 }} + {{- end }} selector: matchLabels: app: {{ .Values.brainstore.writer.name }} diff --git a/braintrust/tests/__fixtures__/pre-rollout-values.yaml b/braintrust/tests/__fixtures__/pre-rollout-values.yaml new file mode 100644 index 0000000..3a7ad3d --- /dev/null +++ b/braintrust/tests/__fixtures__/pre-rollout-values.yaml @@ -0,0 +1,16 @@ +# Simulate values retained by `helm upgrade --reuse-values` from a chart release +# that predates the Brainstore rollout controls. Explicit nulls remove the new +# chart defaults during Helm value coalescing. +brainstore: + reader: + minReadySeconds: null + progressDeadlineSeconds: null + strategy: null + fastreader: + minReadySeconds: null + progressDeadlineSeconds: null + strategy: null + writer: + minReadySeconds: null + progressDeadlineSeconds: null + strategy: null diff --git a/braintrust/tests/brainstore-fastreader_test.yaml b/braintrust/tests/brainstore-fastreader_test.yaml index 9293725..295bb3f 100644 --- a/braintrust/tests/brainstore-fastreader_test.yaml +++ b/braintrust/tests/brainstore-fastreader_test.yaml @@ -18,6 +18,21 @@ tests: - equal: path: spec.replicas value: 1 + - equal: + path: spec.minReadySeconds + value: 0 + - equal: + path: spec.progressDeadlineSeconds + value: 600 + - equal: + path: spec.strategy.type + value: RollingUpdate + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 100% + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 - equal: path: spec.template.spec.containers[0].name value: brainstore-fastreader @@ -28,6 +43,68 @@ tests: path: spec.template.spec.containers[0].envFrom[0].configMapRef.name value: brainstore-fastreader + - it: should render custom Brainstore Fast Reader rollout controls + template: brainstore-fastreader-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.fastreader.minReadySeconds: 120 + brainstore.fastreader.progressDeadlineSeconds: 900 + brainstore.fastreader.strategy.rollingUpdate.maxSurge: 2 + brainstore.fastreader.strategy.rollingUpdate.maxUnavailable: 1 + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.minReadySeconds + value: 120 + - equal: + path: spec.progressDeadlineSeconds + value: 900 + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 2 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 1 + + - it: should preserve legacy rollout defaults when reused values omit rollout controls + template: brainstore-fastreader-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-rollout-values.yaml + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.minReadySeconds + value: 0 + - equal: + path: spec.progressDeadlineSeconds + value: 600 + - equal: + path: spec.strategy.type + value: RollingUpdate + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 100% + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 + + - it: should reject a progress deadline that does not exceed the readiness dwell + template: brainstore-fastreader-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.fastreader.minReadySeconds: 600 + brainstore.fastreader.progressDeadlineSeconds: 600 + release: + namespace: "braintrust" + asserts: + - failedTemplate: + errorMessage: "brainstore.fastreader.progressDeadlineSeconds (600) must be greater than brainstore.fastreader.minReadySeconds (600)" + - it: should include extraEnvVars when provided template: brainstore-fastreader-deployment.yaml values: diff --git a/braintrust/tests/brainstore-reader_test.yaml b/braintrust/tests/brainstore-reader_test.yaml index dd5e99e..5575af9 100644 --- a/braintrust/tests/brainstore-reader_test.yaml +++ b/braintrust/tests/brainstore-reader_test.yaml @@ -18,6 +18,21 @@ tests: - equal: path: spec.replicas value: 1 + - equal: + path: spec.minReadySeconds + value: 0 + - equal: + path: spec.progressDeadlineSeconds + value: 600 + - equal: + path: spec.strategy.type + value: RollingUpdate + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 100% + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 - equal: path: spec.template.spec.containers[0].name value: brainstore-reader @@ -25,6 +40,68 @@ tests: path: spec.template.spec.containers[0].image value: test/brainstore:v1.0.0 + - it: should render custom Brainstore Reader rollout controls + template: brainstore-reader-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.reader.minReadySeconds: 120 + brainstore.reader.progressDeadlineSeconds: 900 + brainstore.reader.strategy.rollingUpdate.maxSurge: 2 + brainstore.reader.strategy.rollingUpdate.maxUnavailable: 1 + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.minReadySeconds + value: 120 + - equal: + path: spec.progressDeadlineSeconds + value: 900 + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 2 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 1 + + - it: should preserve legacy rollout defaults when reused values omit rollout controls + template: brainstore-reader-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-rollout-values.yaml + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.minReadySeconds + value: 0 + - equal: + path: spec.progressDeadlineSeconds + value: 600 + - equal: + path: spec.strategy.type + value: RollingUpdate + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 100% + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 + + - it: should reject a progress deadline that does not exceed the readiness dwell + template: brainstore-reader-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.reader.minReadySeconds: 600 + brainstore.reader.progressDeadlineSeconds: 600 + release: + namespace: "braintrust" + asserts: + - failedTemplate: + errorMessage: "brainstore.reader.progressDeadlineSeconds (600) must be greater than brainstore.reader.minReadySeconds (600)" + - it: should include azure workload identity label when cloud is azure template: brainstore-reader-deployment.yaml values: diff --git a/braintrust/tests/brainstore-writer_test.yaml b/braintrust/tests/brainstore-writer_test.yaml index a2564b5..d2af988 100644 --- a/braintrust/tests/brainstore-writer_test.yaml +++ b/braintrust/tests/brainstore-writer_test.yaml @@ -18,6 +18,21 @@ tests: - equal: path: spec.replicas value: 1 + - equal: + path: spec.minReadySeconds + value: 0 + - equal: + path: spec.progressDeadlineSeconds + value: 600 + - equal: + path: spec.strategy.type + value: RollingUpdate + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 100% + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 - equal: path: spec.template.spec.containers[0].name value: brainstore-writer @@ -25,6 +40,83 @@ tests: path: spec.template.spec.containers[0].image value: test/brainstore:v1.0.0 + - it: should render custom Brainstore Writer rollout controls + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.minReadySeconds: 300 + brainstore.writer.progressDeadlineSeconds: 900 + brainstore.writer.strategy.rollingUpdate.maxSurge: 2 + brainstore.writer.strategy.rollingUpdate.maxUnavailable: 1 + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.minReadySeconds + value: 300 + - equal: + path: spec.progressDeadlineSeconds + value: 900 + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 2 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 1 + + - it: should preserve legacy rollout defaults when reused values omit rollout controls + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/pre-rollout-values.yaml + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.minReadySeconds + value: 0 + - equal: + path: spec.progressDeadlineSeconds + value: 600 + - equal: + path: spec.strategy.type + value: RollingUpdate + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 100% + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 + + - it: should reject a progress deadline that does not exceed the readiness dwell + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.minReadySeconds: 600 + brainstore.writer.progressDeadlineSeconds: 600 + release: + namespace: "braintrust" + asserts: + - failedTemplate: + errorMessage: "brainstore.writer.progressDeadlineSeconds (600) must be greater than brainstore.writer.minReadySeconds (600)" + + - it: should omit rollingUpdate settings for Recreate strategy + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.strategy.type: Recreate + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.strategy.type + value: Recreate + - isNull: + path: spec.strategy.rollingUpdate + - it: should include azure workload identity label when cloud is azure template: brainstore-writer-deployment.yaml values: diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 9e6e706..d702372 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -403,6 +403,15 @@ brainstore: service: {} pod: {} replicas: 2 + # Configure rollout pacing for this Brainstore role. Defaults preserve the + # rollout behavior from earlier chart versions. + minReadySeconds: 0 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 service: name: "" type: ClusterIP @@ -454,6 +463,13 @@ brainstore: service: {} pod: {} replicas: 2 + minReadySeconds: 0 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 service: name: "" type: ClusterIP @@ -505,6 +521,13 @@ brainstore: service: {} pod: {} replicas: 1 + minReadySeconds: 0 + progressDeadlineSeconds: 600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 service: name: "" type: ClusterIP