feat: check what an upgrade's index builds cost before any pod rolls [ENG-1881]

charts/currents/templates/_helpers.tpl

@@ -47,6 +47,13 @@ Create toolbox name as used by the chart label.
 {{- printf "%s-%s" (include "currents.fullname" .) .Values.toolbox.name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{/*
+Create the pre-upgrade index check name as used by the chart label.
+*/}}
+{{- define "currents.indexes.fullname" -}}
+{{- printf "%s-%s" (include "currents.fullname" .) .Values.indexes.name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{- define "currents.url" -}}
 {{- if .context.Values.currents.domains.https -}}
 {{- printf "https://%s" .input -}}

charts/currents/templates/indexes/pre-upgrade-job.yaml

@@ -0,0 +1,133 @@
+{{- $mode := .Values.indexes.preUpgrade.mode | default "gate" }}
+{{- if not (has $mode (list "gate" "build" "off")) }}
+{{- fail (printf "indexes.preUpgrade.mode must be one of \"gate\", \"build\" or \"off\", got %q" $mode) }}
+{{- end }}
+{{- if ne $mode "off" }}
+{{/*
+Pre-upgrade index check.
+
+New indexes are built while the upgrade is in flight, and on a large
+`instances` collection that has taken about an hour with the install serving
+nothing. Nothing told the operator beforehand: the scheduler only logs it once
+the pods have already rolled.
+
+This Job runs the new image's `migrate` CLI against the database the old
+release is still serving, before any pod rolls. `index-plan` only reads, so
+running the new code against the old data is safe.
+
+Helm reports a failed hook as `job failed: BackoffLimitExceeded` and does not
+print the pod's log — which is where the estimate and the collection names
+are. Hence the delete policy below, and the `kubectl logs` line in
+docs/eks/upgrading.md.
+*/}}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "currents.indexes.fullname" . }}
+  labels:
+    {{- include "currents.labels" (dict "context" . "component" .Values.indexes.name) | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-upgrade
+    {{- /* Ahead of anything else a future hook adds: there is no point
+           preparing for an upgrade this check is about to stop. */}}
+    helm.sh/hook-weight: "-10"
+    {{- /* Deliberately not hook-failed. That policy deletes the pod as soon as
+           the Job fails, taking the only copy of the message with it. */}}
+    helm.sh/hook-delete-policy: before-hook-creation
+spec:
+  # One attempt. A second read returns the same plan, and retrying only delays
+  # the operator seeing it.
+  backoffLimit: 0
+  {{- if eq $mode "gate" }}
+  {{- /* Only in gate mode. Reading the plan is a handful of listIndexes and
+         $collStats calls, so anything longer than this is a database that is
+         not answering, and a hook with no deadline holds `helm upgrade` open
+         indefinitely. A build has no deadline — the point of that mode is to
+         wait for it. */}}
+  activeDeadlineSeconds: {{ .Values.indexes.preUpgrade.timeoutSeconds | int }}
+  {{- end }}
+  template:
+    metadata:
+      {{- with .Values.global.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "currents.labels" (dict "context" . "component" .Values.indexes.name) | nindent 8 }}
+    spec:
+      restartPolicy: Never
+      {{- with .Values.global.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "currents.serviceAccountName" . }}
+      {{- with .Values.global.securityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.global.priorityClassName }}
+      priorityClassName: {{ . }}
+      {{- end }}
+      containers:
+        - name: index-check
+          {{- with .Values.global.containerSecurityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- /* The scheduler image: it carries packages/mongo, so the same
+                 migrate CLI the scheduler runs at startup. */}}
+          image: {{ include "currents.image" (dict "context" . "imageRoot" .Values.scheduler.image) }}
+          imagePullPolicy: {{ default .Values.global.imagePullPolicy .Values.scheduler.image.pullPolicy }}
+          {{- if eq $mode "build" }}
+          command:
+            - node
+            - /app/packages/mongo/dist/scripts/migrate.js
+            - sync-indexes
+          {{- else }}
+          command:
+            - node
+            - /app/packages/mongo/dist/scripts/migrate.js
+            - index-plan
+            - --gate
+            - --gate-seconds
+            - {{ .Values.indexes.preUpgrade.thresholdSeconds | int | quote }}
+          {{- end }}
+          env:
+            - name: CURRENTS_ENV
+              value: "onprem"
+            {{- /* global.securityContext sets readOnlyRootFilesystem, and no
+                   CI job runs a pre-upgrade hook, so anything that reaches for
+                   a home directory would fail first at a customer's upgrade.
+                   Point it at the writable mount below. */}}
+            - name: HOME
+              value: /tmp
+            {{- /* The whole helper rather than MONGODB_URI alone, so how Mongo
+                   is wired stays defined in one place. The CLI opens no other
+                   connection; the rest of these are unread. */}}
+            {{- include "currents.connectionConfigEnv" . | nindent 12 }}
+            {{- with (concat .Values.global.env .Values.indexes.env) }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
+          {{- with .Values.indexes.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir: {}
+      {{- with .Values.indexes.nodeSelector | default .Values.global.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.indexes.tolerations | default .Values.global.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.indexes.affinity | default .Values.global.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/currents/values.yaml

@@ -924,6 +924,55 @@ toolbox:
   affinity: {}
 # END Toolbox Configuration
 
+# Index Configuration
+# A release that declares a new index builds it while the upgrade is in flight.
+# On a large `instances` collection that has taken about an hour, and the
+# scheduler only logs it once the pods have already rolled. This section runs a
+# `pre-upgrade` hook Job that reads what the build would cost first.
+indexes:
+  name: index-check
+  preUpgrade:
+    # -- What to do before any pod rolls.
+    #
+    # `gate` reads the plan and refuses the upgrade when the estimated build is
+    # over `thresholdSeconds`, leaving the running release untouched. Nothing is
+    # written, and it finishes in seconds.
+    #
+    # `build` builds the indexes first and waits. `helm upgrade` is held open
+    # for the whole build, so raise `--timeout` past the estimate or Helm gives
+    # up partway and marks the release failed.
+    #
+    # `off` renders no hook. Use it to proceed once you have read the estimate
+    # and accepted the downtime.
+    mode: gate
+    # -- Estimated build time, in seconds, that `gate` refuses to upgrade over.
+    # At the built-in rate that is about 18 GB of collection to scan; on a
+    # deployment that has recorded its own builds and measured a faster disk,
+    # considerably more. Until the first build is recorded the estimate is
+    # deliberately pessimistic, so a first upgrade can stop on a build that
+    # turns out to be quick — the Job's log says when that is the case.
+    thresholdSeconds: 600
+    # -- How long `gate` may take before the upgrade fails anyway. Reading the
+    # plan is a handful of `listIndexes` and `$collStats` calls, so overrunning
+    # this means MongoDB is not answering. Without it a hook holds
+    # `helm upgrade` open indefinitely. Not applied to `build`.
+    timeoutSeconds: 300
+  # -- Additional environment variables for the pre-upgrade Job.
+  env: []
+  # -- Resource limits for the pre-upgrade Job. It reads index metadata and
+  # collection stats, so it needs very little.
+  resources: {}
+  # -- [Node selector] for the pre-upgrade Job
+  # @default -- `{}` (defaults to global.nodeSelector)
+  nodeSelector: {}
+  # -- [Tolerations] for use with node taints
+  # @default -- `[]` (defaults to global.tolerations)
+  tolerations: []
+  # -- Assign custom [affinity] rules to the pod
+  # @default -- `{}` (defaults to the global.affinity preset)
+  affinity: {}
+# END Index Configuration
+
 # Service Account Configuration
 # This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
 serviceAccount:

docs/configuration.md

@@ -266,6 +266,15 @@ The following table lists the configurable parameters of the `currents` chart an
 | toolbox.nodeSelector | object | `{}` (defaults to global.nodeSelector) | [Node selector] for the toolbox pod |
 | toolbox.tolerations | list | `[]` (defaults to global.tolerations) | [Tolerations] for use with node taints |
 | toolbox.affinity | object | `{}` (defaults to the global.affinity preset) | Assign custom [affinity] rules to the pod |
+| indexes.name | string | `"index-check"` |  |
+| indexes.preUpgrade.mode | string | `"gate"` | What to do before any pod rolls.  `gate` reads the plan and refuses the upgrade when the estimated build is over `thresholdSeconds`, leaving the running release untouched. Nothing is written, and it finishes in seconds.  `build` builds the indexes first and waits. `helm upgrade` is held open for the whole build, so raise `--timeout` past the estimate or Helm gives up partway and marks the release failed.  `off` renders no hook. Use it to proceed once you have read the estimate and accepted the downtime. |
+| indexes.preUpgrade.thresholdSeconds | int | `600` | Estimated build time, in seconds, that `gate` refuses to upgrade over. At the built-in rate that is about 18 GB of collection to scan; on a deployment that has recorded its own builds and measured a faster disk, considerably more. Until the first build is recorded the estimate is deliberately pessimistic, so a first upgrade can stop on a build that turns out to be quick — the Job's log says when that is the case. |
+| indexes.preUpgrade.timeoutSeconds | int | `300` | How long `gate` may take before the upgrade fails anyway. Reading the plan is a handful of `listIndexes` and `$collStats` calls, so overrunning this means MongoDB is not answering. Without it a hook holds `helm upgrade` open indefinitely. Not applied to `build`. |
+| indexes.env | list | `[]` | Additional environment variables for the pre-upgrade Job. |
+| indexes.resources | object | `{}` | Resource limits for the pre-upgrade Job. It reads index metadata and collection stats, so it needs very little. |
+| indexes.nodeSelector | object | `{}` (defaults to global.nodeSelector) | [Node selector] for the pre-upgrade Job |
+| indexes.tolerations | list | `[]` (defaults to global.tolerations) | [Tolerations] for use with node taints |
+| indexes.affinity | object | `{}` (defaults to the global.affinity preset) | Assign custom [affinity] rules to the pod |
 | serviceAccount.create | bool | `true` | Specifies whether a service account should be created |
 | serviceAccount.name | string | If not set and create is true, a name is generated using the fullname template | The name of the service account to use. |
 | serviceAccount.annotations | object | `{}` | Optional additional annotations to add to the Service Account. Templates are allowed for both keys and values. |

docs/eks/upgrading.md

@@ -62,6 +62,89 @@ Use this when release notes require new secrets:
 2. Ensure your values file references those secret names and keys.
 3. Run `helm upgrade --install`.
 
+## Index Builds Before an Upgrade
+
+A release that declares a new index builds it during the upgrade, and the upgrade is not finished until the build is. On a large `instances` collection that has taken about an hour.
+
+Before any pod rolls, the chart runs a `pre-upgrade` hook Job, `currents-index-check`, that reads what the build would cost. It only reads. If the estimate is over `indexes.preUpgrade.thresholdSeconds` (600 by default), the Job fails and Helm stops the upgrade with the running release untouched:
+
+```
+Error: UPGRADE FAILED: pre-upgrade hooks failed: job currents-index-check failed: BackoffLimitExceeded
+```
+
+**Helm does not print the Job's output**, and the output is the part you need — the estimate, the collections and the indexes. Read it from the pod, which is kept for this:
+
+```bash
+kubectl logs -n currents job/currents-index-check
+```
+
+```
+Index build estimated at 64 minutes, which is over the limit of 10 minutes.
+
+  instances: 109.0 GB, about 62 minutes (createdAt_1)
+
+Estimated at 30.0 MB/s, the built-in default — this deployment has recorded no index builds of its own, so the real build may be several times faster.
+```
+
+That last line matters. Until an upgrade has recorded a build, the estimate uses a deliberately slow rate, so a first upgrade can stop on a build that is really much quicker. Once a build has been recorded the estimate uses this deployment's own measured rate and the figure is worth trusting.
+
+### Ways Forward
+
+Build the indexes before the upgrade, while the running release keeps serving. Nothing is down for it, and the build records a measured rate that later estimates use.
+
+The build has to come from the image you are upgrading **to**: the running release does not declare the new indexes, so running `sync-indexes` inside a current pod rebuilds what is already there. Render the same Job the hook would run, at the target version and under a name of its own, and apply it:
+
+```bash
+helm template currents currents \
+  --repo https://currents-dev.github.io/helm-charts/ \
+  --version <target-chart-version> \
+  -f currents-helm-config.yaml \
+  --set indexes.preUpgrade.mode=build \
+  --set indexes.name=index-build \
+  --show-only templates/indexes/pre-upgrade-job.yaml \
+  | kubectl apply -n currents -f -
+
+kubectl logs -n currents -f job/currents-index-build
+```
+
+`helm template` reads the chart and writes nothing to the cluster, so `--version` is what picks the image — pass the version you are about to upgrade to. A `currents.imageTag` pinned in your values file wins here exactly as it will during the upgrade. `--set indexes.name=index-build` gives the Job a name of its own, so it neither collides with `currents-index-check` nor deletes the failed check's log. The rendered Job still carries its `helm.sh/hook` annotations; outside of Helm they are ordinary annotations and `kubectl` ignores them.
+
+Index builds read whole collections and saturate the secondaries the API reads from, so pick a quiet window. This is the same work the upgrade would do, moved to a time you choose rather than one Helm picks.
+
+When the Job completes, remove it and upgrade as usual — the check finds nothing pending and passes:
+
+```bash
+kubectl delete job -n currents currents-index-build
+```
+
+Or accept the downtime and upgrade anyway:
+
+```bash
+helm upgrade --install currents currents \
+  --repo https://currents-dev.github.io/helm-charts/ \
+  -f currents-helm-config.yaml \
+  --set indexes.preUpgrade.mode=off
+```
+
+Or have the hook build them as part of the upgrade. Helm waits for the whole build, so raise `--timeout` past the estimate — otherwise Helm gives up partway and marks the release failed:
+
+```bash
+helm upgrade --install currents currents \
+  --repo https://currents-dev.github.io/helm-charts/ \
+  -f currents-helm-config.yaml \
+  --set indexes.preUpgrade.mode=build \
+  --timeout 90m
+```
+
+To see what an upgrade would build without upgrading, run the plan against the running release:
+
+```bash
+kubectl exec -n currents deploy/currents-scheduler -- \
+  node /app/packages/mongo/dist/scripts/migrate.js index-plan
+```
+
+This reports the indexes the running version declares, which says nothing about what a newer chart adds. For that, render the check Job the same way as the build above with `--set indexes.preUpgrade.mode=gate`: it reads the plan from the target image and leaves the database alone. It exits non-zero when the estimate is over the threshold, so the Job shows as failed even though the read succeeded; the log is the answer either way.
+
 ## Running the Upgrade
 
 Recommended upgrade command:
@@ -152,6 +235,33 @@ kubectl describe pod <pod-name> -n currents
 kubectl logs <pod-name> -n currents --all-containers --tail=200
 ```
 
+### Upgrade Stopped by the Index Check
+
+```
+Error: UPGRADE FAILED: pre-upgrade hooks failed: job currents-index-check failed
+```
+
+Read the Job's log and pick a way forward: [Index Builds Before an Upgrade](#index-builds-before-an-upgrade).
+
+If the log shows a connection error rather than an estimate, or the Job ends in `DeadlineExceeded`, MongoDB is not answering the chart's `currents.mongoConnection` secret — the same connection every service uses. Fix that before upgrading; the check is reporting a real problem, not getting in the way.
+
+If the log is the `migrate` usage line instead:
+
+```
+Usage: node dist/scripts/migrate.js <up|status|sync-indexes|baseline|mark> [--dry-run]
+```
+
+then the image the Job ran predates the check and does not know `index-plan --gate`. This happens when `currents.imageTag` is pinned to a release older than the chart. Either unpin it, or skip the check for this upgrade:
+
+```bash
+helm upgrade --install currents currents \
+  --repo https://currents-dev.github.io/helm-charts/ \
+  -f currents-helm-config.yaml \
+  --set indexes.preUpgrade.mode=off
+```
+
+The check stops the upgrade whenever it cannot reach a verdict, rather than assuming the build is short. An upgrade it blocks for a reason you have read and accepted costs one flag; one it waves through silently costs the hour it exists to warn you about.
+
 ### Connection Errors in Logs
 
 If services report MongoDB/ClickHouse/Redis/object storage connection errors, dependencies may still be initializing. Restart the affected deployment: