diff --git a/charts/currents/templates/_helpers.tpl b/charts/currents/templates/_helpers.tpl index c198924..919c475 100644 --- a/charts/currents/templates/_helpers.tpl +++ b/charts/currents/templates/_helpers.tpl @@ -47,6 +47,13 @@ Create toolbox name as used by the chart label. {{- printf "%s-%s" (include "currents.fullname" .) .Values.toolbox.name | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Create the pre-upgrade index check name as used by the chart label. +*/}} +{{- define "currents.indexes.fullname" -}} +{{- printf "%s-%s" (include "currents.fullname" .) .Values.indexes.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{- define "currents.url" -}} {{- if .context.Values.currents.domains.https -}} {{- printf "https://%s" .input -}} diff --git a/charts/currents/templates/indexes/pre-upgrade-job.yaml b/charts/currents/templates/indexes/pre-upgrade-job.yaml new file mode 100644 index 0000000..17eaa8d --- /dev/null +++ b/charts/currents/templates/indexes/pre-upgrade-job.yaml @@ -0,0 +1,133 @@ +{{- $mode := .Values.indexes.preUpgrade.mode | default "gate" }} +{{- if not (has $mode (list "gate" "build" "off")) }} +{{- fail (printf "indexes.preUpgrade.mode must be one of \"gate\", \"build\" or \"off\", got %q" $mode) }} +{{- end }} +{{- if ne $mode "off" }} +{{/* +Pre-upgrade index check. + +New indexes are built while the upgrade is in flight, and on a large +`instances` collection that has taken about an hour with the install serving +nothing. Nothing told the operator beforehand: the scheduler only logs it once +the pods have already rolled. + +This Job runs the new image's `migrate` CLI against the database the old +release is still serving, before any pod rolls. `index-plan` only reads, so +running the new code against the old data is safe. + +Helm reports a failed hook as `job failed: BackoffLimitExceeded` and does not +print the pod's log — which is where the estimate and the collection names +are. Hence the delete policy below, and the `kubectl logs` line in +docs/eks/upgrading.md. +*/}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "currents.indexes.fullname" . }} + labels: + {{- include "currents.labels" (dict "context" . "component" .Values.indexes.name) | nindent 4 }} + annotations: + helm.sh/hook: pre-upgrade + {{- /* Ahead of anything else a future hook adds: there is no point + preparing for an upgrade this check is about to stop. */}} + helm.sh/hook-weight: "-10" + {{- /* Deliberately not hook-failed. That policy deletes the pod as soon as + the Job fails, taking the only copy of the message with it. */}} + helm.sh/hook-delete-policy: before-hook-creation +spec: + # One attempt. A second read returns the same plan, and retrying only delays + # the operator seeing it. + backoffLimit: 0 + {{- if eq $mode "gate" }} + {{- /* Only in gate mode. Reading the plan is a handful of listIndexes and + $collStats calls, so anything longer than this is a database that is + not answering, and a hook with no deadline holds `helm upgrade` open + indefinitely. A build has no deadline — the point of that mode is to + wait for it. */}} + activeDeadlineSeconds: {{ .Values.indexes.preUpgrade.timeoutSeconds | int }} + {{- end }} + template: + metadata: + {{- with .Values.global.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "currents.labels" (dict "context" . "component" .Values.indexes.name) | nindent 8 }} + spec: + restartPolicy: Never + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "currents.serviceAccountName" . }} + {{- with .Values.global.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.global.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + containers: + - name: index-check + {{- with .Values.global.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- /* The scheduler image: it carries packages/mongo, so the same + migrate CLI the scheduler runs at startup. */}} + image: {{ include "currents.image" (dict "context" . "imageRoot" .Values.scheduler.image) }} + imagePullPolicy: {{ default .Values.global.imagePullPolicy .Values.scheduler.image.pullPolicy }} + {{- if eq $mode "build" }} + command: + - node + - /app/packages/mongo/dist/scripts/migrate.js + - sync-indexes + {{- else }} + command: + - node + - /app/packages/mongo/dist/scripts/migrate.js + - index-plan + - --gate + - --gate-seconds + - {{ .Values.indexes.preUpgrade.thresholdSeconds | int | quote }} + {{- end }} + env: + - name: CURRENTS_ENV + value: "onprem" + {{- /* global.securityContext sets readOnlyRootFilesystem, and no + CI job runs a pre-upgrade hook, so anything that reaches for + a home directory would fail first at a customer's upgrade. + Point it at the writable mount below. */}} + - name: HOME + value: /tmp + {{- /* The whole helper rather than MONGODB_URI alone, so how Mongo + is wired stays defined in one place. The CLI opens no other + connection; the rest of these are unread. */}} + {{- include "currents.connectionConfigEnv" . | nindent 12 }} + {{- with (concat .Values.global.env .Values.indexes.env) }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.indexes.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} + {{- with .Values.indexes.nodeSelector | default .Values.global.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.indexes.tolerations | default .Values.global.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.indexes.affinity | default .Values.global.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/currents/values.yaml b/charts/currents/values.yaml index dcce416..6df8e49 100644 --- a/charts/currents/values.yaml +++ b/charts/currents/values.yaml @@ -924,6 +924,55 @@ toolbox: affinity: {} # END Toolbox Configuration +# Index Configuration +# A release that declares a new index builds it while the upgrade is in flight. +# On a large `instances` collection that has taken about an hour, and the +# scheduler only logs it once the pods have already rolled. This section runs a +# `pre-upgrade` hook Job that reads what the build would cost first. +indexes: + name: index-check + preUpgrade: + # -- What to do before any pod rolls. + # + # `gate` reads the plan and refuses the upgrade when the estimated build is + # over `thresholdSeconds`, leaving the running release untouched. Nothing is + # written, and it finishes in seconds. + # + # `build` builds the indexes first and waits. `helm upgrade` is held open + # for the whole build, so raise `--timeout` past the estimate or Helm gives + # up partway and marks the release failed. + # + # `off` renders no hook. Use it to proceed once you have read the estimate + # and accepted the downtime. + mode: gate + # -- Estimated build time, in seconds, that `gate` refuses to upgrade over. + # At the built-in rate that is about 18 GB of collection to scan; on a + # deployment that has recorded its own builds and measured a faster disk, + # considerably more. Until the first build is recorded the estimate is + # deliberately pessimistic, so a first upgrade can stop on a build that + # turns out to be quick — the Job's log says when that is the case. + thresholdSeconds: 600 + # -- How long `gate` may take before the upgrade fails anyway. Reading the + # plan is a handful of `listIndexes` and `$collStats` calls, so overrunning + # this means MongoDB is not answering. Without it a hook holds + # `helm upgrade` open indefinitely. Not applied to `build`. + timeoutSeconds: 300 + # -- Additional environment variables for the pre-upgrade Job. + env: [] + # -- Resource limits for the pre-upgrade Job. It reads index metadata and + # collection stats, so it needs very little. + resources: {} + # -- [Node selector] for the pre-upgrade Job + # @default -- `{}` (defaults to global.nodeSelector) + nodeSelector: {} + # -- [Tolerations] for use with node taints + # @default -- `[]` (defaults to global.tolerations) + tolerations: [] + # -- Assign custom [affinity] rules to the pod + # @default -- `{}` (defaults to the global.affinity preset) + affinity: {} +# END Index Configuration + # Service Account Configuration # This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ serviceAccount: diff --git a/docs/configuration.md b/docs/configuration.md index bdeeda6..2c58f41 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -266,6 +266,15 @@ The following table lists the configurable parameters of the `currents` chart an | toolbox.nodeSelector | object | `{}` (defaults to global.nodeSelector) | [Node selector] for the toolbox pod | | toolbox.tolerations | list | `[]` (defaults to global.tolerations) | [Tolerations] for use with node taints | | toolbox.affinity | object | `{}` (defaults to the global.affinity preset) | Assign custom [affinity] rules to the pod | +| indexes.name | string | `"index-check"` | | +| indexes.preUpgrade.mode | string | `"gate"` | What to do before any pod rolls. `gate` reads the plan and refuses the upgrade when the estimated build is over `thresholdSeconds`, leaving the running release untouched. Nothing is written, and it finishes in seconds. `build` builds the indexes first and waits. `helm upgrade` is held open for the whole build, so raise `--timeout` past the estimate or Helm gives up partway and marks the release failed. `off` renders no hook. Use it to proceed once you have read the estimate and accepted the downtime. | +| indexes.preUpgrade.thresholdSeconds | int | `600` | Estimated build time, in seconds, that `gate` refuses to upgrade over. At the built-in rate that is about 18 GB of collection to scan; on a deployment that has recorded its own builds and measured a faster disk, considerably more. Until the first build is recorded the estimate is deliberately pessimistic, so a first upgrade can stop on a build that turns out to be quick — the Job's log says when that is the case. | +| indexes.preUpgrade.timeoutSeconds | int | `300` | How long `gate` may take before the upgrade fails anyway. Reading the plan is a handful of `listIndexes` and `$collStats` calls, so overrunning this means MongoDB is not answering. Without it a hook holds `helm upgrade` open indefinitely. Not applied to `build`. | +| indexes.env | list | `[]` | Additional environment variables for the pre-upgrade Job. | +| indexes.resources | object | `{}` | Resource limits for the pre-upgrade Job. It reads index metadata and collection stats, so it needs very little. | +| indexes.nodeSelector | object | `{}` (defaults to global.nodeSelector) | [Node selector] for the pre-upgrade Job | +| indexes.tolerations | list | `[]` (defaults to global.tolerations) | [Tolerations] for use with node taints | +| indexes.affinity | object | `{}` (defaults to the global.affinity preset) | Assign custom [affinity] rules to the pod | | serviceAccount.create | bool | `true` | Specifies whether a service account should be created | | serviceAccount.name | string | If not set and create is true, a name is generated using the fullname template | The name of the service account to use. | | serviceAccount.annotations | object | `{}` | Optional additional annotations to add to the Service Account. Templates are allowed for both keys and values. | diff --git a/docs/eks/upgrading.md b/docs/eks/upgrading.md index cc794b6..64504db 100644 --- a/docs/eks/upgrading.md +++ b/docs/eks/upgrading.md @@ -62,6 +62,89 @@ Use this when release notes require new secrets: 2. Ensure your values file references those secret names and keys. 3. Run `helm upgrade --install`. +## Index Builds Before an Upgrade + +A release that declares a new index builds it during the upgrade, and the upgrade is not finished until the build is. On a large `instances` collection that has taken about an hour. + +Before any pod rolls, the chart runs a `pre-upgrade` hook Job, `currents-index-check`, that reads what the build would cost. It only reads. If the estimate is over `indexes.preUpgrade.thresholdSeconds` (600 by default), the Job fails and Helm stops the upgrade with the running release untouched: + +``` +Error: UPGRADE FAILED: pre-upgrade hooks failed: job currents-index-check failed: BackoffLimitExceeded +``` + +**Helm does not print the Job's output**, and the output is the part you need — the estimate, the collections and the indexes. Read it from the pod, which is kept for this: + +```bash +kubectl logs -n currents job/currents-index-check +``` + +``` +Index build estimated at 64 minutes, which is over the limit of 10 minutes. + + instances: 109.0 GB, about 62 minutes (createdAt_1) + +Estimated at 30.0 MB/s, the built-in default — this deployment has recorded no index builds of its own, so the real build may be several times faster. +``` + +That last line matters. Until an upgrade has recorded a build, the estimate uses a deliberately slow rate, so a first upgrade can stop on a build that is really much quicker. Once a build has been recorded the estimate uses this deployment's own measured rate and the figure is worth trusting. + +### Ways Forward + +Build the indexes before the upgrade, while the running release keeps serving. Nothing is down for it, and the build records a measured rate that later estimates use. + +The build has to come from the image you are upgrading **to**: the running release does not declare the new indexes, so running `sync-indexes` inside a current pod rebuilds what is already there. Render the same Job the hook would run, at the target version and under a name of its own, and apply it: + +```bash +helm template currents currents \ + --repo https://currents-dev.github.io/helm-charts/ \ + --version \ + -f currents-helm-config.yaml \ + --set indexes.preUpgrade.mode=build \ + --set indexes.name=index-build \ + --show-only templates/indexes/pre-upgrade-job.yaml \ + | kubectl apply -n currents -f - + +kubectl logs -n currents -f job/currents-index-build +``` + +`helm template` reads the chart and writes nothing to the cluster, so `--version` is what picks the image — pass the version you are about to upgrade to. A `currents.imageTag` pinned in your values file wins here exactly as it will during the upgrade. `--set indexes.name=index-build` gives the Job a name of its own, so it neither collides with `currents-index-check` nor deletes the failed check's log. The rendered Job still carries its `helm.sh/hook` annotations; outside of Helm they are ordinary annotations and `kubectl` ignores them. + +Index builds read whole collections and saturate the secondaries the API reads from, so pick a quiet window. This is the same work the upgrade would do, moved to a time you choose rather than one Helm picks. + +When the Job completes, remove it and upgrade as usual — the check finds nothing pending and passes: + +```bash +kubectl delete job -n currents currents-index-build +``` + +Or accept the downtime and upgrade anyway: + +```bash +helm upgrade --install currents currents \ + --repo https://currents-dev.github.io/helm-charts/ \ + -f currents-helm-config.yaml \ + --set indexes.preUpgrade.mode=off +``` + +Or have the hook build them as part of the upgrade. Helm waits for the whole build, so raise `--timeout` past the estimate — otherwise Helm gives up partway and marks the release failed: + +```bash +helm upgrade --install currents currents \ + --repo https://currents-dev.github.io/helm-charts/ \ + -f currents-helm-config.yaml \ + --set indexes.preUpgrade.mode=build \ + --timeout 90m +``` + +To see what an upgrade would build without upgrading, run the plan against the running release: + +```bash +kubectl exec -n currents deploy/currents-scheduler -- \ + node /app/packages/mongo/dist/scripts/migrate.js index-plan +``` + +This reports the indexes the running version declares, which says nothing about what a newer chart adds. For that, render the check Job the same way as the build above with `--set indexes.preUpgrade.mode=gate`: it reads the plan from the target image and leaves the database alone. It exits non-zero when the estimate is over the threshold, so the Job shows as failed even though the read succeeded; the log is the answer either way. + ## Running the Upgrade Recommended upgrade command: @@ -152,6 +235,33 @@ kubectl describe pod -n currents kubectl logs -n currents --all-containers --tail=200 ``` +### Upgrade Stopped by the Index Check + +``` +Error: UPGRADE FAILED: pre-upgrade hooks failed: job currents-index-check failed +``` + +Read the Job's log and pick a way forward: [Index Builds Before an Upgrade](#index-builds-before-an-upgrade). + +If the log shows a connection error rather than an estimate, or the Job ends in `DeadlineExceeded`, MongoDB is not answering the chart's `currents.mongoConnection` secret — the same connection every service uses. Fix that before upgrading; the check is reporting a real problem, not getting in the way. + +If the log is the `migrate` usage line instead: + +``` +Usage: node dist/scripts/migrate.js [--dry-run] +``` + +then the image the Job ran predates the check and does not know `index-plan --gate`. This happens when `currents.imageTag` is pinned to a release older than the chart. Either unpin it, or skip the check for this upgrade: + +```bash +helm upgrade --install currents currents \ + --repo https://currents-dev.github.io/helm-charts/ \ + -f currents-helm-config.yaml \ + --set indexes.preUpgrade.mode=off +``` + +The check stops the upgrade whenever it cannot reach a verdict, rather than assuming the build is short. An upgrade it blocks for a reason you have read and accepted costs one flag; one it waves through silently costs the hour it exists to warn you about. + ### Connection Errors in Logs If services report MongoDB/ClickHouse/Redis/object storage connection errors, dependencies may still be initializing. Restart the affected deployment: