diff --git a/.github/workflows/helm-ci.yaml b/.github/workflows/helm-ci.yaml index 8f6195c..1813eeb 100644 --- a/.github/workflows/helm-ci.yaml +++ b/.github/workflows/helm-ci.yaml @@ -117,28 +117,49 @@ jobs: echo "Schema validation passed for ${{ matrix.platform }}" ingestor-multiarch: - # Guard: the chart's PINNED ingestor digest must be a multi-arch index - # (linux/amd64 + linux/arm64). Greenfield installs spawn the ingestor from this - # pinned digest (before image-refresh ticks), so an amd64-only pin breaks data - # ingestion on arm64 hosts (Apple Silicon, Graviton) with ImagePullBackOff. This - # would have caught #160 (which pinned the amd64-only v0.3.1). See client#186. - name: Pinned ingestor digest is multi-arch + # Guard: the ingestor image the cluster spawns must be a multi-arch index + # (linux/amd64 + linux/arm64), or arm64 hosts (Apple Silicon, Graviton) + # fail data ingestion with "no match for platform" / ImagePullBackOff. + # jobs-manager spawns ingestion Jobs by the floating tag + # `images.ingestor.tag` (imagePullPolicy=Always) by DEFAULT, so that tag + # must be multi-arch. If an operator opts into pinning via + # `images.ingestor.digest` (empty by default), the pinned digest must be + # multi-arch too. This would have caught #160 (the amd64-only v0.3.1 pin). + # See client#186. + name: Spawned ingestor image is multi-arch runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Assert images.ingestor.digest supports linux/amd64 + linux/arm64 + - name: Assert the ingestor tag (and pinned digest, if any) is multi-arch run: | + repo=$(yq '.images.ingestor.repository' client/values.yaml) + if [ -z "$repo" ] || [ "$repo" = "null" ]; then repo="ghcr.io/tracebloc/ingestor"; fi + tag=$(yq '.images.ingestor.tag' client/values.yaml) digest=$(yq '.images.ingestor.digest' client/values.yaml) - echo "Pinned ingestor digest: $digest" - if [ -z "$digest" ] || [ "$digest" = "null" ]; then - echo "::error::images.ingestor.digest is empty — it must be a pinned multi-arch digest."; exit 1 + + assert_multiarch() { + ref="$1"; label="$2" + echo "Inspecting $label: $ref" + plats=$(docker buildx imagetools inspect "$ref" 2>&1 \ + | awk '/Platform:/{print $2}' | grep -v '^unknown' | sort -u) + echo "Platforms: $(echo "$plats" | paste -sd' ' -)" + echo "$plats" | grep -qx 'linux/arm64' || { echo "::error::$label ($ref) is NOT multi-arch (no linux/arm64). arm64 installs (Apple Silicon, Graviton) would fail ingestion with 'no match for platform' / ImagePullBackOff. See client#186 / #160."; exit 1; } + echo "$plats" | grep -qx 'linux/amd64' || { echo "::error::$label ($ref) is missing linux/amd64."; exit 1; } + echo "OK — $label is multi-arch (amd64 + arm64)." + } + + # The floating tag is the default spawn target — always validate it. + if [ -z "$tag" ] || [ "$tag" = "null" ]; then + echo "::error::images.ingestor.tag is empty — the chart must define a floating tag to spawn by."; exit 1 + fi + assert_multiarch "${repo}:${tag}" "floating tag" + + # A pinned digest is opt-in (empty by default). When set, it must be multi-arch too. + if [ -n "$digest" ] && [ "$digest" != "null" ]; then + assert_multiarch "${repo}@${digest}" "pinned digest" + else + echo "images.ingestor.digest empty (default) — spawning by floating tag; no pinned digest to check." fi - plats=$(docker buildx imagetools inspect "ghcr.io/tracebloc/ingestor@$digest" 2>&1 \ - | awk '/Platform:/{print $2}' | grep -v '^unknown' | sort -u) - echo "Platforms: $(echo "$plats" | paste -sd' ' -)" - echo "$plats" | grep -qx 'linux/arm64' || { echo "::error::Pinned ingestor digest is NOT multi-arch (no linux/arm64). arm64 installs (Apple Silicon, Graviton) would fail ingestion with 'no match for platform' / ImagePullBackOff. Pin a multi-arch :0.x index. See client#186 / #160."; exit 1; } - echo "$plats" | grep -qx 'linux/amd64' || { echo "::error::Pinned ingestor digest is missing linux/amd64."; exit 1; } - echo "OK — pinned ingestor digest is multi-arch (amd64 + arm64)." # Installer script tests (bats + Pester) + the cross-distro prerequisite matrix # live in their own workflow: .github/workflows/installer-tests.yaml diff --git a/client/Chart.yaml b/client/Chart.yaml index 56fece4..a79dbeb 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.5.1 +version: 1.6.0 appVersion: "1.5.1" keywords: - tracebloc diff --git a/client/templates/_helpers.tpl b/client/templates/_helpers.tpl index 574559b..4631f0d 100644 --- a/client/templates/_helpers.tpl +++ b/client/templates/_helpers.tpl @@ -145,27 +145,20 @@ mysql-pvc {{- $imgs := default dict .Values.images -}} {{- $jm := default dict $imgs.jobsManager -}} {{- $pm := default dict $imgs.podsMonitor -}} -{{- $in := default dict $imgs.ingestor -}} {{/* - Per-image pin signals (each one means "skip auto-refresh for this image"): - * jobs-manager / pods-monitor: digest set (non-empty) — same signal as - the deployment uses to switch imagePullPolicy to IfNotPresent. - * ingestor: explicit `autoRefresh: false` flag — asymmetric because - ingestor.digest must be non-empty for jobs-manager to work, so we - can't use digest-presence as the signal there. + Per-image pin signal (means "skip auto-refresh for this image"): + jobs-manager / pods-monitor are pinned when `digest` is set (non-empty) — + the same signal the deployment uses to switch imagePullPolicy to + IfNotPresent. The ingestor is no longer refreshed by this CronJob (it is + spawned by jobs-manager from a floating tag — see the + image-refresh-cronjob.yaml header and submit_ingestion_run in + client-runtime), so the CronJob exists only to refresh the two class-1 + images: when BOTH are pinned there is nothing left for it to do. */}} {{- $jmPinned := $jm.digest -}} {{- $pmPinned := $pm.digest -}} -{{/* - Can't use `default true $in.autoRefresh` here — Go templates treat - the bool `false` as falsy, so `default true false` returns `true` - and flips the pin state on the explicit-disable case. Instead test - for the literal `false` directly; absence (nil) and explicit `true` - both fall through to "not pinned". -*/}} -{{- $inPinned := eq $in.autoRefresh false -}} {{- if not $ir.enabled -}} -{{- else if and (and $jmPinned $pmPinned) $inPinned -}} +{{- else if and $jmPinned $pmPinned -}} {{- else -}} true {{- end -}} diff --git a/client/templates/image-refresh-cronjob.yaml b/client/templates/image-refresh-cronjob.yaml index af4e2e3..df684ce 100644 --- a/client/templates/image-refresh-cronjob.yaml +++ b/client/templates/image-refresh-cronjob.yaml @@ -11,67 +11,35 @@ metadata: data: image-refresh.sh: | #!/bin/sh - # Polls registries for new manifest digests and reconciles them onto - # the jobs-manager Deployment. Handles two image classes with - # different reconciliation semantics: + # Polls the registry for a new manifest digest and reconciles it onto + # the long-lived jobs-manager Deployment (the jobs-manager + pods-monitor + # containers, on docker.io under the floating CLIENT_ENV tag, #154). + # Action on change: `kubectl rollout restart` the deployment. + # Source of truth: an annotation on the deployment metadata + # (`tracebloc.io/last-refreshed--digest`) — comparing the registry + # digest to an annotation we wrote ourselves sidesteps imageID format + # variation across container runtimes (kubernetes/kubernetes#108689 + + # #115199) and multi-arch manifest-list vs platform-manifest divergence. # - # 1. Long-running container images (jobs-manager, pods-monitor) on - # docker.io under the floating CLIENT_ENV tag (#154). - # Action on change: `kubectl rollout restart` the deployment. - # Source of truth: an annotation on the deployment metadata - # (`tracebloc.io/last-refreshed--digest`) — comparing - # the registry digest to an annotation we wrote ourselves - # sidesteps imageID format variation across container runtimes - # (kubernetes/kubernetes#108689 + #115199) and multi-arch - # manifest-list vs platform-manifest divergence. + # The ingestor image is intentionally NOT handled here. jobs-manager + # spawns each ingestion Job by the floating tag `images.ingestor.tag` + # with imagePullPolicy=Always (see jobs-manager-deployment.yaml and + # client-runtime submit_ingestion_run._build_image_reference), so the + # kubelet resolves the ingestor's current digest at every spawn — there + # is nothing for this CronJob to reconcile. Until the floating-tag + # migration a "class-2" pass here kept an INGESTOR_IMAGE_DIGEST env + # current via `kubectl set env`; it was retired because a `helm upgrade` + # that reset the env could revert the ingestor to a stale baseline (the + # regression a customer hit). Spawning by floating tag is revert-proof. # - # 2. Spawned-Job image (ingestor) on ghcr.io under - # images.ingestor.tag (#158). This image isn't a container in - # the jobs-manager pod — it's the image jobs-manager USES when - # spawning each ingestion Job, surfaced via the - # INGESTOR_IMAGE_DIGEST env var on the api container. - # Action on change: `kubectl set env` to patch the env var, - # which triggers a natural rollout via ReplicaSet rotation — - # no explicit `rollout restart` needed. - # Source of truth: same as class 1 — a deployment annotation - # (`tracebloc.io/last-refreshed-ingestor-digest`) records the - # last successfully-rolled digest. Originally this class read - # the live env directly, but that broke rollout-failure-retry - # semantics (`set env` commits the spec before `rollout status` - # waits, so a failed rollout would leave the spec matching the - # registry while old pods kept the old env — next tick saw no - # drift and skipped, leaving the deployment stuck). Annotation - # is updated as the LAST step of each successful refresh; a - # failed rollout leaves the annotation stale and the next tick - # retries. (PR #159 review, bugbot.) + # First-tick contract: annotation missing → record the current registry + # digest WITHOUT restarting (no evidence of drift, no reason to churn + # pods). # - # First-tick contract: - # * Class 1 (annotation missing): record current registry digest - # WITHOUT restarting — no evidence of drift, no reason to churn - # pods. - # * Class 2 (annotation missing): adopt the spec env's - # chart-default value as the baseline annotation, no env change. - # Same don't-churn-on-install principle. Empty spec env - # (corrupted state, manual edit, stale --reuse-values from a - # pre-1.4.1 chart) is treated as a change-needed signal — fill - # from registry on first tick, because an empty value would - # cause jobs-manager to 503 on every ingestion submit. - # - # Class-2 drift recovery (each tick): in addition to comparing the - # annotation to the registry, we ALSO compare it to the live spec - # env. External actors — `kubectl rollout undo`, `kubectl edit`, - # certain `helm upgrade` flag combinations, GitOps reconcilers — - # can revert the spec env without touching the annotation. Without - # the env check, an `annotation == registry → no-op` branch would - # silently leave the deployment on a stale env forever. When env - # drift is detected (annotation == registry but spec env differs), - # we re-apply the annotation's value to the spec; we don't update - # the annotation, since it was already correct. - # - # Parsing: awk/sed/grep + jq. jq used only where JSON-with-dotted- - # keys or container/env-array filtering motivates it; the rest - # stays in pure shell so the script survives image swaps to leaner - # bases — same constraint as the auto-upgrade script in this chart. + # Parsing: awk/sed/grep + jq. jq used only where JSON-with-dotted-keys + # or container/env-array filtering motivates it; the rest stays in pure + # shell so the script survives image swaps to leaner bases — same + # constraint as the auto-upgrade script in this chart. # # All log() output goes to stderr so it never pollutes # command-substitution capture sites like `latest="$(...)"`. @@ -81,13 +49,6 @@ data: log "release=$RELEASE_NAME namespace=$RELEASE_NAMESPACE deployment=$DEPLOYMENT_NAME" log " client images: tracebloc/{jobs-manager,pods-monitor} on docker.io under tag=$IMAGE_TAG" - log " ingestor image: tracebloc/ingestor on ghcr.io under tag=$INGESTOR_TAG (pinned=$INGESTOR_PINNED)" - - # Consecutive failed ghcr ingestor-digest resolves before image-refresh - # escalates from WARN+skip to a failed Job (#186 #2). The chart injects - # this from imageRefresh.ingestorResolveFailureThreshold; default here - # too so a hand-edited CronJob missing the env var still runs under -u. - : "${INGESTOR_RESOLVE_FAIL_THRESHOLD:=3}" # Get an anonymous pull-scope token for one repository on a # registry. Both docker.io and ghcr.io support anonymous tokens for @@ -159,23 +120,6 @@ data: | jq -r --arg k "$_key" '.metadata.annotations[$k] // empty' } - # Read a container's env var value off the deployment. jq filter - # walks containers[name=$c].env[name=$e].value; returns empty - # string when the env var is absent OR explicitly empty (we treat - # both the same — the script's caller decides if empty is an - # error state). - get_container_env() { - _container="$1"; _env="$2" - kubectl get deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" -o json \ - | jq -r --arg c "$_container" --arg e "$_env" ' - .spec.template.spec.containers[] - | select(.name == $c) - | (.env // [])[] - | select(.name == $e) - | .value // empty - ' - } - # ================================================================= # Pass 1 — class-1 images: jobs-manager + pods-monitor. # docker.io, floating CLIENT_ENV tag, annotation-based source of @@ -248,195 +192,6 @@ data: $annotate_args --overwrite fi - # ================================================================= - # Pass 2 — class-2 image: ingestor. - # ghcr.io, floating INGESTOR_TAG, annotation-based source of truth, - # action = kubectl set env (which triggers a natural rollout via - # ReplicaSet rotation — no explicit `rollout restart` needed). - # - # Why annotation (not spec env) is the source of truth, despite - # the env being a more direct read of "what jobs-manager will use - # next": rollout-failure-then-retry safety. `kubectl set env` - # commits the new spec to etcd BEFORE `kubectl rollout status` - # waits for the rollout to complete. If the rollout times out or - # the new ReplicaSet's pods fail to come up, `set -eu` aborts the - # script — but the spec already matches the registry, so on the - # next tick `get_container_env` returns the new digest, compares - # equal to the registry, and skips, leaving the OLD ReplicaSet's - # pods running indefinitely with the old env. Caught in PR #159 - # review (bugbot, medium severity). - # - # With the annotation guarding the success path, the same - # rollout-then-annotate ordering as Pass 1 applies: annotate is - # the LAST step, only runs if `rollout status` succeeded. A failed - # rollout leaves the annotation stale → next tick sees mismatch - # → retries. - # - # First observation: there's no annotation yet but the chart has - # ALREADY populated INGESTOR_IMAGE_DIGEST with the chart-default - # digest (must be non-empty or jobs-manager 503s on every - # ingestion submit). We adopt that spec env as the recorded - # baseline — same "don't churn on install" principle as the - # jobs-manager first-observation contract. The corner case where - # the spec env is empty (corrupted state, manual kubectl edit, - # or stale chart values from --reuse-values) is handled by - # treating empty as a change-needed signal and filling from - # registry on first tick. - # ================================================================= - - log "checking ghcr.io/tracebloc/ingestor:${INGESTOR_TAG} (env=INGESTOR_IMAGE_DIGEST on container=api)" - - if [ "$INGESTOR_PINNED" = "1" ]; then - log " ingestor.autoRefresh: false in values; skipping (operator opted into pinning)" - else - latest_ingestor="$(get_latest_digest "tracebloc/ingestor" "$INGESTOR_TAG" "ghcr.io" || true)" - ingestor_fail_key="tracebloc.io/ingestor-refresh-consecutive-failures" - ingestor_err_key="tracebloc.io/ingestor-refresh-last-error" - if [ -z "$latest_ingestor" ]; then - # Could not resolve the ingestor digest from ghcr.io. A transient - # blip (rate-limit, momentary DNS) and a PERSISTENT failure (egress - # to ghcr.io firewalled, the ghcr token endpoint blocked, a proxy - # that allowlists docker.io but not ghcr.io) are indistinguishable - # on a single tick — so COUNT consecutive failures and escalate - # once they cross the threshold, instead of skipping silently - # forever. #186 (#2): berlin-team sat on the amd64-only baseline - # because every ingestor tick hit this branch while the docker.io - # images (jobs-manager, pods-monitor) refreshed fine — the CronJob - # looked healthy and nothing surfaced the ghcr failure. - prev_fails="$(get_annotation "$ingestor_fail_key" || true)" - case "$prev_fails" in ''|*[!0-9]*) prev_fails=0 ;; esac - if [ "$prev_fails" -ge "$INGESTOR_RESOLVE_FAIL_THRESHOLD" ]; then - # Already escalated on an earlier tick; stay loud (fail the Job) - # without inflating the counter — it caps at the threshold. - log " ERROR: still cannot resolve ghcr.io/tracebloc/ingestor:${INGESTOR_TAG} (>= ${INGESTOR_RESOLVE_FAIL_THRESHOLD} consecutive failures). Ingestor digest is NOT auto-refreshing; ingestion may be stuck on a stale image. Check egress to ghcr.io and its token endpoint from this namespace. See client#186." - exit 1 - fi - fails=$((prev_fails + 1)) - kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \ - "${ingestor_fail_key}=${fails}" --overwrite - if [ "$fails" -ge "$INGESTOR_RESOLVE_FAIL_THRESHOLD" ]; then - # Persistent. Record a human-readable last-error and fail the Job - # so it surfaces in `kubectl get cronjob` and monitoring — the - # same "failed Job = operator-visible" idiom Pass 2's stuck- - # rollout check already relies on. - kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \ - "${ingestor_err_key}=could not resolve ghcr.io/tracebloc/ingestor:${INGESTOR_TAG} for ${fails} consecutive ticks" --overwrite - log " ERROR: could not resolve ghcr.io/tracebloc/ingestor:${INGESTOR_TAG} for ${fails} consecutive ticks (threshold ${INGESTOR_RESOLVE_FAIL_THRESHOLD}). Ingestor digest is NOT auto-refreshing; ingestion may be stuck on a stale image (e.g. an amd64-only baseline on arm64 nodes). Check egress to ghcr.io and its token endpoint from this namespace. Failing the Job so this surfaces in monitoring. See client#186." - exit 1 - fi - log " WARN: could not resolve latest ingestor digest (failure ${fails}/${INGESTOR_RESOLVE_FAIL_THRESHOLD}; transient?); skipping this tick" - else - # Resolved OK. Clear any prior failure streak so a recovered - # registry/egress blip doesn't leave a stale failure annotation - # (and a future failure starts counting from zero again). - if [ -n "$(get_annotation "$ingestor_fail_key" || true)" ]; then - log " ingestor digest resolved; clearing prior failure streak" - kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \ - "${ingestor_fail_key}-" "${ingestor_err_key}-" >/dev/null 2>&1 || true - fi - recorded_ingestor="$(get_annotation "tracebloc.io/last-refreshed-ingestor-digest" || true)" - # Always read spec env too — the annotation alone isn't enough - # because external actors can revert the spec without touching - # the annotation: `kubectl rollout undo`, `kubectl edit - # deployment`, certain `helm upgrade` flag combinations, or a - # GitOps tool reconciling to its source. Without this read the - # annotation-matches-registry no-op branch would silently leave - # the deployment on a stale env forever. Caught in PR #159 - # review (bugbot, medium severity). - current_ingestor="$(get_container_env "api" "INGESTOR_IMAGE_DIGEST" || true)" - log " latest=$latest_ingestor" - log " recorded=${recorded_ingestor:-}" - log " current=${current_ingestor:-}" - - if [ -z "$recorded_ingestor" ]; then - # First observation. Adopt the spec env as the baseline if - # it's non-empty — same don't-churn-on-install principle as - # Pass 1. If it's empty (corrupted state), fall through to - # the change-detected path so we fill it from registry. - if [ -n "$current_ingestor" ]; then - log " first observation; adopting spec env ($current_ingestor) as baseline, no env change" - # Verify the deployment is in a stable state BEFORE - # annotating. The bug this guards against: a previous - # tick's empty-spec-fill branch can succeed at `kubectl - # set env` (which commits to etcd) but fail at `kubectl - # rollout status` and abort. The annotation stays unset. - # Next tick lands here — current_ingestor is non-empty - # because the spec committed — and without this status - # check, we would record success on a stuck rollout while - # running pods stayed on the old/empty env. Caught in PR - # #159 review (bugbot, high severity). `rollout status` - # is a near-instant no-op on a healthy deployment, so - # this adds essentially zero latency on the happy path. - kubectl rollout status -n "$RELEASE_NAMESPACE" "deployment/${DEPLOYMENT_NAME}" \ - --timeout="$ROLLOUT_TIMEOUT" - kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \ - "tracebloc.io/last-refreshed-ingestor-digest=${current_ingestor}" --overwrite - else - log " first observation; spec env empty (would 503 on ingestion submit); filling from registry" - kubectl set env -n "$RELEASE_NAMESPACE" "deployment/${DEPLOYMENT_NAME}" \ - -c api "INGESTOR_IMAGE_DIGEST=${latest_ingestor}" - kubectl rollout status -n "$RELEASE_NAMESPACE" "deployment/${DEPLOYMENT_NAME}" \ - --timeout="$ROLLOUT_TIMEOUT" - kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \ - "tracebloc.io/last-refreshed-ingestor-digest=${latest_ingestor}" --overwrite - log " ingestor refresh complete" - fi - elif [ "$recorded_ingestor" != "$latest_ingestor" ]; then - # Registry drift: the floating tag has moved. Set env to the - # new registry digest, wait for rollout, then annotate. Order - # matters: annotate is LAST so a failed rollout leaves the - # annotation stale and next tick retries (e7cf829). - log " digest changed (${recorded_ingestor} -> ${latest_ingestor}); set env + wait for rollout" - kubectl set env -n "$RELEASE_NAMESPACE" "deployment/${DEPLOYMENT_NAME}" \ - -c api "INGESTOR_IMAGE_DIGEST=${latest_ingestor}" - kubectl rollout status -n "$RELEASE_NAMESPACE" "deployment/${DEPLOYMENT_NAME}" \ - --timeout="$ROLLOUT_TIMEOUT" - kubectl annotate deployment -n "$RELEASE_NAMESPACE" "$DEPLOYMENT_NAME" \ - "tracebloc.io/last-refreshed-ingestor-digest=${latest_ingestor}" --overwrite - log " ingestor refresh complete" - elif [ "$current_ingestor" != "$recorded_ingestor" ]; then - # Env drift: registry hasn't moved (recorded == latest), but - # the spec env has been reverted externally to something else - # (could be empty, the chart default after --reuse-values, or - # an older digest from rollout undo). Re-apply the recorded - # value so the live deployment matches what we last decided. - # No annotate — the recorded value is already correct. - log " spec env drifted (recorded=${recorded_ingestor} but current=${current_ingestor:-}); re-applying recorded value" - kubectl set env -n "$RELEASE_NAMESPACE" "deployment/${DEPLOYMENT_NAME}" \ - -c api "INGESTOR_IMAGE_DIGEST=${recorded_ingestor}" - kubectl rollout status -n "$RELEASE_NAMESPACE" "deployment/${DEPLOYMENT_NAME}" \ - --timeout="$ROLLOUT_TIMEOUT" - log " ingestor env re-applied" - else - # Annotation, registry, and spec env all match. Normally - # this is a healthy no-op. But: a previous tick's - # env-drift branch could have committed a `kubectl set env` - # whose subsequent `kubectl rollout status` then failed - # (set -eu aborted the script). The spec change persisted - # past the abort, so on this tick annotation, registry, - # and spec env all match — but the actual rollout is - # stuck and running pods may be on a stale image. Without - # an explicit rollout-health check here, the no-op branch - # would silently mask that stuck state forever. - # - # `kubectl rollout status` is a near-instant no-op on a - # healthy deployment (returns immediately when there's no - # active rollout), so the cost on the happy path is - # negligible. On a stuck deployment it times out, `set -eu` - # aborts, and the Job is visibly failed in `kubectl get - # cronjob`. The operator then sees that image-refresh is - # stuck and can investigate (the underlying cause — - # broken image, registry-pull failure — is outside what - # image-refresh can autonomously fix). - # Caught in PR #162 review (bugbot, medium severity). - log " digest unchanged and spec env matches; verifying deployment health" - kubectl rollout status -n "$RELEASE_NAMESPACE" "deployment/${DEPLOYMENT_NAME}" \ - --timeout="$ROLLOUT_TIMEOUT" - log " fully in sync; no-op" - fi - fi - fi - log "tick complete" --- apiVersion: batch/v1 @@ -502,42 +257,16 @@ spec: value: {{ .Values.env.CLIENT_ENV | default "prod" | quote }} - name: ROLLOUT_TIMEOUT value: {{ .Values.imageRefresh.rolloutTimeout | quote }} - # Per-image opt-out flags. For class-1 images (jobs-manager - # / pods-monitor), pinning is signalled by setting - # `images..digest` to a non-empty value — same - # signal the deployment uses to switch imagePullPolicy - # to IfNotPresent. For the class-2 image (ingestor) the - # signal is the explicit `images.ingestor.autoRefresh: - # false` flag, because ingestor.digest must be non-empty - # for jobs-manager to function (see values.yaml comment - # block). All three guards are nil-safe. + # Per-image opt-out flags. Pinning a class-1 image is + # signalled by setting `images..digest` to a non-empty + # value — the same signal the deployment uses to switch + # imagePullPolicy to IfNotPresent. Both guards are nil-safe. + # (The ingestor is no longer refreshed here — it is spawned + # by jobs-manager from the floating tag; see header.) - name: JOBS_MANAGER_PINNED value: {{ ternary "1" "0" (not (empty (default dict (default dict .Values.images).jobsManager).digest)) | quote }} - name: PODS_MONITOR_PINNED value: {{ ternary "1" "0" (not (empty (default dict (default dict .Values.images).podsMonitor).digest)) | quote }} - # `default true ...` would mishandle the explicit-false - # case because Go templates treat `false` as falsy and - # `default` overrides it. Test `eq false` - # directly — absence (nil) and explicit `true` both - # fall through to "not pinned" (value="0"). - - name: INGESTOR_PINNED - value: {{ ternary "1" "0" (eq (default dict (default dict .Values.images).ingestor).autoRefresh false) | quote }} - # Floating ghcr.io tag polled for ingestor digest changes. - # The `default "0.3"` fallback handles --reuse-values - # upgrades from pre-v1.4.1 stored manifests that lack - # the key entirely. Must stay in sync with the chart - # default in values.yaml — both `prod` (the original - # default) would 404 on ghcr.io since the team uses - # semver-style float tags. Caught in PR #162 review. - - name: INGESTOR_TAG - value: {{ (default dict (default dict .Values.images).ingestor).tag | default "0.3" | quote }} - # Consecutive failed ghcr ingestor-digest resolves before - # image-refresh stops silently skipping and fails the Job - # loudly (#186 #2). nil-guarded with a `default 3` fallback - # so --reuse-values upgrades from pre-this-PR stored - # manifests (which lack the key) still render. - - name: INGESTOR_RESOLVE_FAIL_THRESHOLD - value: {{ (default dict .Values.imageRefresh).ingestorResolveFailureThreshold | default 3 | quote }} securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index 3570feb..23c1f25 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -93,13 +93,30 @@ spec: value: {{ include "tracebloc.clientLogsPvc" . | quote }} - name: MYSQL_HOST value: "mysql-client" - # client-runtime#40: default ingestor image digest for the - # POST /internal/submit-ingestion-run endpoint. Auto-upgrade - # keeps this current; the ingestor subchart no longer requires - # the customer to pin a digest. Nil-guarded so `--reuse-values` - # from a pre-this-PR release doesn't crash templating on the - # missing `images.ingestor` key — empty value means jobs-manager - # accepts only request-body overrides and returns 503 if absent. + # Ingestor image wiring for the POST /internal/submit-ingestion-run + # endpoint. jobs-manager spawns each ingestion Job from these values + # (see client-runtime submit_ingestion_run._build_image_reference). + # + # Default (digest empty): spawn by the floating tag with + # imagePullPolicy=Always, so every ingestion Job resolves the current + # published ingestor digest at spawn time — exactly how training pods + # are spawned. There is no chart-pinned digest to keep current, so a + # `helm upgrade` that resets values can no longer revert the ingestor + # image (the regression behind the customer "non-numeric" report). The + # image-refresh CronJob no longer touches the ingestor — its class-2 + # pass was retired in favour of this. + # + # Pinning is opt-in: set `images.ingestor.digest` to a canonical + # multi-arch digest to lock every ingestion Job to that exact image + # (imagePullPolicy then drops to IfNotPresent for reproducibility — + # debugging a release before rollout, certification, air-gapped repro). + # Repository + tag are overridable for private mirrors. All three are + # nil-guarded so `--reuse-values` from a release that predates the + # `images.ingestor` key still renders. + - name: INGESTOR_IMAGE_REPOSITORY + value: {{ (default dict .Values.images.ingestor).repository | default "ghcr.io/tracebloc/ingestor" | quote }} + - name: INGESTOR_IMAGE_TAG + value: {{ (default dict .Values.images.ingestor).tag | default "0.3" | quote }} - name: INGESTOR_IMAGE_DIGEST value: {{ (default dict .Values.images.ingestor).digest | default "" | quote }} - name: REQUESTS_PROXY_URL diff --git a/client/tests/image_refresh_test.yaml b/client/tests/image_refresh_test.yaml index 42f458d..06f2b4e 100644 --- a/client/tests/image_refresh_test.yaml +++ b/client/tests/image_refresh_test.yaml @@ -39,12 +39,13 @@ tests: - hasDocuments: count: 0 - - it: cronjob template renders nothing when ALL THREE images are pinned - # All three managed images opted out → nothing for image-refresh to - # do. ingestor's pin signal is `autoRefresh: false` (asymmetric - # with the other two, where `digest != ""` is the signal — see - # values.yaml comment for why ingestor needs an explicit flag). - # #158 — auto-refresh ingestor. + - it: cronjob template renders nothing when both class-1 images are pinned + # jobs-manager + pods-monitor are the only images this CronJob + # refreshes — the ingestor is spawned by jobs-manager from a floating + # tag, not refreshed here. When both class-1 images are digest-pinned + # there is nothing left to reconcile, so the CronJob renders nothing. + # A pinned image is signalled by a non-empty `digest` — the same + # signal the deployment uses to switch imagePullPolicy to IfNotPresent. template: templates/image-refresh-cronjob.yaml set: images: @@ -52,13 +53,11 @@ tests: digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" podsMonitor: digest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - ingestor: - autoRefresh: false asserts: - hasDocuments: count: 0 - - it: rbac template renders nothing when ALL THREE images are pinned + - it: rbac template renders nothing when both class-1 images are pinned template: templates/image-refresh-rbac.yaml set: images: @@ -66,23 +65,24 @@ tests: digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" podsMonitor: digest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - ingestor: - autoRefresh: false asserts: - hasDocuments: count: 0 - - it: cronjob STILL renders when only jobs-manager + pods-monitor pinned (ingestor unpinned) - # Regression guard for the asymmetric pin signal. After #158, BOTH - # jobs-manager AND pods-monitor pinned is NOT sufficient to skip - # rendering — ingestor still needs auto-refresh. + - it: cronjob STILL renders when the ingestor is pinned but a class-1 image is not + # Regression guard: the ingestor no longer gates this CronJob. Setting + # any ingestor field (here a pinned digest) must NOT make the CronJob + # disappear — as long as a class-1 image (here pods-monitor) can still + # drift, the CronJob must render. This is the inverse of the old + # "ingestor keeps the CronJob alive" behaviour, retired with the + # floating-tag migration. template: templates/image-refresh-cronjob.yaml set: images: jobsManager: digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - podsMonitor: - digest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ingestor: + digest: "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" asserts: - hasDocuments: count: 2 @@ -109,7 +109,9 @@ tests: content: name: PODS_MONITOR_PINNED value: "0" - - contains: + # The ingestor is no longer wired into this CronJob — no + # INGESTOR_PINNED / INGESTOR_TAG / INGESTOR_RESOLVE_FAIL_THRESHOLD env. + - notContains: path: spec.jobTemplate.spec.template.spec.containers[0].env content: name: INGESTOR_PINNED @@ -212,114 +214,25 @@ tests: - matchRegex: path: data["image-refresh.sh"] pattern: '(?s)kubectl rollout restart.*kubectl annotate deployment' - # #158 regression guards for ingestor (class-2) handling. - # Different action than jobs-manager/pods-monitor: ingestor uses - # `kubectl set env` on the api container's INGESTOR_IMAGE_DIGEST. - # Both registry hosts must appear (docker.io for class-1 via - # auth.docker.io; ghcr.io for class-2). The set-env action - # triggers a natural rollout via ReplicaSet rotation. - - matchRegex: - path: data["image-refresh.sh"] - pattern: "kubectl set env" - - matchRegex: + # The ingestor is spawned by jobs-manager from a floating tag and is + # no longer reconciled by this script — the entire class-2 pass was + # removed (no `kubectl set env` on INGESTOR_IMAGE_DIGEST, no ghcr.io + # poll for the spawned image, no last-refreshed-ingestor-digest + # annotation, no get_container_env helper). These guards lock the + # removal so a refactor can't quietly reintroduce the revert-prone + # mechanism the floating tag replaced. (The script's design-rationale + # comments still mention the retired mechanism as history, so these + # target the actual code shapes — the env assignment, the helper + # name, the annotation key — not the words.) + - notMatchRegex: path: data["image-refresh.sh"] pattern: 'INGESTOR_IMAGE_DIGEST=' - - matchRegex: - path: data["image-refresh.sh"] - pattern: "ghcr\\.io" - - matchRegex: - path: data["image-refresh.sh"] - pattern: "auth\\.docker\\.io" - # `get_container_env` reads - # spec.template.spec.containers[name=api].env[name=$e].value. - # Used on first-observation to adopt the chart-default env as - # baseline. Via jq because the same kubectl jsonpath - # bracket-notation issues from #156 apply to array-of-objects - # filtering with nested fields. - - matchRegex: + - notMatchRegex: path: data["image-refresh.sh"] pattern: "get_container_env" - - matchRegex: - path: data["image-refresh.sh"] - pattern: 'select\(\.name == \$c\)' - # Empty-env handling: jobs-manager 503s if INGESTOR_IMAGE_DIGEST - # is empty, so a fresh deployment where the env got reset must - # be re-filled from the registry rather than left empty. - - matchRegex: - path: data["image-refresh.sh"] - pattern: "would 503 on ingestion submit" - # Per-image opt-out flag for ingestor — INGESTOR_PINNED=1 → skip. - - matchRegex: - path: data["image-refresh.sh"] - pattern: 'ingestor\.autoRefresh: false in values' - # Ingestor uses an annotation as source of truth, NOT the live - # spec env. Caught in PR #159 review (bugbot, medium severity): - # if spec env were source of truth, a failed rollout after - # `kubectl set env` would leave the spec matching the registry - # while old ReplicaSet pods kept running with the old env. The - # next tick would see no drift and skip, leaving the deployment - # stuck. With annotation as source of truth, the annotation is - # updated ONLY after `rollout status` succeeds — a failed - # rollout leaves annotation stale → next tick retries. - - matchRegex: - path: data["image-refresh.sh"] - pattern: "tracebloc\\.io/last-refreshed-ingestor-digest" - # Order-of-operations for ingestor: set env → wait → annotate. - # The annotate MUST come last so a failed rollout leaves the - # annotation stale and the next tick retries. - - matchRegex: - path: data["image-refresh.sh"] - pattern: '(?s)kubectl set env.*rollout status.*kubectl annotate.*last-refreshed-ingestor-digest' - # First-observation positive test for ingestor: when no - # annotation exists yet AND the spec env is non-empty, the - # script must adopt the spec env as the baseline (don't-churn- - # on-install). Locks the branch that #159 review flagged as - # untested. Empty-spec-env branch is covered by the existing - # "would 503" regex above. - - matchRegex: - path: data["image-refresh.sh"] - pattern: 'first observation; adopting spec env' - # Adopt-as-baseline branch MUST run `kubectl rollout status` - # before `kubectl annotate`. Caught in #159 review (bugbot, - # high severity): if a previous tick's empty-spec-fill set env - # and then rollout failed, this tick would find current_ingestor - # non-empty (spec already committed) but the deployment stuck. - # Annotating without waiting would record success on a stuck - # rollout. The status check is near-instant on a healthy - # deployment, so the cost on the happy path is negligible. - - matchRegex: - path: data["image-refresh.sh"] - pattern: '(?s)adopting spec env.*rollout status.*kubectl annotate.*last-refreshed-ingestor-digest=\$\{current_ingestor\}' - # Env-drift recovery (#159 bugbot follow-up): the no-op branch - # MUST also read the spec env, not just compare annotation to - # registry. External actors (kubectl rollout undo, kubectl edit, - # GitOps reconciler) can revert the env without touching the - # annotation. Without the second comparison, drift would go - # undetected indefinitely. - - matchRegex: - path: data["image-refresh.sh"] - pattern: 'spec env drifted' - # The drift-recovery branch must re-apply the RECORDED value to - # the spec env, not the registry value (registry already matches - # recorded by definition of this branch). And it must NOT update - # the annotation — the recorded value is already correct, so an - # annotate is dead-write churn. - - matchRegex: - path: data["image-refresh.sh"] - pattern: 'INGESTOR_IMAGE_DIGEST=\$\{recorded_ingestor\}' - # No-op branch must verify rollout health, NOT just exit. A - # previous tick's env-drift `kubectl set env` could have - # committed a spec change whose `rollout status` then failed - # — annotation/registry/spec env all match (set env's spec - # write persists past the abort) but pods may be stuck. The - # health check makes that stuck state visible as a failed Job. - # Caught in PR #162 review (bugbot, medium severity). - - matchRegex: - path: data["image-refresh.sh"] - pattern: 'verifying deployment health' - - matchRegex: + - notMatchRegex: path: data["image-refresh.sh"] - pattern: '(?s)digest unchanged and spec env matches.*kubectl rollout status' + pattern: "last-refreshed-ingestor-digest" - it: CronJob has the right schedule, concurrency, and SA wiring template: templates/image-refresh-cronjob.yaml @@ -370,148 +283,10 @@ tests: content: name: IMAGE_TAG value: prod - # #158: ingestor-specific env vars. Default tag is "0.3" - # (matches team's ghcr.io publishing convention — see the - # dedicated test below for rationale). - - contains: - path: spec.jobTemplate.spec.template.spec.containers[0].env - content: - name: INGESTOR_TAG - value: "0.3" - - contains: - path: spec.jobTemplate.spec.template.spec.containers[0].env - content: - name: INGESTOR_PINNED - value: "0" - - - it: ingestor INGESTOR_PINNED flag flips to "1" when autoRefresh=false - # The asymmetric pin signal: ingestor uses `autoRefresh: false` - # rather than `digest != ""` (which is the signal for jobs-manager - # / pods-monitor) because ingestor.digest must always be non-empty - # for jobs-manager to function. Caught a bug during #158 dev: the - # naive `default true autoRefresh` rendering treats explicit - # `false` as falsy and flips back to true. Test pins the corrected - # `eq value false` idiom. - template: templates/image-refresh-cronjob.yaml - documentIndex: 1 - set: - images: - ingestor: - autoRefresh: false - asserts: - - contains: - path: spec.jobTemplate.spec.template.spec.containers[0].env - content: - name: INGESTOR_PINNED - value: "1" - - - it: ingestor INGESTOR_TAG defaults to "0.3" (semver float tag — team's ghcr.io publishing convention) - # Original default was "prod" which doesn't exist on the team's - # ghcr.io/tracebloc/ingestor repo. Caught in PR #162 review - # (bugbot, medium severity) — auto-refresh silently no-op'd - # every tick because manifest resolution 404'd. The team's - # actual publishing convention uses semver-style float tags - # (`0`, `0.3`); `0.3` is the conservative default (patch-only - # auto-track) and matches what dev was tested with. - template: templates/image-refresh-cronjob.yaml - documentIndex: 1 - asserts: - - contains: - path: spec.jobTemplate.spec.template.spec.containers[0].env - content: - name: INGESTOR_TAG - value: "0.3" - # Regression guard: must NOT regress back to `prod`. - - notContains: - path: spec.jobTemplate.spec.template.spec.containers[0].env - content: - name: INGESTOR_TAG - value: prod - - - it: ingestor INGESTOR_TAG falls back to "0.3" (not "prod") when images.ingestor.tag is absent - # Simulates a --reuse-values upgrade from a pre-v1.4.1 stored - # manifest that has no images.ingestor.tag key. The chart-default - # path (handled by the test above) renders "0.3" because the - # default flows through; this test exercises the RUNTIME FALLBACK - # inside the template (`| default "X"`) when the key is missing - # from stored values. Both paths must converge on "0.3". - # Caught in PR #162 review (bugbot, medium severity). - template: templates/image-refresh-cronjob.yaml - documentIndex: 1 - set: - images: - ingestor: - tag: null - asserts: - - contains: - path: spec.jobTemplate.spec.template.spec.containers[0].env - content: - name: INGESTOR_TAG - value: "0.3" - - - it: ingestor INGESTOR_TAG is overridable - template: templates/image-refresh-cronjob.yaml - documentIndex: 1 - set: - images: - ingestor: - tag: "staging" - asserts: - - contains: - path: spec.jobTemplate.spec.template.spec.containers[0].env - content: - name: INGESTOR_TAG - value: staging - - - it: ingestor INGESTOR_RESOLVE_FAIL_THRESHOLD defaults to "3" (#186 #2) - # Below this many consecutive ghcr resolve failures, image-refresh - # WARNs + skips; at/above it the script fails the Job loudly instead - # of silently leaving jobs-manager on a stale digest. - template: templates/image-refresh-cronjob.yaml - documentIndex: 1 - asserts: - - contains: - path: spec.jobTemplate.spec.template.spec.containers[0].env - content: - name: INGESTOR_RESOLVE_FAIL_THRESHOLD - value: "3" - - - it: ingestor INGESTOR_RESOLVE_FAIL_THRESHOLD falls back to "3" when imageRefresh key is absent - # --reuse-values upgrade from a pre-this-PR stored manifest lacks the - # key; the template's `| default 3` must still render a usable value. - template: templates/image-refresh-cronjob.yaml - documentIndex: 1 - set: - imageRefresh: - ingestorResolveFailureThreshold: null - asserts: - - contains: - path: spec.jobTemplate.spec.template.spec.containers[0].env - content: - name: INGESTOR_RESOLVE_FAIL_THRESHOLD - value: "3" - - - it: ingestor INGESTOR_RESOLVE_FAIL_THRESHOLD is overridable - template: templates/image-refresh-cronjob.yaml - documentIndex: 1 - set: - imageRefresh: - ingestorResolveFailureThreshold: 5 - asserts: - - contains: - path: spec.jobTemplate.spec.template.spec.containers[0].env - content: - name: INGESTOR_RESOLVE_FAIL_THRESHOLD - value: "5" - - - it: schema rejects ingestorResolveFailureThreshold below 1 - template: templates/image-refresh-cronjob.yaml - set: - imageRefresh: - ingestorResolveFailureThreshold: 0 - asserts: - - failedTemplate: - errorPattern: "must not be valid against schema|Must be greater than or equal to 1" + # The ingestor is spawned by jobs-manager from a floating tag, so this + # CronJob no longer carries INGESTOR_TAG / INGESTOR_PINNED / + # INGESTOR_RESOLVE_FAIL_THRESHOLD env (the dedicated ingestor-env tests + # were removed with the class-2 pass). - it: schema rejects ingestor.tag=latest template: templates/image-refresh-cronjob.yaml diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index 5bbd631..fc4720f 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -91,6 +91,59 @@ tests: name: JOB_IMAGE_HOST value: "docker.io/" + - it: spawns the ingestor by floating tag by default (revert-proof across helm upgrade) + # Default (no pinned digest): jobs-manager spawns each ingestion Job by + # the floating tag with imagePullPolicy=Always, so the cluster always + # runs the current published ingestor and a `helm upgrade` that resets + # values cannot revert it. Wires REPOSITORY + TAG + an empty DIGEST into + # client-runtime submit_ingestion_run._build_image_reference. + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_REPOSITORY + value: "ghcr.io/tracebloc/ingestor" + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_TAG + value: "0.3" + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_DIGEST + value: "" + + - it: pins the ingestor digest when images.ingestor.digest is set (opt-in reproducibility) + set: + images: + ingestor: + digest: "sha256:dfdaa7e633a8df0f46b403e9422eba5c16bdb7d9c39047e6d3738f9e38fbdba8" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_DIGEST + value: "sha256:dfdaa7e633a8df0f46b403e9422eba5c16bdb7d9c39047e6d3738f9e38fbdba8" + + - it: ingestor repository + tag are overridable for a private mirror / air-gapped registry + set: + images: + ingestor: + repository: "private.registry/ingestor" + tag: "0.4" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_REPOSITORY + value: "private.registry/ingestor" + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_TAG + value: "0.4" + # client-runtime#92: SINGLE_NODE gates jobs-manager's GPU->CPU pending # fallback; it defaults to hostPath.enabled so managed (dynamic-PVC) # clusters never auto-downgrade while installer/bare-metal single-host diff --git a/client/values.schema.json b/client/values.schema.json index ba93595..c5a16d3 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -299,22 +299,23 @@ }, "ingestor": { "type": "object", + "description": "Image used for spawned ingestion Jobs. jobs-manager builds the reference from repository/tag/digest (client-runtime submit_ingestion_run). Default (digest empty): spawn by the floating tag with imagePullPolicy=Always, so each Job resolves the current published digest at spawn — revert-proof across `helm upgrade`, mirroring how training pods are spawned. Set digest to pin (IfNotPresent).", "properties": { - "digest": { + "repository": { "type": "string", - "pattern": "^(sha256:[a-f0-9]{64})?$", - "description": "Canonical ghcr.io/tracebloc/ingestor digest (sha256:<64 hex>). Initial value baked into the chart; image-refresh dynamically updates the live env from this baseline (#158). Must be non-empty — an empty value would cause jobs-manager to 503 on every ingestion submission." + "minLength": 1, + "description": "Ingestor image repository. Override for a private mirror / air-gapped registry (must host the same tag/digest). Surfaces into jobs-manager as INGESTOR_IMAGE_REPOSITORY. Default ghcr.io/tracebloc/ingestor." }, "tag": { "type": "string", "minLength": 1, "not": { "const": "latest" }, - "description": "Floating ghcr.io/tracebloc/ingestor tag polled by image-refresh. `latest` is rejected — image-refresh behaviour must not drift." + "description": "Floating ghcr.io/tracebloc/ingestor tag spawned by default (imagePullPolicy=Always). `latest` is rejected — spawn behaviour must not drift. Surfaces as INGESTOR_IMAGE_TAG." }, - "autoRefresh": { - "type": "boolean", - "default": true, - "description": "Per-image opt-out for image-refresh. Default true. Set false to lock the env at `digest` — image-refresh skips this image. Asymmetric with jobsManager/podsMonitor (which use digest='' as the opt-in signal) because ingestor's digest must be non-empty for jobs-manager to function." + "digest": { + "type": "string", + "pattern": "^(sha256:[a-f0-9]{64})?$", + "description": "Optional opt-in pin (sha256:<64 hex>). Empty (default) = spawn by the floating `tag`. When set, every ingestion Job runs this exact image with imagePullPolicy=IfNotPresent (reproducibility). A pinned digest must be a multi-arch index (#186). Surfaces as INGESTOR_IMAGE_DIGEST." } } }, @@ -589,12 +590,6 @@ "pattern": "^[0-9]+(s|m|h)$", "description": "Passed to `kubectl rollout status --timeout`. Allow headroom for bare-metal image pull." }, - "ingestorResolveFailureThreshold": { - "type": "integer", - "minimum": 1, - "default": 3, - "description": "Consecutive failed ghcr.io ingestor-digest resolves before image-refresh fails the Job loudly instead of silently skipping. See #186 (#2). Higher tolerates flakier egress; very high keeps the old skip-forever behaviour." - }, "suspend": { "type": "boolean", "default": false, diff --git a/client/values.yaml b/client/values.yaml index ef3ff3b..8115d62 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -196,75 +196,59 @@ images: digest: "" # client-runtime#40 / client#125: the ingestor image is spawned by # jobs-manager at ingestion-submission time, not as a long-lived pod. - # The digest surfaces into jobs-manager as the INGESTOR_IMAGE_DIGEST - # env var, which it uses when spawning each ingestion Job. + # jobs-manager builds the image reference from the three keys below + # (repository / tag / digest) and spawns each ingestion Job from it — + # see client-runtime submit_ingestion_run._build_image_reference. # - # Lifecycle from v1.4.1 (#158): - # * The pinned `digest` below is the INITIAL value baked into the - # chart at release time — fresh installs land on this. Keep it - # non-empty so jobs-manager is usable from the moment the pod - # starts (an empty value would cause 503s on ingestion-submit - # until the next image-refresh tick). - # * The image-refresh CronJob then polls - # `ghcr.io/tracebloc/ingestor:` on its schedule and - # updates the LIVE deployment's INGESTOR_IMAGE_DIGEST env via - # `kubectl set env` whenever the registry serves a new digest. - # Each new ingestor image push to the floating tag is picked up - # within ~15 min — no chart release required for ingestor image - # bumps. - # * Helm's 3-way merge preserves image-refresh's writes across - # future `helm upgrade`s as long as the chart's pinned `digest` - # below doesn't change between chart versions. The pinned value - # stays as a fallback for greenfield installs; ongoing rollouts - # are owned by image-refresh. - # * Bumping `digest` below in a future chart release IS still valid - # and remains the explicit-baseline path (#161 was the first - # example: v0.3.0 → v0.3.1 for the MLM tokenizer + metadata - # parity fixes; see commit history for the rationale on any - # given bump). Helm will re-sync the env to the new pinned value - # on the next upgrade. Image-refresh reconverges on the registry - # digest on its next tick if the two differ. + # Default mechanism (revert-proof, always-current): + # * `digest` empty → jobs-manager spawns each ingestion Job by the + # floating `tag` with imagePullPolicy=Always. The kubelet resolves + # the tag's current digest on every spawn (a cheap manifest check; + # already-present layers are reused), so the cluster always runs the + # current published ingestor release. This mirrors how training pods + # are spawned and is immune to a `helm upgrade` resetting values — + # the failure mode that the old pinned-digest + image-refresh design + # suffered (a customer was stuck on a stale baseline after upgrade, + # surfacing as a "non-numeric" validation error on data that a newer + # ingestor handled correctly). + # * New ingestor releases under the same `tag` are picked up on the + # next ingestion submit — no chart release, no CronJob, no + # `kubectl set env`. The image-refresh CronJob's ingestor (class-2) + # pass was retired; image-refresh now only rolls the long-lived + # jobs-manager pod. # - # Customers can also override per-install via the ingestor subchart's - # `image.digest` for one-off testing, independent of this value. + # Opt-in pinning (reproducibility): + # * Set `digest` to a canonical digest to lock every ingestion Job to + # that exact image; jobs-manager then uses imagePullPolicy=IfNotPresent. + # Use for debugging a release before rollout, certification, or + # air-gapped repro. A single request can also be pinned via the submit + # endpoint's `image_digest` body field, independent of this value. + # * A pinned `digest` MUST be a multi-arch index (linux/amd64 + + # linux/arm64): an amd64-only pin breaks ingestion on arm64 hosts + # (Apple Silicon, Graviton) with "no match for platform" / + # ImagePullBackOff (#186; the amd64-only v0.3.1 pin in #160 was the + # original regression). The same applies to the floating `tag` — the + # team publishes the tag as a multi-arch index. Enforced by the + # `ingestor-multiarch` job in .github/workflows/helm-ci.yaml. ingestor: - # Current baseline digest: v0.3.6 (2026-06-08, from #227). Skips - # 0.3.3–0.3.5 to land on the latest 0.3.x patch, carrying the - # cumulative NULL / empty-value ingestion fixes on top of v0.3.2 — - # tracebloc/data-ingestors#150 (stop counting missing values as - # "non-numeric" in DataValidator — the regression a customer hit on - # the stale 0.3.2 baseline), #167 / #168 / #170 / #172 (VARCHAR/CHAR/ - # TEXT, DATE/TIME, JSON null + empty-string tolerance) and #176 / #179 - # (null-like values round-trip end-to-end as SQL NULL; Python bool not - # stringified). Bump only when greenfield installs should start on a - # different version; ongoing rollouts are managed by image-refresh. - # - # This digest MUST be a multi-arch index (linux/amd64 + linux/arm64). - # jobs-manager spawns the ingestor Job by this pinned digest, so an - # amd64-only pin breaks ingestion on arm64 hosts (Apple Silicon, - # Graviton) with "no match for platform" / ImagePullBackOff (#186; the - # amd64-only v0.3.1 pin in #160 was the regression). Enforced by the - # `ingestor-multiarch` job in .github/workflows/helm-ci.yaml. - digest: "sha256:dfdaa7e633a8df0f46b403e9422eba5c16bdb7d9c39047e6d3738f9e38fbdba8" - # Floating tag polled by image-refresh. The team's ghcr.io - # publishing convention uses semver-style float tags — `0` tracks - # the latest 0.x, `0.3` tracks the latest 0.3.x patch. Default is - # `0.3` (patch-only auto-track): a future 0.4 release will NOT be - # picked up automatically; the chart's pinned `digest` would need - # to bump and either explicitly move the chart's default tag to - # `0.4` or an operator would override here. Pick `0` if you want - # auto-tracking of minor releases too. `latest` is rejected by - # the schema — image-refresh behaviour must not drift. + # Image repository. Override for a private mirror / air-gapped registry + # (which must host the same `tag` / `digest`). Surfaces into jobs-manager + # as INGESTOR_IMAGE_REPOSITORY. + repository: "ghcr.io/tracebloc/ingestor" + # Floating tag spawned by default (imagePullPolicy=Always). The team's + # ghcr.io publishing convention uses semver-style float tags — `0` tracks + # the latest 0.x, `0.3` tracks the latest 0.3.x patch. `0.3` (patch-only + # auto-track) is the default: a future 0.4 release will NOT be picked up + # until this is moved to `0.4` (or set to `0` for minor auto-track too). + # `latest` is rejected by the schema. tag: "0.3" - # Per-image opt-out for image-refresh. Default true (auto-refresh - # active). Set to false to lock the env at the `digest` value above - # — image-refresh will skip this image and leave the deployment's - # INGESTOR_IMAGE_DIGEST env alone. Used by operators who need - # reproducible pinning for debugging, certification, or air-gapped - # mirroring. Unlike jobs-manager (where setting `digest` itself is - # the opt-in to pinning), ingestor needs an explicit flag because - # its `digest` must be non-empty for jobs-manager to function. - autoRefresh: true + # Opt-in pin. Empty (default) = spawn by the floating `tag` above. Set to + # a canonical multi-arch digest to lock all ingestion Jobs to one exact + # image (see the comment block above). Last known-good baseline, for easy + # pinning: v0.3.6 (cumulative NULL / empty-value ingestion fixes — + # data-ingestors#150/#167/#168/#170/#172/#176/#179): + # sha256:dfdaa7e633a8df0f46b403e9422eba5c16bdb7d9c39047e6d3738f9e38fbdba8 + digest: "" podsMonitor: digest: "" resourceMonitor: @@ -528,17 +512,6 @@ imageRefresh: # on bare-metal first-boot can take minutes, and the CronJob slot # shouldn't hold all night if a rollout genuinely wedges. rolloutTimeout: "10m" - # Consecutive failed ghcr.io ingestor-digest resolves before image-refresh - # stops silently skipping and fails the Job loudly (visible in - # `kubectl get cronjob` / monitoring, plus a `tracebloc.io/ingestor-refresh- - # last-error` annotation on the jobs-manager deployment). Default 3 ≈ 45 min - # at the 15-min schedule: long enough to ride out a transient registry blip, - # short enough to surface a persistent egress/proxy failure (e.g. a cluster - # that reaches docker.io but not ghcr.io — the #186 (#2) root cause). A - # failed resolve BELOW this count still just WARNs and skips the tick. - # Raise it for flakier egress; set it very high to keep the old - # skip-forever behaviour. Only affects the ingestor (ghcr) image. - ingestorResolveFailureThreshold: 3 # CronJob spec knobs. Override per-cluster if needed. suspend: false # Lower history limits than autoUpgrade — this Job runs ~96x/day, the diff --git a/ingestor/README.md b/ingestor/README.md index 736f73f..a6c6f1d 100644 --- a/ingestor/README.md +++ b/ingestor/README.md @@ -123,9 +123,9 @@ The customer never builds an image. The customer never writes a Dockerfile. The The ingestor has two independent update lifecycles, and customers usually only need to think about one. -**Image: always current, automatically.** New `ghcr.io/tracebloc/ingestor` releases roll out to your cluster via the parent `tracebloc/client` chart's auto-upgrade cronjob (`autoUpgrade.enabled: true`, default). The cronjob runs `helm repo update` + `helm upgrade tracebloc/client` daily, which writes the new digest into the `INGESTOR_IMAGE_DIGEST` env on the running `tracebloc-jobs-manager` deployment. Your next `helm install tracebloc/ingestor ...` uses the new image automatically — no digest to pin, no version to track, no redeploy of anything you've already installed. +**Image: always current, automatically.** jobs-manager spawns each ingestion Job by the floating `ghcr.io/tracebloc/ingestor` tag with `imagePullPolicy: Always`, so every run resolves the current published image at spawn time. New ingestor releases under that tag are picked up on your next ingestion — no digest to pin, no version to track, and no redeploy of anything you've already installed. A cluster `helm upgrade` cannot revert the image, because there is no pinned digest to reset (the failure mode the old `INGESTOR_IMAGE_DIGEST`-pinning design had). -**Chart: refresh your local cache before each install.** Helm's repo cache on _your workstation_ is independent of the cluster. The cluster's cronjob can refresh its own cache, but it cannot reach your laptop. Run `helm repo update` before each install to pick up new chart features (new values, new templates, new defaults). A stale cache still works — it just locks you out of chart-level options added since you last refreshed. **The image you run does not depend on the chart version**: jobs-manager picks the current `INGESTOR_IMAGE_DIGEST` regardless of which subchart version submitted the request. +**Chart: refresh your local cache before each install.** Helm's repo cache on _your workstation_ is independent of the cluster. Run `helm repo update` before each install to pick up new chart features (new values, new templates, new defaults). A stale cache still works — it just locks you out of chart-level options added since you last refreshed. **The image you run does not depend on the chart version**: jobs-manager spawns the current image by floating tag regardless of which subchart version submitted the request. This stratification is intentional. The image picks up bugfixes and security patches without anyone restating their dataset configs; the chart only changes when there's a real protocol or UX shift. @@ -141,12 +141,12 @@ Nothing to upgrade. The chart is fire-and-forget: each `helm install` POSTs once ## Pinning a specific image version -The dominant install path leaves `image.digest` empty and lets jobs-manager pick the cluster's current ingestor version (set by the parent client chart's `images.ingestor.digest`, kept current by the auto-upgrade cronjob). Override only when you have a specific reason: +The dominant install path leaves `image.digest` empty and lets jobs-manager spawn the cluster's current ingestor version by its floating tag (`imagePullPolicy: Always`). Override only when you have a specific reason: | Scenario | What to do | |---|---| | Reproducing an older ingestion run for audit / debugging | `--set image.digest=sha256:` | -| Testing a new ingestor release before cluster-wide rollout | `--set image.digest=sha256:` ahead of the auto-upgrade tick | +| Testing a new ingestor release before cluster-wide rollout | `--set image.digest=sha256:` | | Air-gapped mirror with frozen versions | Use both `--set image.repository=...` and `--set image.digest=sha256:...` | When set, the digest must be the full canonical form (`sha256:` + 64 lowercase hex chars). Tags like `v0.3.0` are rejected by jobs-manager. See the [data-ingestors releases page](https://github.com/tracebloc/data-ingestors/releases) for current digests. diff --git a/ingestor/values.yaml b/ingestor/values.yaml index 91a2ab1..343be25 100644 --- a/ingestor/values.yaml +++ b/ingestor/values.yaml @@ -20,22 +20,24 @@ ingestConfig: "" # -- Image of the official tracebloc/ingestor that will run the dataset. # # Default behaviour (client-runtime#40 / client#125): leave `digest` -# empty and let the cluster's auto-upgrade flow pick the version. -# jobs-manager reads `INGESTOR_IMAGE_DIGEST` (set by the tracebloc -# client chart's `images.ingestor.digest` value) and uses it when -# spawning the ingestor Job. New ingestor releases roll out -# automatically when the cluster's daily auto-upgrade cronjob bumps the -# chart — no per-dataset chart change needed. +# empty. jobs-manager then spawns the ingestor Job by its floating tag +# with imagePullPolicy=Always, so every run resolves the current +# published ingestor release at spawn time — no version to pin, and a +# cluster `helm upgrade` cannot revert it. (Repository + tag come from +# the parent tracebloc client chart's `images.ingestor` values.) # # Set `digest` here ONLY when you need to pin a specific version, e.g. # reproducing an older run or testing a new release before cluster-wide -# rollout. Must be the full canonical sha256 digest; tags are rejected. +# rollout. jobs-manager then runs that exact digest (IfNotPresent). Must +# be the full canonical sha256 digest; tags are rejected. image: # Repository defaults to the GHCR-published image. Customers running an - # air-gapped mirror override this. + # air-gapped mirror override this (and the parent client chart's + # images.ingestor.repository — that is what jobs-manager spawns from). repository: ghcr.io/tracebloc/ingestor - # Optional override. Empty (default) means "use whatever the cluster - # is currently rolling". When set must be sha256:<64 lowercase hex>. + # Optional per-request pin. Empty (default) = jobs-manager spawns the + # current published image by floating tag. When set must be + # sha256:<64 lowercase hex>. digest: "" # -- jobs-manager Service hostname + port to POST to. When left empty