From ccb699c9642756304dd2c597c4e91d9fda4b8c0e Mon Sep 17 00:00:00 2001 From: morazow Date: Wed, 26 Aug 2026 21:26:41 +0200 Subject: [PATCH 1/3] [server] Serve cluster health from standby coordinators for readiness probes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A standby coordinator rejects every RPC except apiVersions with NotCoordinatorLeaderException, so a Kubernetes readiness probe cannot tell a healthy standby from a wedged one (#4105). - Allow GET_CLUSTER_HEALTH through the standby gate. - Add optional is_leader / leader_elected fields to GetClusterHealthResponse: a standby answers with status UNKNOWN and its election state; the leader additionally reports its role. - Add a coordinator role to ClusterHealthReadinessCheck and readiness-check.sh: a leader is ready regardless of cluster color, a standby only when the group has an elected leader. A standby of an older version rejects the RPC; the probe maps that to the API-unsupported exit code (relevant for hand-run probes against a remote server — in the chart, probe and server ship in one image). --- .../src/main/resources/bin/readiness-check.sh | 18 +- .../rpc/netty/server/FlussRequestHandler.java | 7 +- fluss-rpc/src/main/proto/FlussApi.proto | 6 + .../CoordinatorLeaderElection.java | 29 ++++ .../coordinator/CoordinatorService.java | 22 ++- .../tools/ClusterHealthReadinessCheck.java | 155 ++++++++++++++---- .../CoordinatorHighAvailabilityITCase.java | 44 +++++ .../CoordinatorLeaderElectionTest.java | 34 ++++ .../ClusterHealthReadinessCheckTest.java | 124 ++++++++++++++ 9 files changed, 401 insertions(+), 38 deletions(-) create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/tools/ClusterHealthReadinessCheckTest.java diff --git a/fluss-dist/src/main/resources/bin/readiness-check.sh b/fluss-dist/src/main/resources/bin/readiness-check.sh index 7d2ac5d9a3f..1e4a4ea21f1 100644 --- a/fluss-dist/src/main/resources/bin/readiness-check.sh +++ b/fluss-dist/src/main/resources/bin/readiness-check.sh @@ -24,13 +24,17 @@ # # Two-step readiness probe for Kubernetes StatefulSet rolling upgrades: # -# Step 1 — Local TCP port check: verify this TabletServer process is alive +# Step 1 — Local TCP port check: verify this server process is alive # and has bound its RPC port. Fast, no external dependency. # -# Step 2 — Cluster health check: call the LOCAL TabletServer's Cluster -# Health API (which forwards to the Coordinator over the internal -# listener) and pass only if status is GREEN. -# YELLOW/RED/UNKNOWN means recovery is incomplete — block the upgrade. +# Step 2 — Cluster health check, role-dependent (READINESS_ROLE): +# tablet (default) — call the LOCAL TabletServer's Cluster Health +# API (which forwards to the Coordinator over the internal +# listener) and pass only if status is GREEN. YELLOW/RED/UNKNOWN +# means recovery is incomplete — block the upgrade. +# coordinator — call the LOCAL CoordinatorServer's Cluster Health +# API. A leader passes regardless of cluster color; a standby +# passes only when the coordinator group has an elected leader. # # Both steps must pass for the pod to be marked Ready. # @@ -41,6 +45,8 @@ # # Environment variables (set by helm template or container spec): # FLUSS_HOME - Fluss installation directory +# READINESS_ROLE - Role of the local server the probe talks to: +# "tablet" (default) or "coordinator". # READINESS_TIMEOUT_MS - Timeout for Health API call (default: 5000) # READINESS_TCP_HOST - TabletServer host the probe talks to. Defaults # to ${POD_IP} when set (the typical Kubernetes case where bind.listeners @@ -66,6 +72,7 @@ set -o pipefail # ---- Configuration ---- FLUSS_HOME="${FLUSS_HOME:-/opt/fluss}" +ROLE="${READINESS_ROLE:-tablet}" TIMEOUT_MS="${READINESS_TIMEOUT_MS:-5000}" # Probe the local sidecar tablet inside the same pod — it forwards the request # to the Coordinator internally, so the probe never has to know the @@ -145,6 +152,7 @@ run_recovery_check() { -Xmx64m \ -classpath "${classpath}" \ org.apache.fluss.server.tools.ClusterHealthReadinessCheck \ + --role "${ROLE}" \ --host "${TCP_HOST}" \ --port "${TCP_PORT}" \ --timeoutMs "${timeout_ms}" 2>&1) diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussRequestHandler.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussRequestHandler.java index 2ff5e0691bb..d3abc8bf4e9 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussRequestHandler.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussRequestHandler.java @@ -65,7 +65,12 @@ public void processRequest(FlussRequest request) { request.getPrincipal())); // check if the coordinator server is the current leader if the API is a coordinator // TODO: we should only check coordinator APIs instead of all APIs - if (isCoordinator && api.getApiKey() != ApiKeys.API_VERSIONS) { + // GET_CLUSTER_HEALTH is answerable by a standby coordinator: it reports the + // server's own role and election state so Kubernetes readiness probes can tell + // a healthy standby from a wedged one. + if (isCoordinator + && api.getApiKey() != ApiKeys.API_VERSIONS + && api.getApiKey() != ApiKeys.GET_CLUSTER_HEALTH) { if (!((CoordinatorGateway) service).isLeader()) { request.fail( new NotCoordinatorLeaderException( diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto index d03acb92fbe..4baf0224874 100644 --- a/fluss-rpc/src/main/proto/FlussApi.proto +++ b/fluss-rpc/src/main/proto/FlussApi.proto @@ -814,6 +814,12 @@ message GetClusterHealthResponse { required int32 num_leader_replicas = 3; required int32 active_leader_replicas = 4; required int32 status = 5; // PbClusterHealthStatus: GREEN=0, YELLOW=1, RED=2, UNKNOWN=3 + // Whether the answering coordinator is the current leader. A standby coordinator + // answers with is_leader=false, status=UNKNOWN and zeroed replica counts. + optional bool is_leader = 6; + // Whether the coordinator group currently has an elected leader (possibly another + // server). Lets a standby's readiness probe certify a functioning group. + optional bool leader_elected = 7; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorLeaderElection.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorLeaderElection.java index 0ba51dafe09..c913b995f89 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorLeaderElection.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorLeaderElection.java @@ -190,6 +190,35 @@ public boolean isLeader() { return !closing.get() && state == State.LEADER; } + /** + * Returns whether the coordinator group currently has an elected leader — this server or any + * other participant. Returns {@code false} when the state cannot be determined (e.g. ZooKeeper + * unreachable), because an unknown leader must not be reported as present to a readiness probe. + * + *

On a standby this performs a synchronous ZooKeeper read, so it is intended for + * probe-frequency callers only (the Kubernetes readiness probe calls it at most every few + * seconds), not for hot paths. + * + *

TODO: when leader initialization fails, {@code becomeLeader} transitions this server back + * to STANDBY but the {@link LeaderLatch} keeps the ZK leadership, so this method can report an + * elected leader while no functional leader exists until the latch is released or the session + * expires. Pre-existing election behavior; revisit together with latch relinquishing. + */ + public boolean isLeaderElected() { + if (closing.get()) { + return false; + } + if (state == State.LEADER) { + return true; + } + try { + return leaderLatch.getLeader().isLeader(); + } catch (Exception e) { + LOG.debug("Failed to read leader election state for server {}", serverId, e); + return false; + } + } + private void submitLeadershipEvent(Runnable leadershipEvent) { if (closing.get()) { return; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index 04119bb0fd9..04b674e8680 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -1584,12 +1584,32 @@ public CompletableFuture getClusterHealth( authorizer.authorize(currentSession(), OperationType.DESCRIBE, Resource.cluster()); } + if (!isLeader()) { + // A standby has no CoordinatorContext (the event processor is leader-only), so it + // answers from its own election state: status UNKNOWN, zeroed counts, and the role + // fields a readiness probe needs to tell a healthy standby from a wedged one. + return CompletableFuture.completedFuture(computeStandbyClusterHealth()); + } + AccessContextEvent event = - new AccessContextEvent<>(CoordinatorService::computeClusterHealth); + new AccessContextEvent<>( + ctx -> computeClusterHealth(ctx).setIsLeader(true).setLeaderElected(true)); eventManagerSupplier.get().put(event); return event.getResultFuture(); } + private GetClusterHealthResponse computeStandbyClusterHealth() { + GetClusterHealthResponse response = new GetClusterHealthResponse(); + response.setNumReplicas(0); + response.setInSyncReplicas(0); + response.setNumLeaderReplicas(0); + response.setActiveLeaderReplicas(0); + response.setStatus(3); // PbClusterHealthStatus.UNKNOWN + response.setIsLeader(false); + response.setLeaderElected(coordinatorLeaderElection.isLeaderElected()); + return response; + } + @VisibleForTesting static GetClusterHealthResponse computeClusterHealth(CoordinatorContext ctx) { GetClusterHealthResponse response = new GetClusterHealthResponse(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tools/ClusterHealthReadinessCheck.java b/fluss-server/src/main/java/org/apache/fluss/server/tools/ClusterHealthReadinessCheck.java index 6e9c70ae13d..65d1794cabb 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tools/ClusterHealthReadinessCheck.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tools/ClusterHealthReadinessCheck.java @@ -21,10 +21,13 @@ import org.apache.fluss.cluster.ServerType; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.NotCoordinatorLeaderException; import org.apache.fluss.exception.UnsupportedVersionException; import org.apache.fluss.metrics.registry.MetricRegistryImpl; import org.apache.fluss.rpc.GatewayClientProxy; import org.apache.fluss.rpc.RpcClient; +import org.apache.fluss.rpc.gateway.AdminReadOnlyGateway; +import org.apache.fluss.rpc.gateway.CoordinatorGateway; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.messages.GetClusterHealthRequest; import org.apache.fluss.rpc.messages.GetClusterHealthResponse; @@ -32,18 +35,25 @@ import org.apache.fluss.utils.ExceptionUtils; import java.util.Collections; +import java.util.Locale; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; /** - * Lightweight readiness-check CLI for Fluss tablet-server pods. + * Lightweight readiness-check CLI for Fluss server pods. * - *

The probe connects to a tablet server (typically the local one on {@code 127.0.0.1}) and - * issues a single {@code getClusterHealth} call. The tablet server forwards the request to the - * coordinator over its internal listener and returns the cluster-wide health snapshot. This means - * the readiness probe never has to know the coordinator's address and survives coordinator pod - * restarts cleanly. + *

In the default {@code tablet} role, the probe connects to a tablet server (typically the local + * one on {@code 127.0.0.1}) and issues a single {@code getClusterHealth} call. The tablet server + * forwards the request to the coordinator over its internal listener and returns the cluster-wide + * health snapshot. This means the readiness probe never has to know the coordinator's address and + * survives coordinator pod restarts cleanly. Readiness requires status GREEN. + * + *

In the {@code coordinator} role, the probe connects to the local coordinator server. A leader + * coordinator is Ready regardless of cluster color — its readiness must not depend on tablet + * health, and the tablet probes already gate on cluster recovery. A standby coordinator answers + * with its role and election state and is Ready only when the coordinator group currently has an + * elected leader, so "all coordinator pods Ready" certifies a functioning group. * *

Inputs

* @@ -53,9 +63,11 @@ * * * *

This API is designed for the situation like a Kubernetes readiness-probe gate during * rolling upgrades: only proceed to the next pod when the status is {@code GREEN}, ensuring all * replicas have fully recovered before the next server is restarted. * + *

Unlike other admin operations, a standby coordinator answers this request instead + * of rejecting it with a {@code NotCoordinatorLeaderException} (so a readiness probe can gate + * on it). If the client's coordinator address is stale — e.g. during a coordinator failover, + * before the client's metadata refreshes — the returned {@link ClusterHealth} carries {@link + * ClusterHealth#isServedByLeader()} {@code false}, status {@code UNKNOWN}, and zeroed replica + * counts. Callers that monitor cluster health should check {@link + * ClusterHealth#isServedByLeader()} before interpreting the counts. + * * @return a {@link CompletableFuture} that completes with the cluster health information. * @since 1.0 */ diff --git a/fluss-client/src/main/java/org/apache/fluss/client/admin/ClusterHealth.java b/fluss-client/src/main/java/org/apache/fluss/client/admin/ClusterHealth.java index 8f751bc1691..266b2512215 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/admin/ClusterHealth.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/admin/ClusterHealth.java @@ -24,6 +24,11 @@ /** * Cluster health information returned by {@link Admin#getClusterHealth()}. * + *

A standby coordinator answers this request instead of rejecting it (so Kubernetes readiness + * probes can gate on it). A response served by a standby carries {@link #isServedByLeader()} {@code + * false}, status {@link ClusterHealthStatus#UNKNOWN}, and zeroed replica counts — callers that + * monitor cluster health should check {@link #isServedByLeader()} before interpreting the counts. + * * @since 1.0 */ @PublicEvolving @@ -34,18 +39,24 @@ public final class ClusterHealth { private final int numLeaderReplicas; private final int activeLeaderReplicas; private final ClusterHealthStatus status; + private final boolean servedByLeader; + private final boolean leaderElected; public ClusterHealth( int numReplicas, int inSyncReplicas, int numLeaderReplicas, int activeLeaderReplicas, - ClusterHealthStatus status) { + ClusterHealthStatus status, + boolean servedByLeader, + boolean leaderElected) { this.numReplicas = numReplicas; this.inSyncReplicas = inSyncReplicas; this.numLeaderReplicas = numLeaderReplicas; this.activeLeaderReplicas = activeLeaderReplicas; this.status = Objects.requireNonNull(status, "status"); + this.servedByLeader = servedByLeader; + this.leaderElected = leaderElected; } public int getNumReplicas() { @@ -68,6 +79,26 @@ public ClusterHealthStatus getStatus() { return status; } + /** + * Whether the coordinator that answered is the current leader. {@code false} means a standby + * answered (e.g. the client's coordinator address was stale during a failover): the status is + * {@link ClusterHealthStatus#UNKNOWN} and the replica counts are zero, not cluster facts. + * Responses from servers that predate this field report {@code true} — a standby of those + * versions rejects the request instead of answering. + */ + public boolean isServedByLeader() { + return servedByLeader; + } + + /** + * Whether the coordinator group currently has an elected leader — the answering server or any + * other participant. Only meaningful when {@link #isServedByLeader()} is {@code false}; a + * leader-served response always reports {@code true}. + */ + public boolean isLeaderElected() { + return leaderElected; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -81,13 +112,21 @@ public boolean equals(Object o) { && inSyncReplicas == that.inSyncReplicas && numLeaderReplicas == that.numLeaderReplicas && activeLeaderReplicas == that.activeLeaderReplicas - && status == that.status; + && status == that.status + && servedByLeader == that.servedByLeader + && leaderElected == that.leaderElected; } @Override public int hashCode() { return Objects.hash( - numReplicas, inSyncReplicas, numLeaderReplicas, activeLeaderReplicas, status); + numReplicas, + inSyncReplicas, + numLeaderReplicas, + activeLeaderReplicas, + status, + servedByLeader, + leaderElected); } @Override @@ -103,6 +142,10 @@ public String toString() { + activeLeaderReplicas + ", status=" + status + + ", servedByLeader=" + + servedByLeader + + ", leaderElected=" + + leaderElected + '}'; } } diff --git a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java index 3c7512945dc..24d1fb8630e 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java @@ -882,12 +882,18 @@ public static GetTableStatsRequest makeGetTableStatsRequest(List bu } public static ClusterHealth toClusterHealth(GetClusterHealthResponse resp) { + // A response without is_leader comes from a leader that predates the field — a standby + // of those versions rejects the request instead of answering. + boolean servedByLeader = !resp.hasIsLeader() || resp.isIsLeader(); + boolean leaderElected = resp.hasLeaderElected() ? resp.isLeaderElected() : servedByLeader; return new ClusterHealth( resp.getNumReplicas(), resp.getInSyncReplicas(), resp.getNumLeaderReplicas(), resp.getActiveLeaderReplicas(), - toClusterHealthStatus(resp.getStatus())); + toClusterHealthStatus(resp.getStatus()), + servedByLeader, + leaderElected); } private static ClusterHealthStatus toClusterHealthStatus(int pbStatus) { From 51147b61adc9da0390649c83a4a3e717c051c8c7 Mon Sep 17 00:00:00 2001 From: morazow Date: Wed, 26 Aug 2026 21:26:56 +0200 Subject: [PATCH 3/3] [helm] First-class coordinator HA: standby-aware readiness gate - Replace the coordinator TCP readiness probe with the role-aware cluster-health exec probe (liveness stays TCP); knobs under coordinator.readinessProbe.* mirror the tablet ones, including healthCheckAuth for SASL-enforcing client listeners. - Document coordinator high availability: multi-replica client bootstrap, PodDisruptionBudget, rolling-update behavior, probe semantics, and the version-skew behavior (an older image burns the healthCheckTimeoutSeconds budget per standby pod before the TCP fallback latches). Defaults stay at one replica. Closes #4105. --- helm/templates/sts-coordinator.yaml | 31 ++++-- helm/tests/readiness_probe_test.yaml | 102 ++++++++++++++++++ helm/values.yaml | 46 ++++++++ .../install-deploy/deploying-with-helm.md | 82 +++++++++++--- 4 files changed, 243 insertions(+), 18 deletions(-) create mode 100644 helm/tests/readiness_probe_test.yaml diff --git a/helm/templates/sts-coordinator.yaml b/helm/templates/sts-coordinator.yaml index 583cfcd9cbd..0deb4e71108 100644 --- a/helm/templates/sts-coordinator.yaml +++ b/helm/templates/sts-coordinator.yaml @@ -140,12 +140,31 @@ spec: tcpSocket: port: {{ .Values.listeners.client.port }} readinessProbe: - failureThreshold: 100 - timeoutSeconds: 1 - initialDelaySeconds: 10 - periodSeconds: 3 - tcpSocket: - port: {{ .Values.listeners.client.port }} + failureThreshold: {{ .Values.coordinator.readinessProbe.failureThreshold | default 360 }} + timeoutSeconds: {{ .Values.coordinator.readinessProbe.timeoutSeconds | default 10 }} + initialDelaySeconds: {{ .Values.coordinator.readinessProbe.initialDelaySeconds | default 15 }} + periodSeconds: {{ .Values.coordinator.readinessProbe.periodSeconds | default 5 }} + exec: + command: + - /bin/bash + - -c + - | + export FLUSS_SERVER_ID=${POD_NAME##*-} + # Probe the LOCAL coordinator: a leader passes regardless of cluster + # color; a standby passes only when the coordinator group currently + # has an elected leader — so a standby that binds its port but never + # joins the group stays NotReady instead of looking healthy via TCP. + export READINESS_ROLE=coordinator + export READINESS_TIMEOUT_MS={{ .Values.coordinator.readinessProbe.rpcTimeoutMs | default 5000 }} + export READINESS_TCP_PORT={{ .Values.listeners.client.port }} + # bind.listeners binds the coordinator's CLIENT endpoint to ${POD_IP}, + # not 0.0.0.0, so we must probe the pod's own IP instead of 127.0.0.1. + export READINESS_TCP_HOST=${POD_IP} + export READINESS_HEALTH_CHECK_TIMEOUT_SECONDS={{ .Values.coordinator.readinessProbe.healthCheckTimeoutSeconds | default 1200 }} + {{- with .Values.coordinator.readinessProbe.healthCheckAuth }} + export READINESS_HEALTH_CHECK_AUTH={{ . | quote }} + {{- end }} + exec $FLUSS_HOME/bin/readiness-check.sh resources: {{- toYaml .Values.resources.coordinatorServer | nindent 12 }} volumeMounts: diff --git a/helm/tests/readiness_probe_test.yaml b/helm/tests/readiness_probe_test.yaml new file mode 100644 index 00000000000..fe9caad3d11 --- /dev/null +++ b/helm/tests/readiness_probe_test.yaml @@ -0,0 +1,102 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: readiness probes +templates: + - templates/sts-coordinator.yaml + - templates/sts-tablet.yaml +tests: + - it: coordinator readiness is an exec probe in the coordinator role by default + template: templates/sts-coordinator.yaml + asserts: + - isNotEmpty: + path: spec.template.spec.containers[0].readinessProbe.exec + - equal: + path: spec.template.spec.containers[0].readinessProbe.failureThreshold + value: 360 + - equal: + path: spec.template.spec.containers[0].readinessProbe.periodSeconds + value: 5 + - matchRegex: + path: spec.template.spec.containers[0].readinessProbe.exec.command[2] + pattern: "READINESS_ROLE=coordinator" + - matchRegex: + path: spec.template.spec.containers[0].readinessProbe.exec.command[2] + pattern: "READINESS_TCP_PORT=9124" + - notMatchRegex: + path: spec.template.spec.containers[0].readinessProbe.exec.command[2] + pattern: "READINESS_HEALTH_CHECK_AUTH" + - it: coordinator liveness stays a TCP probe + template: templates/sts-coordinator.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.tcpSocket.port + value: 9124 + - isNull: + path: spec.template.spec.containers[0].livenessProbe.exec + - it: coordinator readiness knobs are overridable + template: templates/sts-coordinator.yaml + set: + coordinator: + readinessProbe: + failureThreshold: 42 + timeoutSeconds: 3 + initialDelaySeconds: 7 + periodSeconds: 11 + rpcTimeoutMs: 2500 + healthCheckTimeoutSeconds: 300 + asserts: + - equal: + path: spec.template.spec.containers[0].readinessProbe.failureThreshold + value: 42 + - equal: + path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds + value: 3 + - equal: + path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds + value: 7 + - equal: + path: spec.template.spec.containers[0].readinessProbe.periodSeconds + value: 11 + - matchRegex: + path: spec.template.spec.containers[0].readinessProbe.exec.command[2] + pattern: "READINESS_TIMEOUT_MS=2500" + - matchRegex: + path: spec.template.spec.containers[0].readinessProbe.exec.command[2] + pattern: "READINESS_HEALTH_CHECK_TIMEOUT_SECONDS=300" + - it: coordinator readiness passes healthCheckAuth through when set + template: templates/sts-coordinator.yaml + set: + coordinator: + readinessProbe: + healthCheckAuth: "client.security.protocol:SASL;client.sasl.mechanism:PLAIN" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].readinessProbe.exec.command[2] + pattern: "READINESS_HEALTH_CHECK_AUTH=\"client.security.protocol:SASL;client.sasl.mechanism:PLAIN\"" + - it: tablet readiness keeps its defaults and does not set a role + template: templates/sts-tablet.yaml + asserts: + - isNotEmpty: + path: spec.template.spec.containers[0].readinessProbe.exec + - equal: + path: spec.template.spec.containers[0].readinessProbe.failureThreshold + value: 360 + - notMatchRegex: + path: spec.template.spec.containers[0].readinessProbe.exec.command[2] + pattern: "READINESS_ROLE" diff --git a/helm/values.yaml b/helm/values.yaml index de9c8eb42e0..1113ca4c403 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -121,11 +121,57 @@ tablet: topologySpreadConstraints: [] coordinator: + # The default stays 1. Running more than one replica gives you warm standbys + # that take over via leader election on coordinator failure — see the + # "Coordinator High Availability" section of the deployment documentation + # (website/docs/install-deploy/deploying-with-helm.md) for PDB, + # rolling-update, and client bootstrap guidance. numberOfReplicas: 1 storage: enabled: false size: 1Gi storageClass: + # Readiness probe configuration. The probe talks to the LOCAL coordinator + # (POD_IP:client-port) and asks for its role and election state: + # - the leader is Ready regardless of cluster color (its readiness must not + # depend on tablet health; the tablet probes gate on cluster recovery), + # - a standby is Ready only when the coordinator group currently has an + # elected leader. + # The probe script ships inside the Fluss image, so this chart needs an image + # with role-aware cluster health. With an older image, a standby coordinator + # stays NotReady until healthCheckTimeoutSeconds expires and the probe + # latches TCP-only readiness — the rollout completes, but each standby pod + # burns the full budget first. + readinessProbe: + # Timeout in ms for the RPC call. + rpcTimeoutMs: 5000 + # Standard Kubernetes probe parameters. + # + # The total time Kubernetes will wait before flipping the pod to NotReady is + # initialDelaySeconds + failureThreshold * periodSeconds + # = 15 + 360 * 5 = 1815s (~30 minutes) by default. + # IMPORTANT: this window MUST be larger than healthCheckTimeoutSeconds + # below, otherwise Kubernetes will mark the pod NotReady BEFORE the + # in-script fallback has a chance to latch the TCP fast path. If you tune + # healthCheckTimeoutSeconds upward, raise failureThreshold accordingly so + # the inequality (failureThreshold * periodSeconds > healthCheckTimeoutSeconds) + # always holds. + failureThreshold: 360 + timeoutSeconds: 10 + initialDelaySeconds: 15 + periodSeconds: 5 + # Maximum wall-clock seconds the election gate is allowed to keep a + # freshly-created coordinator pod NotReady on its first boot. After this + # budget is exhausted the probe latches the TCP-only fast path so a broken + # election cannot wedge a rolling upgrade indefinitely. + # Default: 1200 (20 minutes). + healthCheckTimeoutSeconds: 1200 + # Optional client auth configuration passed to the probe's RpcClient as a + # single-line string of semicolon-separated `key:value` pairs (no quoting + # or escaping needed). Required only when the local client listener + # enforces SASL. Example: + # healthCheckAuth: "client.security.protocol:SASL;client.sasl.mechanism:PLAIN;client.security.sasl.username:admin;client.security.sasl.password:admin-pass" + healthCheckAuth: "" extraVolumes: [] extraVolumeMounts: [] initContainers: [] diff --git a/website/docs/install-deploy/deploying-with-helm.md b/website/docs/install-deploy/deploying-with-helm.md index 4a483e14a74..1e3373045c9 100644 --- a/website/docs/install-deploy/deploying-with-helm.md +++ b/website/docs/install-deploy/deploying-with-helm.md @@ -442,6 +442,12 @@ making rotation fully hands-off. |-----------|-------------|---------| | `tablet.numberOfReplicas` | Number of TabletServer replicas to deploy | `3` | +### Coordinator Server Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `coordinator.numberOfReplicas` | Number of CoordinatorServer replicas to deploy. Replicas beyond the elected leader run as warm standbys — see [Coordinator High Availability](#coordinator-high-availability) | `1` | + ### Scheduling Parameters | Parameter | Description | Default | @@ -870,14 +876,71 @@ tablet: | Parameter | Default | Description | |--------------------|---------|------------------------------------------------------------------------------------------------------------------------------------------------| | `rpcTimeoutMs` | `5000` | Timeout in milliseconds for the RPC call to the Coordinator. | -| `failureThreshold` | `200` | Max consecutive probe failures before marking the pod as unready. With `periodSeconds=5`, this allows up to ~16 minutes for recovery. | +| `failureThreshold` | `360` | Max consecutive probe failures before marking the pod as unready. With `periodSeconds=5`, this allows up to ~30 minutes for recovery. | | `periodSeconds` | `5` | How often the probe runs. | :::note -The CoordinatorServer does not need the Cluster Health API probe — it does not host data replicas, so a simple TCP check is sufficient. The Coordinator should be upgraded **after** all TabletServers are fully upgraded and recovered. ::: +### Coordinator High Availability + +CoordinatorServer high availability is supported: with `coordinator.numberOfReplicas` greater +than `1`, one replica is elected leader and the others run as warm standbys that take over via +leader election when the leader fails. The default stays `1` — raising it is an explicit choice. + +```yaml +coordinator: + numberOfReplicas: 3 + podDisruptionBudget: + enabled: true + maxUnavailable: 1 +``` + +#### Coordinator Readiness Probe + +A standby coordinator binds the same client port as the leader, so a TCP check cannot tell a +healthy standby from a wedged one. The coordinator readiness probe instead asks the local server +for its role and election state: + +- The **leader** is Ready regardless of cluster health color. Its readiness controls the DNS + record clients bootstrap against, so it must not depend on TabletServer health — the + TabletServer probes already gate rolling upgrades on cluster recovery. +- A **standby** is Ready only when the coordinator group currently has an elected leader. "All + coordinator pods Ready" therefore certifies a functioning group, and a rolling update will not + proceed past a pod whose group has no leader. + +The gate runs on a pod's first boot; once it has passed (or its +`healthCheckTimeoutSeconds` budget is exhausted), the probe latches a cheap TCP-only check for +the rest of the pod's lifetime, so a later ZooKeeper blip cannot flip all coordinators NotReady +at once. The probe parameters mirror the TabletServer ones under +`coordinator.readinessProbe.*`, including `healthCheckAuth` for clusters whose client listener +enforces SASL. + +Version skew: the probe script ships inside the Fluss image, so this chart version needs an +image that supports role-aware cluster health. With an **older image** (one that already ships +`readiness-check.sh` but without role support), a standby coordinator stays NotReady until the +`healthCheckTimeoutSeconds` budget (20 minutes by default) expires and the probe latches the +TCP-only fallback — the rollout completes, but each standby pod burns the full budget first. +Images that predate `readiness-check.sh` entirely never become Ready under this chart. Keep the +chart and image versions in step, or lower `healthCheckTimeoutSeconds` for the transition +rollout. An **older chart** with a newer image keeps its TCP-only coordinator probe and is +unaffected. + +#### Running More Than One Replica + +- **Client bootstrap:** list every coordinator replica in `bootstrap.servers`, e.g. + `coordinator-server-0.coordinator-server-hs.:9124,coordinator-server-1.coordinator-server-hs.:9124,coordinator-server-2.coordinator-server-hs.:9124`. + A standby answers client requests with `NotCoordinatorLeaderException`; clients retry other + bootstrap addresses, so any live replica in the list keeps bootstrap working during failover. +- **PodDisruptionBudget:** enable `coordinator.podDisruptionBudget` with `maxUnavailable: 1` so + voluntary disruptions (node drains) never take down more than one coordinator at a time. +- **Rolling updates:** the StatefulSet updates one pod at a time from the highest ordinal down. + When the leader is restarted, a standby takes over via leader election; the restarted pod + rejoins as a standby and flips Ready once it sees the elected leader. +- **Spreading:** use `coordinator.affinity` or `coordinator.topologySpreadConstraints` to keep + replicas on separate nodes or zones — colocated replicas share their failure domain. + ## Custom Container Images ### Building Custom Images @@ -912,7 +975,7 @@ image: ### Health Checks -The chart includes liveness and readiness probes. By default, both use TCP socket checks: +The chart includes liveness and readiness probes. Liveness uses a TCP socket check: ```yaml livenessProbe: @@ -921,17 +984,12 @@ livenessProbe: initialDelaySeconds: 10 periodSeconds: 3 failureThreshold: 100 - -readinessProbe: - tcpSocket: - port: 9124 - initialDelaySeconds: 10 - periodSeconds: 3 - failureThreshold: 100 ``` -For TabletServers, you can enable the Cluster Health readiness probe for safe rolling upgrades. -See [Cluster Health Readiness Probe](#cluster-health-readiness-probe) for details. +Readiness is role-aware: TabletServers gate on cluster health for safe rolling upgrades (see +[Cluster Health Readiness Probe](#cluster-health-readiness-probe)), and CoordinatorServers gate +on leader/standby election state (see +[Coordinator Readiness Probe](#coordinator-readiness-probe)). ### Logs