From 45808f7d9f41e657189979c2ed114dbda178d87d Mon Sep 17 00:00:00 2001 From: Oleksandr Piskun Date: Tue, 8 Sep 2026 16:23:32 +0000 Subject: [PATCH 1/2] fix(k8s): fail fast in /info when the API server does not answer The /info probe inherited the 60 s K8S_HTTP_TIMEOUT. AppAPI waits 5 s for /info in its daemon checks, so an unreachable API server surfaced as "Cannot reach HaRP" after a 5 s hang on every admin page load. The probe now has its own 3 s timeout and /info reports why the cluster is unreachable in a new "error" field. Signed-off-by: Oleksandr Piskun --- haproxy_agent.py | 49 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/haproxy_agent.py b/haproxy_agent.py index 8676ec1..065e534 100644 --- a/haproxy_agent.py +++ b/haproxy_agent.py @@ -58,6 +58,9 @@ K8S_API_SERVER = f"https://{host}:{port}" K8S_HTTP_TIMEOUT = aiohttp.ClientTimeout(total=60.0) +# `/info` probes the API server with its own short timeout: AppAPI waits 5 s for `/info` in its daemon checks, so a +# probe that inherits K8S_HTTP_TIMEOUT makes an unreachable API server look like an unreachable HaRP. +K8S_PROBE_TIMEOUT = aiohttp.ClientTimeout(total=3.0) _k8s_session: aiohttp.ClientSession | None = None K8S_NAME_MAX_LENGTH = 63 # Set up the logging configuration @@ -832,12 +835,10 @@ async def get_info(request: web.Request): k8s_status: dict[str, Any] = {"enabled": K8S_ENABLED} if K8S_ENABLED: k8s_status["api_server"] = K8S_API_SERVER or "" - try: - _ensure_k8s_configured() - status, _, _ = await _k8s_request("GET", "/api") - k8s_status["reachable"] = status == 200 - except Exception: - k8s_status["reachable"] = False + reachable, error = await _k8s_probe() + k8s_status["reachable"] = reachable + if error: + k8s_status["error"] = error return web.json_response({ "version": HARP_VERSION, @@ -2272,6 +2273,42 @@ async def _k8s_request( raise web.HTTPServiceUnavailable(text="Error communicating with Kubernetes API") from e +async def _k8s_probe() -> tuple[bool, str]: + """Check whether the Kubernetes API server answers; returns ``(reachable, error)``. + + Used by ``/info`` only. Unlike ``_k8s_request`` it fails fast (K8S_PROBE_TIMEOUT) and keeps the reason, so the + AppAPI daemon check can tell a broken cluster connection from a broken HaRP one. + """ + try: + _ensure_k8s_configured() + except web.HTTPServiceUnavailable as e: + return False, e.text or "Kubernetes backend is not configured." + url = f"{K8S_API_SERVER}/api" + headers = {"Authorization": f"Bearer {_get_k8s_token()}", "Accept": "application/json"} + try: + session = _get_k8s_session() # may raise on a broken HP_K8S_CA_FILE + async with session.get(url, headers=headers, timeout=K8S_PROBE_TIMEOUT) as resp: + if resp.status == 200: + return True, "" + if resp.status in (401, 403): + error = ( + f"Kubernetes API server answered HTTP {resp.status}; check the bearer token " + "(HP_K8S_BEARER_TOKEN or HP_K8S_BEARER_TOKEN_FILE)." + ) + else: + error = f"Kubernetes API server answered HTTP {resp.status}." + except TimeoutError: + error = f"Kubernetes API server did not answer within {K8S_PROBE_TIMEOUT.total:g}s (DNS, connect or request)." + except aiohttp.ClientSSLError as e: + error = f"TLS error connecting to the Kubernetes API server: {e}" + except aiohttp.ClientError as e: + error = f"Cannot connect to the Kubernetes API server: {e}" + except Exception as e: # `/info` must always answer + error = f"Kubernetes API probe failed: {e}" + LOGGER.warning("Kubernetes API probe (%s) failed: %s", url, error) + return False, error + + def _k8s_parse_env(env_list: list[str]) -> list[dict[str, str]]: """Convert ['KEY=VALUE', ...] to Kubernetes env entries.""" result: list[dict[str, str]] = [] From 95150736bf9607c154c589b987c86a41262c00c0 Mon Sep 17 00:00:00 2001 From: Oleksandr Piskun Date: Tue, 8 Sep 2026 17:09:52 +0000 Subject: [PATCH 2/2] fix(k8s): surface the API server response in the /info probe error Address review feedback on the probe added in the previous commit: - read the response body inside the request context, like `_k8s_request` already does, and append it (truncated to 200 chars) to the error. The Kubernetes `Status.message` names the actual problem, e.g. `forbidden: User "..." cannot get path "/api"`. Leaving the body unread also made aiohttp drop the pooled connection whenever the body did not arrive with the headers, which is exactly the degraded API server this probe targets. - report HTTP 401 and 403 separately: 401 is authentication, so the bearer token hint is right; 403 is authorization, where a valid token is denied by RBAC and the token hint sends operators the wrong way. - add the missing docstring to `get_info`. Signed-off-by: Oleksandr Piskun --- haproxy_agent.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/haproxy_agent.py b/haproxy_agent.py index 065e534..6993f6b 100644 --- a/haproxy_agent.py +++ b/haproxy_agent.py @@ -832,6 +832,7 @@ async def get_heartbeat(request: web.Request): async def get_info(request: web.Request): + """Report HaRP's version and backend status; used by the AppAPI daemon check.""" k8s_status: dict[str, Any] = {"enabled": K8S_ENABLED} if K8S_ENABLED: k8s_status["api_server"] = K8S_API_SERVER or "" @@ -2288,15 +2289,24 @@ async def _k8s_probe() -> tuple[bool, str]: try: session = _get_k8s_session() # may raise on a broken HP_K8S_CA_FILE async with session.get(url, headers=headers, timeout=K8S_PROBE_TIMEOUT) as resp: + # Read the body inside the context, like `_k8s_request` does: it carries the Kubernetes `Status.message` + # that says what actually went wrong, and leaving it unread makes aiohttp drop the pooled connection + # whenever the body does not arrive with the headers. + body = (await resp.text())[:200].strip() if resp.status == 200: return True, "" - if resp.status in (401, 403): + if resp.status == 401: error = ( - f"Kubernetes API server answered HTTP {resp.status}; check the bearer token " + "Kubernetes API server answered HTTP 401; check the bearer token " "(HP_K8S_BEARER_TOKEN or HP_K8S_BEARER_TOKEN_FILE)." ) + elif resp.status == 403: + # 401 is authentication, 403 is authorization: a valid token can still be denied by RBAC. + error = "Kubernetes API server answered HTTP 403; check the bearer token's RBAC permissions." else: error = f"Kubernetes API server answered HTTP {resp.status}." + if body: + error = f"{error} Response: {body}" except TimeoutError: error = f"Kubernetes API server did not answer within {K8S_PROBE_TIMEOUT.total:g}s (DNS, connect or request)." except aiohttp.ClientSSLError as e: