From eed8524d162d58ba03048c51974dd2870437f488 Mon Sep 17 00:00:00 2001 From: Rasmus Larsen Date: Sat, 7 Nov 2026 17:18:58 +0000 Subject: [PATCH] Retry builder chunk-store timeouts as infrastructure failures A builder build whose conversion timed out reaching the chunk index (or S3) counted as a recipe failure: 10 minutes between attempts and failed for good after 3. The gateway now retries the chunk store client's own transport failures like sandbox builds' infrastructure failures. The chunk index logs requests slower than 10 s, and a client that left before its reply is no longer logged as "object store unavailable". Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 +++++++++++ docs/images.md | 5 ++++- sandboxes/chunk_index.py | 31 +++++++++++++++++++++++------- sandboxes/gateway/image_recipes.py | 12 +++++++++--- tests/test_image_recipes.py | 11 ++++++++++- 5 files changed, 58 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef5cfd63..d393c673 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## Unreleased + +- **A builder build that timed out reaching the chunk store is retried as an + infrastructure failure**: after 30 s, doubling, up to 6 attempts, the same as + sandbox builds. Before, it counted as a recipe failure: 10 minutes between + attempts, and after 3 the recipe was failed for good. On 2026-11-07, five + builders converting at once kept the chunk index past its 60 s client + timeout, and 50 builds failed this way. +- The chunk index logs requests slower than 10 s. A client that left before + its reply is logged as that, not as "object store unavailable". + ## 0.10.11 - 2026-11-05 (gateway, autoscaler) - **A sandbox deleted while its create is still running no longer keeps diff --git a/docs/images.md b/docs/images.md index 5d8574a1..780f053b 100644 --- a/docs/images.md +++ b/docs/images.md @@ -85,7 +85,10 @@ submits missing builds; it is idempotent and cheap, and callers poll it. A trainer ensures the next step's tasks so builds overlap the current step; without it training is still correct, just slower on first use. Build state is the gateway's, because builders forget builds when they scale down: lost -builds are resubmitted and failures retry a few times. A built recipe image is +builds are resubmitted. A failed build is retried after 10 minutes, at most 3 +attempts in all. A failure of the build machinery rather than the recipe (a +sandbox build's chunk-store, S3 or registry error, or a builder's chunk store +timeout) is retried after 30 s, doubling, at most 6 attempts. A built recipe image is released (OCI manifest deleted), since the recipe can always rebuild it. ## Prepared images diff --git a/sandboxes/chunk_index.py b/sandboxes/chunk_index.py index ee736d19..15eac0a4 100644 --- a/sandboxes/chunk_index.py +++ b/sandboxes/chunk_index.py @@ -48,6 +48,11 @@ STORE_RETRY_SECONDS = 300.0 MAX_BACKOFF_SECONDS = 15.0 SLOW_S3_SECONDS = 2.0 # Object-store calls slower than this are logged. +SLOW_REQUEST_SECONDS = 10.0 # Index requests slower than this are logged. +# How http_request's transport failures read, for callers that classify them +# (the gateway retries a build that failed this way as infrastructure). +TRANSPORT_FAILURE = "chunk store request failed:" +DEADLINE_EXCEEDED = "chunk store request deadline exceeded" # Retried besides 5xx: Hetzner occasionally answers a valid presigned GET 405. RETRYABLE_STATUSES = frozenset({405, 408, 429}) # Bootstraps and chunk maps are named by their digest, so they never change: @@ -147,7 +152,7 @@ def http_request(method, url, *, headers=None, body=None, timeout=30.0, max_byte try: while size <= max_bytes: if time.monotonic() >= deadline: - raise TimeoutError("chunk store request deadline exceeded") + raise TimeoutError(DEADLINE_EXCEEDED) chunk = response.read(min(1 << 20, max_bytes + 1 - size)) if not chunk: complete = True @@ -159,7 +164,7 @@ def http_request(method, url, *, headers=None, body=None, timeout=30.0, max_byte # Only a fully read response may return its connection to the pool. response.release_conn() if complete else response.close() except urllib3.exceptions.HTTPError as exc: - raise OSError(f"chunk store request failed: {type(exc).__name__}") from exc + raise OSError(f"{TRANSPORT_FAILURE} {type(exc).__name__}") from exc if response.status >= 400: raise RegistryRequestError(response.status, method, redact(url), payload[:256].decode(errors="replace")) if len(payload) > max_bytes: @@ -776,17 +781,29 @@ def do_POST(self): # noqa: N802 self._serve(lambda: handler(body), "application/octet-stream" if binary else "application/json") def _serve(self, call, content_type): + started = time.monotonic() try: - return self._reply(200, call(), content_type) + status, payload = 200, call() except MissingChunks as exc: - return self._reply(409, {"error": str(exc)}) + status, payload = 409, {"error": str(exc)} except UnknownRoot as exc: - return self._reply(404, {"error": str(exc)}) + status, payload = 404, {"error": str(exc)} except (ValueError, TypeError, KeyError) as exc: - return self._reply(400, {"error": str(exc)}) + status, payload = 400, {"error": str(exc)} except (OSError, RegistryRequestError) as exc: _LOG.warning("chunk index request failed: %s", exc) - return self._reply(502, {"error": "object store unavailable"}) + status, payload = 502, {"error": "object store unavailable"} + elapsed = time.monotonic() - started + endpoint = urlsplit(self.path).path + if elapsed > SLOW_REQUEST_SECONDS: + _LOG.warning("slow chunk index %s %s: %.1f s (%d)", self.command, endpoint, elapsed, status) + try: + self._reply(status, payload, content_type if status == 200 else "application/json") + except (BrokenPipeError, ConnectionResetError): + # The client's timeout ended it first; the work above still happened. + self.close_connection = True + _LOG.warning("chunk index client left before its %s %s reply: %.1f s (%d)", + self.command, endpoint, elapsed, status) class ChunkIndexServer(ThreadingHTTPServer): diff --git a/sandboxes/gateway/image_recipes.py b/sandboxes/gateway/image_recipes.py index 669ae4bc..6553d860 100644 --- a/sandboxes/gateway/image_recipes.py +++ b/sandboxes/gateway/image_recipes.py @@ -42,6 +42,8 @@ import threading import time +from ..chunk_index import DEADLINE_EXCEEDED, TRANSPORT_FAILURE + _LOG = logging.getLogger(__name__) RETENTIONS = ("pinned", "cached") @@ -61,6 +63,10 @@ # object-store outage window (minutes) then neither keeps a first-use build's # rollouts waiting 10 minutes nor uses up the recipe's attempts. INFRASTRUCTURE_ERROR = "sandbox build infrastructure:" +# Builder builds report the chunk store client's own transport failures as they +# read (a timeout reaching the chunk index or S3): infrastructure too. +INFRASTRUCTURE_PREFIXES = (INFRASTRUCTURE_ERROR, TRANSPORT_FAILURE, DEADLINE_EXCEEDED) +_INFRASTRUCTURE_SQL = "(" + " OR ".join(["COALESCE(i.error, '') LIKE ? || '%'"] * len(INFRASTRUCTURE_PREFIXES)) + ")" INFRASTRUCTURE_BACKOFF_SECONDS = 30 INFRASTRUCTURE_ATTEMPTS = 6 # The index's reads settle builds that finished since their last ensure, at most @@ -105,7 +111,7 @@ def recipe_identity(recipe): def retry_policy(error, attempts): """(attempt limit, backoff seconds) after a build failed with ``error`` on its ``attempts``-th attempt.""" - if (error or "").startswith(INFRASTRUCTURE_ERROR): + if (error or "").startswith(INFRASTRUCTURE_PREFIXES): backoff = INFRASTRUCTURE_BACKOFF_SECONDS * 2 ** max(0, attempts - 1) return INFRASTRUCTURE_ATTEMPTS, min(FAILURE_BACKOFF_SECONDS, backoff) return FAILURE_ATTEMPTS, FAILURE_BACKOFF_SECONDS @@ -385,8 +391,8 @@ def summary(self): with self._db(write=False) as db: for environment, kind, state, attempts, infrastructure, names in db.execute( "SELECT r.environment, i.kind, i.state, i.attempts, " - "COALESCE(i.error, '') LIKE ? || '%', COUNT(*) FROM recipes r " - "JOIN recipe_images i USING (image_id) GROUP BY 1, 2, 3, 4, 5", (INFRASTRUCTURE_ERROR,)): + f"{_INFRASTRUCTURE_SQL}, COUNT(*) FROM recipes r JOIN recipe_images i USING (image_id) GROUP BY 1, 2, 3, 4, 5", + INFRASTRUCTURE_PREFIXES): row = out.setdefault(environment or "(none)", {"names": 0, "tasks": 0, "prepared": 0, "recipes": 0, "states": dict.fromkeys(NAME_STATES, 0)}) row["names"] += names diff --git a/tests/test_image_recipes.py b/tests/test_image_recipes.py index e200d741..519fd5d5 100644 --- a/tests/test_image_recipes.py +++ b/tests/test_image_recipes.py @@ -114,8 +114,17 @@ def test_failures_back_off_then_give_up(self): def test_infrastructure_failures_retry_soon_and_longer(self): # An object-store outage is not recipe rot: 30 s, doubling, and more attempts. + # Sandbox builds mark it; builders report the chunk store client's own + # transport failures (2026-11-07: chunk index timeouts used up attempts). + for error in ("sandbox build infrastructure: OSError: chunk store request failed: ReadTimeoutError", + "chunk store request failed: ReadTimeoutError", + "chunk store request deadline exceeded"): + with self.subTest(error=error): + self.setUp() + self.assert_retried_as_infrastructure(error) + + def assert_retried_as_infrastructure(self, error): self.store.register([recipe("t:1", environment="tmax", tasks=["task_1"])]) - error = "sandbox build infrastructure: OSError: chunk store request failed: ReadTimeoutError" for attempt in range(1, INFRASTRUCTURE_ATTEMPTS + 1): self.ensurer.ensure(["t:1"]) self.builders.finish(f"b{attempt}", "failed", error=error)