diff --git a/CHANGELOG.md b/CHANGELOG.md index eec376df..dc8ed7ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,14 @@ takes `--nydus-image` and `--nydus-image-sha256` (the deployment's `chunk_store.nydus_image_sha256`) and refuses any other binary. Production's builder bundle was repacked with it on 2026-11-07. +- **A builder build that timed out reaching the chunk store is retried as an + infrastructure failure**: after 30 s, doubling, up to 6 attempts, the same as + sandbox builds. Before, it counted as a recipe failure: 10 minutes between + attempts, and after 3 the recipe was failed for good. On 2026-11-07, five + builders converting at once kept the chunk index past its 60 s client + timeout, and 50 builds failed this way. +- The chunk index logs requests slower than 10 s. A client that left before + its reply is logged as that, not as "object store unavailable". ## 0.10.11 - 2026-11-05 (gateway, autoscaler) diff --git a/docs/images.md b/docs/images.md index 5d8574a1..780f053b 100644 --- a/docs/images.md +++ b/docs/images.md @@ -85,7 +85,10 @@ submits missing builds; it is idempotent and cheap, and callers poll it. A trainer ensures the next step's tasks so builds overlap the current step; without it training is still correct, just slower on first use. Build state is the gateway's, because builders forget builds when they scale down: lost -builds are resubmitted and failures retry a few times. A built recipe image is +builds are resubmitted. A failed build is retried after 10 minutes, at most 3 +attempts in all. A failure of the build machinery rather than the recipe (a +sandbox build's chunk-store, S3 or registry error, or a builder's chunk store +timeout) is retried after 30 s, doubling, at most 6 attempts. A built recipe image is released (OCI manifest deleted), since the recipe can always rebuild it. ## Prepared images diff --git a/sandboxes/chunk_index.py b/sandboxes/chunk_index.py index ee736d19..15eac0a4 100644 --- a/sandboxes/chunk_index.py +++ b/sandboxes/chunk_index.py @@ -48,6 +48,11 @@ STORE_RETRY_SECONDS = 300.0 MAX_BACKOFF_SECONDS = 15.0 SLOW_S3_SECONDS = 2.0 # Object-store calls slower than this are logged. +SLOW_REQUEST_SECONDS = 10.0 # Index requests slower than this are logged. +# How http_request's transport failures read, for callers that classify them +# (the gateway retries a build that failed this way as infrastructure). +TRANSPORT_FAILURE = "chunk store request failed:" +DEADLINE_EXCEEDED = "chunk store request deadline exceeded" # Retried besides 5xx: Hetzner occasionally answers a valid presigned GET 405. RETRYABLE_STATUSES = frozenset({405, 408, 429}) # Bootstraps and chunk maps are named by their digest, so they never change: @@ -147,7 +152,7 @@ def http_request(method, url, *, headers=None, body=None, timeout=30.0, max_byte try: while size <= max_bytes: if time.monotonic() >= deadline: - raise TimeoutError("chunk store request deadline exceeded") + raise TimeoutError(DEADLINE_EXCEEDED) chunk = response.read(min(1 << 20, max_bytes + 1 - size)) if not chunk: complete = True @@ -159,7 +164,7 @@ def http_request(method, url, *, headers=None, body=None, timeout=30.0, max_byte # Only a fully read response may return its connection to the pool. response.release_conn() if complete else response.close() except urllib3.exceptions.HTTPError as exc: - raise OSError(f"chunk store request failed: {type(exc).__name__}") from exc + raise OSError(f"{TRANSPORT_FAILURE} {type(exc).__name__}") from exc if response.status >= 400: raise RegistryRequestError(response.status, method, redact(url), payload[:256].decode(errors="replace")) if len(payload) > max_bytes: @@ -776,17 +781,29 @@ def do_POST(self): # noqa: N802 self._serve(lambda: handler(body), "application/octet-stream" if binary else "application/json") def _serve(self, call, content_type): + started = time.monotonic() try: - return self._reply(200, call(), content_type) + status, payload = 200, call() except MissingChunks as exc: - return self._reply(409, {"error": str(exc)}) + status, payload = 409, {"error": str(exc)} except UnknownRoot as exc: - return self._reply(404, {"error": str(exc)}) + status, payload = 404, {"error": str(exc)} except (ValueError, TypeError, KeyError) as exc: - return self._reply(400, {"error": str(exc)}) + status, payload = 400, {"error": str(exc)} except (OSError, RegistryRequestError) as exc: _LOG.warning("chunk index request failed: %s", exc) - return self._reply(502, {"error": "object store unavailable"}) + status, payload = 502, {"error": "object store unavailable"} + elapsed = time.monotonic() - started + endpoint = urlsplit(self.path).path + if elapsed > SLOW_REQUEST_SECONDS: + _LOG.warning("slow chunk index %s %s: %.1f s (%d)", self.command, endpoint, elapsed, status) + try: + self._reply(status, payload, content_type if status == 200 else "application/json") + except (BrokenPipeError, ConnectionResetError): + # The client's timeout ended it first; the work above still happened. + self.close_connection = True + _LOG.warning("chunk index client left before its %s %s reply: %.1f s (%d)", + self.command, endpoint, elapsed, status) class ChunkIndexServer(ThreadingHTTPServer): diff --git a/sandboxes/gateway/image_recipes.py b/sandboxes/gateway/image_recipes.py index 669ae4bc..6553d860 100644 --- a/sandboxes/gateway/image_recipes.py +++ b/sandboxes/gateway/image_recipes.py @@ -42,6 +42,8 @@ import threading import time +from ..chunk_index import DEADLINE_EXCEEDED, TRANSPORT_FAILURE + _LOG = logging.getLogger(__name__) RETENTIONS = ("pinned", "cached") @@ -61,6 +63,10 @@ # object-store outage window (minutes) then neither keeps a first-use build's # rollouts waiting 10 minutes nor uses up the recipe's attempts. INFRASTRUCTURE_ERROR = "sandbox build infrastructure:" +# Builder builds report the chunk store client's own transport failures as they +# read (a timeout reaching the chunk index or S3): infrastructure too. +INFRASTRUCTURE_PREFIXES = (INFRASTRUCTURE_ERROR, TRANSPORT_FAILURE, DEADLINE_EXCEEDED) +_INFRASTRUCTURE_SQL = "(" + " OR ".join(["COALESCE(i.error, '') LIKE ? || '%'"] * len(INFRASTRUCTURE_PREFIXES)) + ")" INFRASTRUCTURE_BACKOFF_SECONDS = 30 INFRASTRUCTURE_ATTEMPTS = 6 # The index's reads settle builds that finished since their last ensure, at most @@ -105,7 +111,7 @@ def recipe_identity(recipe): def retry_policy(error, attempts): """(attempt limit, backoff seconds) after a build failed with ``error`` on its ``attempts``-th attempt.""" - if (error or "").startswith(INFRASTRUCTURE_ERROR): + if (error or "").startswith(INFRASTRUCTURE_PREFIXES): backoff = INFRASTRUCTURE_BACKOFF_SECONDS * 2 ** max(0, attempts - 1) return INFRASTRUCTURE_ATTEMPTS, min(FAILURE_BACKOFF_SECONDS, backoff) return FAILURE_ATTEMPTS, FAILURE_BACKOFF_SECONDS @@ -385,8 +391,8 @@ def summary(self): with self._db(write=False) as db: for environment, kind, state, attempts, infrastructure, names in db.execute( "SELECT r.environment, i.kind, i.state, i.attempts, " - "COALESCE(i.error, '') LIKE ? || '%', COUNT(*) FROM recipes r " - "JOIN recipe_images i USING (image_id) GROUP BY 1, 2, 3, 4, 5", (INFRASTRUCTURE_ERROR,)): + f"{_INFRASTRUCTURE_SQL}, COUNT(*) FROM recipes r JOIN recipe_images i USING (image_id) GROUP BY 1, 2, 3, 4, 5", + INFRASTRUCTURE_PREFIXES): row = out.setdefault(environment or "(none)", {"names": 0, "tasks": 0, "prepared": 0, "recipes": 0, "states": dict.fromkeys(NAME_STATES, 0)}) row["names"] += names diff --git a/tests/test_image_recipes.py b/tests/test_image_recipes.py index e200d741..519fd5d5 100644 --- a/tests/test_image_recipes.py +++ b/tests/test_image_recipes.py @@ -114,8 +114,17 @@ def test_failures_back_off_then_give_up(self): def test_infrastructure_failures_retry_soon_and_longer(self): # An object-store outage is not recipe rot: 30 s, doubling, and more attempts. + # Sandbox builds mark it; builders report the chunk store client's own + # transport failures (2026-11-07: chunk index timeouts used up attempts). + for error in ("sandbox build infrastructure: OSError: chunk store request failed: ReadTimeoutError", + "chunk store request failed: ReadTimeoutError", + "chunk store request deadline exceeded"): + with self.subTest(error=error): + self.setUp() + self.assert_retried_as_infrastructure(error) + + def assert_retried_as_infrastructure(self, error): self.store.register([recipe("t:1", environment="tmax", tasks=["task_1"])]) - error = "sandbox build infrastructure: OSError: chunk store request failed: ReadTimeoutError" for attempt in range(1, INFRASTRUCTURE_ATTEMPTS + 1): self.ensurer.ensure(["t:1"]) self.builders.finish(f"b{attempt}", "failed", error=error)