Retry builder chunk-store timeouts as infrastructure failures

CHANGELOG.md

@@ -10,6 +10,14 @@
   takes `--nydus-image` and `--nydus-image-sha256` (the deployment's
   `chunk_store.nydus_image_sha256`) and refuses any other binary. Production's
   builder bundle was repacked with it on 2026-11-07.
+- **A builder build that timed out reaching the chunk store is retried as an
+  infrastructure failure**: after 30 s, doubling, up to 6 attempts, the same as
+  sandbox builds. Before, it counted as a recipe failure: 10 minutes between
+  attempts, and after 3 the recipe was failed for good. On 2026-11-07, five
+  builders converting at once kept the chunk index past its 60 s client
+  timeout, and 50 builds failed this way.
+- The chunk index logs requests slower than 10 s. A client that left before
+  its reply is logged as that, not as "object store unavailable".
 
 ## 0.10.11 - 2026-11-05 (gateway, autoscaler)
 

docs/images.md

@@ -85,7 +85,10 @@ submits missing builds; it is idempotent and cheap, and callers poll it. A
 trainer ensures the next step's tasks so builds overlap the current step;
 without it training is still correct, just slower on first use. Build state is
 the gateway's, because builders forget builds when they scale down: lost
-builds are resubmitted and failures retry a few times. A built recipe image is
+builds are resubmitted. A failed build is retried after 10 minutes, at most 3
+attempts in all. A failure of the build machinery rather than the recipe (a
+sandbox build's chunk-store, S3 or registry error, or a builder's chunk store
+timeout) is retried after 30 s, doubling, at most 6 attempts. A built recipe image is
 released (OCI manifest deleted), since the recipe can always rebuild it.
 
 ## Prepared images

sandboxes/chunk_index.py

@@ -48,6 +48,11 @@
 STORE_RETRY_SECONDS = 300.0
 MAX_BACKOFF_SECONDS = 15.0
 SLOW_S3_SECONDS = 2.0  # Object-store calls slower than this are logged.
+SLOW_REQUEST_SECONDS = 10.0  # Index requests slower than this are logged.
+# How http_request's transport failures read, for callers that classify them
+# (the gateway retries a build that failed this way as infrastructure).
+TRANSPORT_FAILURE = "chunk store request failed:"
+DEADLINE_EXCEEDED = "chunk store request deadline exceeded"
 # Retried besides 5xx: Hetzner occasionally answers a valid presigned GET 405.
 RETRYABLE_STATUSES = frozenset({405, 408, 429})
 # Bootstraps and chunk maps are named by their digest, so they never change:
@@ -147,7 +152,7 @@ def http_request(method, url, *, headers=None, body=None, timeout=30.0, max_byte
         try:
             while size <= max_bytes:
                 if time.monotonic() >= deadline:
-                    raise TimeoutError("chunk store request deadline exceeded")
+                    raise TimeoutError(DEADLINE_EXCEEDED)
                 chunk = response.read(min(1 << 20, max_bytes + 1 - size))
                 if not chunk:
                     complete = True
@@ -159,7 +164,7 @@ def http_request(method, url, *, headers=None, body=None, timeout=30.0, max_byte
             # Only a fully read response may return its connection to the pool.
             response.release_conn() if complete else response.close()
     except urllib3.exceptions.HTTPError as exc:
-        raise OSError(f"chunk store request failed: {type(exc).__name__}") from exc
+        raise OSError(f"{TRANSPORT_FAILURE} {type(exc).__name__}") from exc
     if response.status >= 400:
         raise RegistryRequestError(response.status, method, redact(url), payload[:256].decode(errors="replace"))
     if len(payload) > max_bytes:
@@ -776,17 +781,29 @@ def do_POST(self):  # noqa: N802
         self._serve(lambda: handler(body), "application/octet-stream" if binary else "application/json")
 
     def _serve(self, call, content_type):
+        started = time.monotonic()
         try:
-            return self._reply(200, call(), content_type)
+            status, payload = 200, call()
         except MissingChunks as exc:
-            return self._reply(409, {"error": str(exc)})
+            status, payload = 409, {"error": str(exc)}
         except UnknownRoot as exc:
-            return self._reply(404, {"error": str(exc)})
+            status, payload = 404, {"error": str(exc)}
         except (ValueError, TypeError, KeyError) as exc:
-            return self._reply(400, {"error": str(exc)})
+            status, payload = 400, {"error": str(exc)}
         except (OSError, RegistryRequestError) as exc:
             _LOG.warning("chunk index request failed: %s", exc)
-            return self._reply(502, {"error": "object store unavailable"})
+            status, payload = 502, {"error": "object store unavailable"}
+        elapsed = time.monotonic() - started
+        endpoint = urlsplit(self.path).path
+        if elapsed > SLOW_REQUEST_SECONDS:
+            _LOG.warning("slow chunk index %s %s: %.1f s (%d)", self.command, endpoint, elapsed, status)
+        try:
+            self._reply(status, payload, content_type if status == 200 else "application/json")
+        except (BrokenPipeError, ConnectionResetError):
+            # The client's timeout ended it first; the work above still happened.
+            self.close_connection = True
+            _LOG.warning("chunk index client left before its %s %s reply: %.1f s (%d)",
+                         self.command, endpoint, elapsed, status)
 
 
 class ChunkIndexServer(ThreadingHTTPServer):

sandboxes/gateway/image_recipes.py

@@ -42,6 +42,8 @@
 import threading
 import time
 
+from ..chunk_index import DEADLINE_EXCEEDED, TRANSPORT_FAILURE
+
 _LOG = logging.getLogger(__name__)
 
 RETENTIONS = ("pinned", "cached")
@@ -61,6 +63,10 @@
 # object-store outage window (minutes) then neither keeps a first-use build's
 # rollouts waiting 10 minutes nor uses up the recipe's attempts.
 INFRASTRUCTURE_ERROR = "sandbox build infrastructure:"
+# Builder builds report the chunk store client's own transport failures as they
+# read (a timeout reaching the chunk index or S3): infrastructure too.
+INFRASTRUCTURE_PREFIXES = (INFRASTRUCTURE_ERROR, TRANSPORT_FAILURE, DEADLINE_EXCEEDED)
+_INFRASTRUCTURE_SQL = "(" + " OR ".join(["COALESCE(i.error, '') LIKE ? || '%'"] * len(INFRASTRUCTURE_PREFIXES)) + ")"
 INFRASTRUCTURE_BACKOFF_SECONDS = 30
 INFRASTRUCTURE_ATTEMPTS = 6
 # The index's reads settle builds that finished since their last ensure, at most
@@ -105,7 +111,7 @@ def recipe_identity(recipe):
 def retry_policy(error, attempts):
     """(attempt limit, backoff seconds) after a build failed with ``error`` on
     its ``attempts``-th attempt."""
-    if (error or "").startswith(INFRASTRUCTURE_ERROR):
+    if (error or "").startswith(INFRASTRUCTURE_PREFIXES):
         backoff = INFRASTRUCTURE_BACKOFF_SECONDS * 2 ** max(0, attempts - 1)
         return INFRASTRUCTURE_ATTEMPTS, min(FAILURE_BACKOFF_SECONDS, backoff)
     return FAILURE_ATTEMPTS, FAILURE_BACKOFF_SECONDS
@@ -385,8 +391,8 @@ def summary(self):
         with self._db(write=False) as db:
             for environment, kind, state, attempts, infrastructure, names in db.execute(
                     "SELECT r.environment, i.kind, i.state, i.attempts, "
-                    "COALESCE(i.error, '') LIKE ? || '%', COUNT(*) FROM recipes r "
-                    "JOIN recipe_images i USING (image_id) GROUP BY 1, 2, 3, 4, 5", (INFRASTRUCTURE_ERROR,)):
+                    f"{_INFRASTRUCTURE_SQL}, COUNT(*) FROM recipes r JOIN recipe_images i USING (image_id) GROUP BY 1, 2, 3, 4, 5",
+                    INFRASTRUCTURE_PREFIXES):
                 row = out.setdefault(environment or "(none)", {"names": 0, "tasks": 0, "prepared": 0, "recipes": 0,
                                                                "states": dict.fromkeys(NAME_STATES, 0)})
                 row["names"] += names

tests/test_image_recipes.py

@@ -114,8 +114,17 @@ def test_failures_back_off_then_give_up(self):
 
     def test_infrastructure_failures_retry_soon_and_longer(self):
         # An object-store outage is not recipe rot: 30 s, doubling, and more attempts.
+        # Sandbox builds mark it; builders report the chunk store client's own
+        # transport failures (2026-11-07: chunk index timeouts used up attempts).
+        for error in ("sandbox build infrastructure: OSError: chunk store request failed: ReadTimeoutError",
+                      "chunk store request failed: ReadTimeoutError",
+                      "chunk store request deadline exceeded"):
+            with self.subTest(error=error):
+                self.setUp()
+                self.assert_retried_as_infrastructure(error)
+
+    def assert_retried_as_infrastructure(self, error):
         self.store.register([recipe("t:1", environment="tmax", tasks=["task_1"])])
-        error = "sandbox build infrastructure: OSError: chunk store request failed: ReadTimeoutError"
         for attempt in range(1, INFRASTRUCTURE_ATTEMPTS + 1):
             self.ensurer.ensure(["t:1"])
             self.builders.finish(f"b{attempt}", "failed", error=error)