From da630fd5f782dd7c980e18db6323ad1ebdf9dd07 Mon Sep 17 00:00:00 2001
From: Jason Pascucci <jpascucci@nvidia.com>
Date: Tue, 3 Nov 2026 12:58:33 -0800
Subject: [PATCH] CI: tolerate transient sccache backend failures

closes #3273

Skip startup cache checks in Linux wheel builds so transient GHA cache
failures do not prevent compilation. Configure cache I/O backoff after
five errors within five seconds.

Apply the same backoff to Linux sdist builds using local-disk sccache.
Omit startup-check skipping there because the disk check performs no I/O.

Validation: actionlint, YAML configuration checks, and git diff --check.
---
 .github/workflows/build-wheel.yml      | 12 ++++++++++++
 .github/workflows/test-sdist-linux.yml |  3 +++
 2 files changed, 15 insertions(+)

diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml
index 03dd258fa19..9772c94e070 100644
--- a/.github/workflows/build-wheel.yml
+++ b/.github/workflows/build-wheel.yml
@@ -238,6 +238,7 @@ jobs:
           # extras: uv puts uv on PATH (required on Windows).
           CIBW_BUILD_FRONTEND: "build[uv]"
           # CIBW mounts the host filesystem under /host
+          # Skip startup cache probes and back off on repeated cache errors (#3273).
           CIBW_ENVIRONMENT_LINUX: >
             CUDA_PATH=/host/${{ env.CUDA_PATH }}
             CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
@@ -247,6 +248,9 @@ jobs:
             CXX="/host/${{ env.SCCACHE_PATH }} c++"
             SCCACHE_GHA_ENABLED=true
             SCCACHE_PREPROCESSOR_CACHE_GHA_ENABLED=true
+            SCCACHE_SKIP_CACHE_CHECK=true
+            SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_COUNT=5
+            SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_WINDOW_SIZE_SECS=5
             ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
             ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
             ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
@@ -335,6 +339,7 @@ jobs:
           # See the cuda.bindings step above for UV_* vs PIP_*.
           CIBW_BUILD_FRONTEND: "build[uv]"
           # CIBW mounts the host filesystem under /host
+          # Skip startup cache probes and back off on repeated cache errors (#3273).
           CIBW_ENVIRONMENT_LINUX: >
             CUDA_PATH=/host/${{ env.CUDA_PATH }}
             CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
@@ -346,6 +351,9 @@ jobs:
             CXX="/host/${{ env.SCCACHE_PATH }} c++"
             SCCACHE_GHA_ENABLED=true
             SCCACHE_PREPROCESSOR_CACHE_GHA_ENABLED=true
+            SCCACHE_SKIP_CACHE_CHECK=true
+            SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_COUNT=5
+            SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_WINDOW_SIZE_SECS=5
             ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
             ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
             ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
@@ -614,6 +622,7 @@ jobs:
           # See the cuda.bindings step above for UV_* vs PIP_*.
           CIBW_BUILD_FRONTEND: "build[uv]"
           # CIBW mounts the host filesystem under /host
+          # Skip startup cache probes and back off on repeated cache errors (#3273).
           CIBW_ENVIRONMENT_LINUX: >
             CUDA_PATH=/host/${{ env.CUDA_PATH }}
             CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
@@ -625,6 +634,9 @@ jobs:
             CXX="/host/${{ env.SCCACHE_PATH }} c++"
             SCCACHE_GHA_ENABLED=true
             SCCACHE_PREPROCESSOR_CACHE_GHA_ENABLED=true
+            SCCACHE_SKIP_CACHE_CHECK=true
+            SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_COUNT=5
+            SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_WINDOW_SIZE_SECS=5
             ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
             ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
             ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
diff --git a/.github/workflows/test-sdist-linux.yml b/.github/workflows/test-sdist-linux.yml
index 13805b7fda3..a4dad6edbc5 100644
--- a/.github/workflows/test-sdist-linux.yml
+++ b/.github/workflows/test-sdist-linux.yml
@@ -34,6 +34,9 @@ jobs:
       BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }}
       BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }}
       BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }}
+      # Back off on repeated local-disk cache I/O errors (#3273).
+      SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_COUNT: "5"
+      SCCACHE_CACHE_RATE_LIMIT_ON_ERROR_WINDOW_SIZE_SECS: "5"
     timeout-minutes: 60
     runs-on: linux-amd64-cpu8
     steps: