From 8835a30e35ec7677fe166bdc8a6eb1f7692d18de Mon Sep 17 00:00:00 2001 From: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:37:31 -0800 Subject: [PATCH 1/2] Stop caching Linux arm64 builds The first Prime Caches run on main saved 10.6 GB, over the 10 GB repository quota. GitHub then evicts the least recently used entries, and each push to main re-primes them. Windows entries came out larger than estimated. The Linux arm64 build entries take 1.9 GiB. Those jobs finished about 8 minutes before the slowest job in a fully cold run, so caching them does not shorten PR CI. Dropping them brings the total to about 8.6 GB. The prime job timeout default drops to 45 minutes. 90 minutes covered two release builds on the 2-CPU Linux arm64 VMs, and the slowest remaining job took 13 minutes. Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> --- .github/workflows/PrimeCaches.yml | 27 +++++++++------------------ .github/workflows/dep_rust.yml | 4 ++-- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/.github/workflows/PrimeCaches.yml b/.github/workflows/PrimeCaches.yml index 103a5499..c2fb3323 100644 --- a/.github/workflows/PrimeCaches.yml +++ b/.github/workflows/PrimeCaches.yml @@ -51,13 +51,13 @@ jobs: with: prime_cache: true - # Populate the host caches (key {arch}-{os}-{hypervisor}-{config}) used by - # dep_rust.yml. Release entries hold both wasmtime versions, because the LTS - # and latest release jobs share them. + # Populate the Windows host caches (key {arch}-{os}-{hypervisor}-{config}) + # used by dep_rust.yml. Release entries hold both wasmtime versions, because + # the LTS and latest release jobs share them. Linux builds stay uncached to + # fit the repository cache quota. host: needs: wasm-examples - # Linux arm64 runs on 2-CPU VMs and builds release twice. - timeout-minutes: ${{ fromJSON(vars.PRIME_CACHES_JOB_TIMEOUT || '90') }} + timeout-minutes: ${{ fromJSON(vars.PRIME_CACHES_JOB_TIMEOUT || '45') }} strategy: fail-fast: false matrix: @@ -74,12 +74,6 @@ jobs: - arch: arm64 hypervisor: hyperv config: release - - arch: arm64 - hypervisor: kvm - config: debug - - arch: arm64 - hypervisor: kvm - config: release # Spread load across vendor pools: AMD when run_id ends in 0,2,4,6,8, Intel otherwise. runs-on: ${{ fromJson(matrix.arch == 'X64' && format('["self-hosted", "Windows", "X64", "1ES.Pool=hld-win2025-{0}", "JobId=prime-cache-{1}-{2}-{3}-{4}-{5}"]', @@ -89,13 +83,11 @@ jobs: github.run_id, github.run_number, github.run_attempt) - || matrix.hypervisor == 'hyperv' - && format('["self-hosted", "Windows", "ARM64", "1ES.Pool=hl-windows-arm", "JobId=prime-cache-{0}-{1}-{2}-{3}"]', + || format('["self-hosted", "Windows", "ARM64", "1ES.Pool=hl-windows-arm", "JobId=prime-cache-{0}-{1}-{2}-{3}"]', matrix.config, github.run_id, github.run_number, - github.run_attempt) - || '["self-hosted", "Linux", "arm64", "kvm"]') }} + github.run_attempt)) }} env: JUST_OVERRIDE_ARCH: ${{ matrix.arch == 'X64' && 'x86_64' || 'aarch64' }} steps: @@ -154,10 +146,9 @@ jobs: - name: Build host, tests and examples if: steps.prime.outputs.skip != 'true' run: | - features="${{ matrix.hypervisor == 'kvm' && 'kvm' || '' }}" - just prime-caches ${{ matrix.config }} $features + just prime-caches ${{ matrix.config }} if [ "${{ matrix.config }}" = release ]; then - just prime-caches release ${features:+$features,}wasmtime_latest + just prime-caches release wasmtime_latest fi # A job step, not a post-job step, so the save runs before rust-cache's diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 9a5beba4..57019a90 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -134,7 +134,7 @@ jobs: # PrimeCaches writes these entries. Keep the steps before them in sync # with its host job, because the key hashes toolchains and build env vars. - name: Rust cache - if: matrix.hypervisor == 'hyperv' || (matrix.arch == 'arm64' && matrix.hypervisor == 'kvm') + if: matrix.hypervisor == 'hyperv' uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: prefix-key: v1-rust @@ -143,7 +143,7 @@ jobs: # Restores after rust-cache, which deletes the sysroot on a partial match. - name: Restore sysroot cache - if: matrix.hypervisor == 'hyperv' || (matrix.arch == 'arm64' && matrix.hypervisor == 'kvm') + if: matrix.hypervisor == 'hyperv' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: target/hyperlight-wasm-runtime/sysroot From 8332821821ee3162ee8a6045cfcd842a800e6be6 Mon Sep 17 00:00:00 2001 From: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:37:57 -0800 Subject: [PATCH 2/2] Rebuild the cached Windows surrogate in CI hyperlight-host builds hyperlight_surrogate.exe in target//build/hls, outside its OUT_DIR. rust-cache deletes that directory but keeps the build script state, so Windows jobs that restore a primed cache fail to compile hyperlight-host. Every Windows build, benchmark and code check job fails this way once main has primed the caches. Windows jobs now clean hyperlight-host after restoring the cache, so its build script runs again. This matches hyperlight-js and can be removed once hyperlight-host includes the fix that moves the surrogate into OUT_DIR. Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> --- .github/workflows/dep_code_checks.yml | 9 +++++++++ .github/workflows/dep_rust.yml | 14 ++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/.github/workflows/dep_code_checks.yml b/.github/workflows/dep_code_checks.yml index 10c23664..c6cc508d 100644 --- a/.github/workflows/dep_code_checks.yml +++ b/.github/workflows/dep_code_checks.yml @@ -168,6 +168,15 @@ jobs: name: guest-modules-X64-lts path: . + # rust-cache deletes the Windows surrogate that hyperlight-host builds + # outside OUT_DIR, but keeps its build script state. Remove once + # hyperlight-host includes https://github.com/hyperlight-dev/hyperlight/pull/2038. + - name: Invalidate cached Windows surrogate build + if: steps.prime.outputs.skip != 'true' + run: | + cargo clean -p hyperlight-host --profile dev + cargo clean -p hyperlight-host --profile release + - name: clippy (debug) if: steps.prime.outputs.skip != 'true' run: just clippy debug diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 57019a90..b99506bc 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -149,6 +149,13 @@ jobs: path: target/hyperlight-wasm-runtime/sysroot key: sysroot-${{ matrix.arch }}-${{ runner.os }}-${{ matrix.hypervisor }}-${{ hashFiles('rust-toolchain.toml') }} + # rust-cache deletes the Windows surrogate that hyperlight-host builds + # outside OUT_DIR, but keeps its build script state. Remove once + # hyperlight-host includes https://github.com/hyperlight-dev/hyperlight/pull/2038. + - name: Invalidate cached Windows surrogate build + if: matrix.hypervisor == 'hyperv' + run: cargo clean -p hyperlight-host --profile ${{ matrix.config == 'debug' && 'dev' || 'release' }} + - name: Download Wasm Modules uses: actions/download-artifact@v5 with: @@ -240,6 +247,13 @@ jobs: path: target/hyperlight-wasm-runtime/sysroot key: sysroot-X64-${{ runner.os }}-hyperv-${{ hashFiles('rust-toolchain.toml') }} + # rust-cache deletes the Windows surrogate that hyperlight-host builds + # outside OUT_DIR, but keeps its build script state. Remove once + # hyperlight-host includes https://github.com/hyperlight-dev/hyperlight/pull/2038. + - name: Invalidate cached Windows surrogate build + if: matrix.hypervisor == 'hyperv' + run: cargo clean -p hyperlight-host --profile release + - name: Download benchmarks from "latest" run: just bench-download ${{ runner.os }} ${{ matrix.hypervisor }} ${{ matrix.cpu }} dev-latest env: