diff --git a/.github/build.bat b/.github/build.bat index 6fccce039..865e78ea6 100755 --- a/.github/build.bat +++ b/.github/build.bat @@ -20,9 +20,14 @@ REM sccache is on PATH AND the probe succeeds; otherwise the build proceeds REM uncached and green. The Visual Studio generator jobs do NOT set USE_CACHE, so REM this stays inert for them (and the VS generator ignores REM CMAKE_{C,CXX}_COMPILER_LAUNCHER anyway -- only Ninja/Makefiles honor it). +REM SCCACHE_IDLE_TIMEOUT=0 (unless the job sets it): the server the probe starts lives +REM until the job ends. With sccache's default of 600 s it stopped during the long +REM stretches of a build and a new one counted only what came after, so the stats +REM table below showed a fraction of the requests (the Windows ROCm job: 67 of ~160). REM --------------------------------------------------------------------------- set "LAUNCH=" if /I "%USE_CACHE%"=="true" ( + if not defined SCCACHE_IDLE_TIMEOUT set "SCCACHE_IDLE_TIMEOUT=0" where sccache >nul 2>&1 if errorlevel 1 ( echo build.bat: USE_CACHE=true but sccache not on PATH; building WITHOUT cache. diff --git a/.github/build.sh b/.github/build.sh index 929f960ca..f06bd10eb 100755 --- a/.github/build.sh +++ b/.github/build.sh @@ -67,6 +67,12 @@ if [ "${USE_CACHE:-true}" = "true" ] && [ -n "${SCCACHE_WEBDAV_TOKEN:-}${SCCACHE fi fi +# The server the probe below starts should live until the job ends. With sccache's default idle +# timeout (600 s) it stopped during the long stretches of a build and a new one counted only what +# came after, so the stats table showed a fraction of the requests (the Linux ROCm job: 2 of ~160). +# A job (or a developer) that sets SCCACHE_IDLE_TIMEOUT keeps its own value. +export SCCACHE_IDLE_TIMEOUT="${SCCACHE_IDLE_TIMEOUT:-0}" + # Health-check before trusting sccache as the compiler launcher. Because sccache *is* the # launcher (cmake runs `sccache ...` for every TU), a present-but-crashing sccache # fails every compile and reds the whole build — exactly the in-container panic that stalled @@ -110,8 +116,12 @@ sccache_can_wrap_compiler() { # USE_CACHE is true AND sccache + a cache token are present AND the probe confirms sccache can # wrap the compiler — so it stays inert before the DEPOT_TOKEN secret is configured, on fork PRs # (secrets hidden), and when sccache would crash; all of those just compile normally. sccache is -# content-addressed, so a cache hit is bit-identical to a fresh -O3 compile (release-safe), and -# it degrades to direct compilation if the cache is unreachable. +# content-addressed, so a cache hit is the object the same command line compiles (release-safe), +# and it degrades to direct compilation if the cache is unreachable. "The same command line" is +# the precise claim: for HIP TUs clang hashes the command line into the per-TU CUID symbol +# (`__hip_cuid_`), and sccache runs the compiler with a slightly different one, so a HIP +# object built through sccache differs from one built without it in that name -- code identical, +# measured with a pinned -cuid: byte-identical. Between two sccache builds nothing differs. LAUNCH="" if [ "${USE_CACHE:-true}" = "true" ] && command -v sccache >/dev/null 2>&1 \ && [ -n "${SCCACHE_WEBDAV_TOKEN:-}${SCCACHE_GHA_ENABLED:-}" ] \ @@ -141,6 +151,21 @@ if [ "${USE_CACHE:-true}" = "true" ] && command -v sccache >/dev/null 2>&1 \ echo "build.sh: nvcc runs uncached (CUDA build detected, SCCACHE_WRAP_NVCC is not 'true'); sccache wraps the C/C++ TUs only" fi ;; esac + # ROCm builds: on Linux ggml compiles its ~150 HIP TUs as the CMake language HIP, which the + # C/CXX launchers do not reach -- the Linux ROCm job's stats table counted 2 requests (both + # C/C++) for a 2:50 h build of 27 GPU targets. sccache 0.18.0 caches clang's HIP compiles + # (measured in Docker with TheRock's clang 23, the job's 10.0.0 wheels and a WebDAV backend: + # 157/157 hits on the second build, 310 s -> 10 s for one target). One caveat, local only: with + # the *disk* backend sccache's preprocessor cache mode reads clang's -E output as text, and with + # --offload-compress that output is a compressed bundle -- some TUs then fail with "Failed to + # parse included file path" (mozilla/sccache#2788, unmerged). That mode exists only for the disk + # backend, so CI over Depot never takes that path; a local disk-cache build sets + # SCCACHE_DIRECT=false. + case " $* " in + *" -DGGML_HIP=ON "* | *" -DGGML_HIP=1 "* | *" -DGGML_HIP=on "*) + LAUNCH="$LAUNCH -DCMAKE_HIP_COMPILER_LAUNCHER=sccache" + echo "build.sh: sccache will also wrap the HIP compiler (HIP build detected)" ;; + esac echo "build.sh: sccache ON (endpoint=${SCCACHE_WEBDAV_ENDPOINT:-default}), building with -j${JOBS}" else echo "build.sh: sccache OFF, building with -j${JOBS}" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5dd3d9808..7d9148aa1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1780,13 +1780,10 @@ jobs: # * -DBUILD_TESTING: the option refuses it (jllama_test calls the CPU and RPC backends, which # are modules here, not link-time libraries) -- test-cpp-windows-x86_64 runs the C++ suite in # a static build with the same compiler, the split the Linux CPU jobs have. - # * sccache: sccache 0.18.0 refuses any TU carrying an `-Xclang` argument, and CMake's - # Platform/Windows-Clang.cmake puts one on every plain-clang TU for the runtime-library - # selection (`-Xclang --dependent-lib=libcmt`; llama.cpp adds `-Xclang -fno-pch-timestamp` - # on top for its own TUs) -- measured: every TU a miss, the build green and the cache inert - # -- so the install would only cost time. The cl.exe TUs of the other Windows Ninja jobs - # still cache; this one builds cold on every run, and so does the ROCm job (TheRock's clang - # is a plain clang too; CLAUDE.md "sccache on every Windows Ninja job but one"). + # * sccache: like every Windows Ninja job. It went without from b11538 until the override + # llama/cmake/msvc-runtime-gnu-clang.cmake: CMake's runtime-library flags for plain clang + # carry `-Xclang -flto-visibility-public-std`, which sccache refuses, so every TU was a miss + # (the ROCm job, TheRock's plain clang, the same; CLAUDE.md "sccache on every Windows Ninja job"). # The MSVC job above keeps the Visual Studio generator as the test-only second toolchain; 32-bit # Windows is not built any more (upstream builds none either, and a 32-bit process addresses # ~2 GB, less than most models). @@ -1796,6 +1793,11 @@ jobs: name: Build Windows 2025 x86_64 (modular, CPU variants, clang, default) needs: [startgate, build-webui] runs-on: windows-2025-vs2026 + # sccache again since llama/cmake/msvc-runtime-gnu-clang.cmake: CMake no longer puts the + # -Xclang runtime flags sccache refuses on these plain-clang TUs (CLAUDE.md, "sccache on every + # Windows Ninja job"). + env: + SCCACHE_WEBDAV_TOKEN: ${{ secrets.DEPOT_TOKEN }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Download shared WebUI assets @@ -1810,6 +1812,10 @@ jobs: - name: Display CPU Info shell: bash run: bash .github/print-host-info.sh + - name: Install sccache (shared compiler cache) + if: env.USE_CACHE == 'true' && env.SCCACHE_WEBDAV_TOKEN != '' + continue-on-error: true + uses: ./.github/actions/install-sccache-windows # `clang --version` first: which LLVM the image put on PATH decides whether patches/0017's # compiler half is exercised as a warning (clang 20) or as the error it fixes (clang 22+). - name: Build libraries diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a23a96b6..461a6b2e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -696,6 +696,19 @@ the first two items break every existing consumer at upgrade. backend filter had changed loaded a second copy of the library, or extracted a second module set, and replaced the build info while the JVM kept using the first library. A further call now loads nothing and names a property changed since the load once on stderr; a failed attempt is retried. +- **CI: both ROCm jobs and the Windows plain-clang CPU job compile through sccache again.** On Windows, + CMake's runtime-library flags for plain clang carry `-Xclang -flto-visibility-public-std`, which + sccache refuses, so every TU of `build-windows-x86_64` and of the ROCm module (TheRock's clang, ~150 + HIP TUs for 23 GPU targets, the pipeline's critical path at 2:40-4:15 h) was a miss; a make-rules + override (`llama/cmake/msvc-runtime-gnu-clang.cmake`) selects the same runtime with the driver flag + `-fms-runtime-lib=static` instead. On Linux the HIP TUs are the CMake language HIP, which the C/C++ + launchers never reached; `build.sh` adds `CMAKE_HIP_COMPILER_LAUNCHER`. Measured with one GPU target + before the change: 325 s -> 25 s (Windows) and 310 s -> 10 s (Linux) on a warm cache, objects + identical (the duplicate `/DEFAULTLIB:libcmt.lib` CMake wrote is gone; HIP objects built through + sccache differ from uncached ones in the per-TU CUID symbol name only). A cold run -- the first after + this, and a llama.cpp bump that touches the HIP headers -- takes as long as before. `build.sh` and + `build.bat` also keep the sccache server alive for the whole job, so the job summary's statistics + count every request. - **CI: the Linux CUDA build no longer routes nvcc through sccache by default** (`SCCACHE_WRAP_NVCC=true` in `build.sh` opts back in). With CUDA 13.4 and sccache 0.18.0 the nvcc launcher failed on the first `.cu` files in two confirmed runs (`Missing "cubin" file output`, `Compiler killed by signal 126`), and the diff --git a/CLAUDE.md b/CLAUDE.md index 6118aacd2..08b9cd62b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -251,10 +251,10 @@ measurements behind the Windows step are in `docs/handover/local-agent-report-b1 loadable without the merge step (before, 57 Java tests failed on the load). A GPU module whose runtime is missing fails silently (`SEM_FAILCRITICALERRORS`, ggml logs it at `GGML_LOG_DEBUG` only), which is why `JNI_OnLoad` logs the backend registry and the devices ggml ends up with at INFO. And - sccache is off on the x86-64 CPU job: sccache 0.18.0 refuses any TU carrying an `-Xclang` argument, - and CMake puts one on every plain-clang Windows TU for the runtime-library selection (see "sccache - on every Windows Ninja job but one"; measured: every TU a miss, build green) -- the job builds cold, - and so does the Windows ROCm job, whose HIP TUs are plain-clang C++ on Windows (4:15 h, `TODO.md`). + the plain-clang jobs (this CPU job, the ROCm module) cache through sccache only because + `llama/cmake/msvc-runtime-gnu-clang.cmake` keeps CMake's `-Xclang` runtime flags off their TUs (see + "sccache on every Windows Ninja job"); before it every TU was a miss and the ROCm job built for + 3-4 h on every run. **And the `windows-2025-vs2026` runners are a CPU lottery** (an AMD EPYC 7763 and an Intel Xeon 6973P-C in two consecutive runs), so which CPU module `ggml_backend_load_best` picks in a Windows test or smoke job -- `haswell` on the AMD box, `sapphirerapids` with AMX on the Intel one -- @@ -440,7 +440,7 @@ repository name (`?repo` entries, see `sharedfiles.py`), `SUPPORT.md` and `ISSUE **Composite actions (`.github/actions/`, this repo only).** `restore-models` (restore the shared GGUF cache + `validate-models.sh`; the one place a job gets its models — 16 call sites) and -`install-sccache-windows` (9 call sites; the caller keeps `if:` and `continue-on-error`), +`install-sccache-windows` (10 call sites; the caller keeps `if:` and `continue-on-error`), `build-core` (parent + `llama`, optionally more modules, built without tests/checks/javadoc/signing, `install` or `package` -- the one copy of that skip-flag list, 9 call sites) and `publish-cpu-aar-local` (download + stage the Android CPU natives, publish the CPU AAR to mavenLocal; @@ -820,8 +820,8 @@ Windows-specific wiring (the rest is the natives-jar wiring, see "Natives jars" 3. **`.github/workflows/publish.yml`** — build jobs (all `windows-2025-vs2026`, `ilammy/msvc-dev-cmd@v1`, sccache v0.18.0 zip + Depot WebDAV), each uploading `natives-`: - `build-windows-x86_64` — **modular CPU, plain clang** (`-DJLLAMA_BACKEND_DL=ON - -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++`; no `-DBUILD_TESTING`, no sccache - install -- `test-cpp-windows-x86_64` builds the static library with the tests, same compiler). + -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++`; no `-DBUILD_TESTING` -- + `test-cpp-windows-x86_64` builds the static library with the tests, same compiler). - `build-windows-x86_64-msvc` — **MSVC CPU, build + `ctest`, no jar** (artifact `msvc-windows-x86_64` for `test-java-windows-x86_64-msvc`). - `build-windows-x86_64-cuda` — CUDA `13.4` assembled from NVIDIA's redist archives (upstream's @@ -839,27 +839,41 @@ way CMake's `FindVulkan` couldn't read → switched to `jakoch/install-vulkan-sd FindVulkan-compatible). Because every Windows build job is in the `package`/publish `needs:` graph, a GPU-toolchain failure blocks packaging — the same release-gating policy every build job follows. -**sccache on every Windows Ninja job but one.** Every Ninja build job but one installs sccache -(x86_64 or the native `aarch64` release) through the composite action -`.github/actions/install-sccache-windows` (arch from `RUNNER_ARCH`; its version must equal -`SCCACHE_DL_VERSION` in `build.sh`) with the same `USE_CACHE` / `SCCACHE_WEBDAV_*` env; the MSVC -build-and-test job cannot, because the Visual Studio generator ignores compiler launchers, and -`build-windows-x86_64` (plain clang, modular CPU) does not, because its TUs do not cache: sccache 0.18.0 -refuses any `-Xclang` argument (`Can't handle UnknownFlag arguments with -Xclang`), and every TU a plain -clang -- the GNU driver -- compiles on Windows carries one, because CMake's `Platform/Windows-Clang.cmake` -implements `CMAKE_MSVC_RUNTIME_LIBRARY` (the hybrid CRT every Windows job sets) as `-Xclang --flto-visibility-public-std -D_MT -Xclang --dependent-lib=libcmt`; llama.cpp's root CMakeLists adds -`-Xclang -fno-pch-timestamp` to its own TUs on top, which ggml's never see (ggml is added before it, see -"Modular natives"). Measured at b11538: 719 of 720 compile requests non-cacheable, 0.00% hits, build -green. Re-test on an sccache bump, and if it caches, add the install step back. **The Windows ROCm job is -in the same position** and keeps its install step: TheRock's clang is a plain clang too, and upstream -compiles the HIP sources as C++ on Windows (`CXX_IS_HIPCC` on `WIN32`), so the ~140 HIP TUs -- each -compiled for the 23 `GPU_TARGETS` -- do go through the launcher and are all refused (run 38069595101: the -server's table counted 67 requests, fewer than the TUs, every one of them non-cacheable for `-Xclang`, -0 compilations). That job is the pipeline's critical path -- 4:15 h for its build step in that run, -3:35 h in the last full run before the modular natives, against 2:00 h for the Linux ROCm job and 1:41 h -for the Windows CUDA job -- and only a sccache that caches those TUs could shorten it; `TODO.md` -("Modular natives -- follow-ups") has the open measurement. It cannot red a build, by +**sccache on every Windows Ninja job.** Every Ninja build job installs sccache (x86_64 or the native +`aarch64` release) through the composite action `.github/actions/install-sccache-windows` (arch from +`RUNNER_ARCH`; its version must equal `SCCACHE_DL_VERSION` in `build.sh`) with the same `USE_CACHE` / +`SCCACHE_WEBDAV_*` env; only the MSVC build-and-test job cannot, because the Visual Studio generator +ignores compiler launchers. + +**The plain-clang jobs cache because of `llama/cmake/msvc-runtime-gnu-clang.cmake`.** sccache refuses +any TU carrying an `-Xclang` argument it does not know (`Can't handle UnknownFlag arguments with +-Xclang`; 0.18.0, and `main` on 2026-10-22), and CMake's `Platform/Windows-Clang.cmake` implements +`CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded` (the hybrid CRT every Windows job sets) for clang with the +GNU front end as `-Xclang -flto-visibility-public-std -D_MT -Xclang --dependent-lib=libcmt` -- and +`-flto-visibility-public-std` is the one sccache does not know (`--dependent-lib` and llama.cpp's own +`-Xclang -fno-pch-timestamp` it does). So `build-windows-x86_64` (plain clang, modular CPU; measured at +b11538: 719 of 720 requests non-cacheable) ran without sccache, and **the Windows ROCm job** -- TheRock's +clang is a plain clang, and upstream compiles the HIP sources as C++ on Windows (`CXX_IS_HIPCC` on +`WIN32`) -- sent its ~150 HIP TUs, each compiled for the 23 `GPU_TARGETS`, through a launcher that refused +every one: 2:40 h to 4:15 h for its build step, the pipeline's critical path. The override file, loaded +through `CMAKE_USER_MAKE_RULES_OVERRIDE` (set before `project()` in `llama/CMakeLists.txt`, so CMake +reads it after its platform files while enabling each language), maps `MultiThreaded[Debug]` to the +driver flag `-fms-runtime-lib=static[_dbg]`, which `llama/CMakeLists.txt` already passed next to CMake's +sequence: same runtime selection in cc1, no `-Xclang`. **Measured before the change** (locally, MSVC +14.44 as on the `windows-2022` runner, TheRock 10.0.0 clang 23, Ninja Multi-Config, sccache 0.18.0 over +a local WebDAV server, `GPU_TARGETS=gfx1100`): the build as it was, 157 requests all refused, 325 s; +with the override, 0 refused, 377 s cold and **25 s warm**. Nine TUs (C, C++, seven HIP) compiled with +a pinned `-cuid` from the same build directory: every section identical between the two flag sets -- +`.text` and the HIP device code included -- except `.drectve`, where CMake's sequence repeated +`/DEFAULTLIB:libcmt.lib`; through sccache and without it identical up to the COFF timestamp clang +writes on every compile. Two things that did **not** work, so nobody tries them again: an empty +`CMAKE_MSVC_RUNTIME_LIBRARY` reaches `try_compile`, whose C compiler check then links `-nostdlib` +without a CRT (`undefined symbol: mainCRTStartup`, configure fails); and `-fuse-cuid=none` to compare +whole DLLs makes every TU define the same `__hip_cuid_` (`duplicate symbol` at link). The CUID is also +why a HIP DLL built through sccache is not byte-identical to one built without it: clang hashes the +command line into `__hip_cuid_`, sccache's command line differs slightly, and a hash with a +leading zero is one byte shorter and shifts the device code layout -- names and addresses, not code +(see "CI build cache"). It cannot red a build, by three guards in `build.bat`: the install step is `continue-on-error`; a probe compiles through sccache before it is trusted; and — because that probe only proves `cl.exe`, while arm64 builds with `clang-cl`, ROCm with its own `clang` and SYCL with `icx` — **a configure or build that fails with sccache as the @@ -1228,7 +1242,23 @@ Because `sccache` is **content-addressed** and llama.cpp is pinned (`JLLAMA_GIT_ *changed* files. Depot's cache is **shared across all branches** (unlike GitHub's per-branch `actions/cache`), so every branch builds incrementally; a `b` version bump naturally invalidates the upstream entries (their content changed) with no manual step. It -stays `-O3` and is **bit-identical** to a clean build (release-safe). +stays `-O3`, and a hit is the object the same command line compiles (release-safe). That is +**bit-identical** to a clean build for C and C++; for HIP it is identical up to the per-TU CUID symbol +name, which clang derives from the command line sccache slightly changes (measured with a pinned +`-cuid`: byte-identical; see "sccache on every Windows Ninja job"). + +**HIP on Linux needs its own launcher.** There ggml compiles the HIP TUs as the CMake language HIP, +which `CMAKE_{C,CXX}_COMPILER_LAUNCHER` do not reach, so `build.sh` adds +`-DCMAKE_HIP_COMPILER_LAUNCHER=sccache` when `GGML_HIP` is in the cmake args. Before it the Linux ROCm +job counted 2 requests (both C/C++) for a 2:50 h build. Measured in Docker with the job's toolchain +(TheRock 10.0.0, clang 23, `GPU_TARGETS=gfx1100`, sccache 0.18.0 over a local WebDAV server): 310 s +uncached, 157 of 157 hits and 10 s warm. With sccache's **disk** backend (a local build, never CI) some +TUs fail with `Failed to parse included file path`: its preprocessor cache mode reads clang's `-E` +output as text, and with `--offload-compress` that output is a compressed bundle +(mozilla/sccache#2788, unmerged); `SCCACHE_DIRECT=false` avoids it, and the mode does not exist for +the WebDAV backend. Both scripts also default `SCCACHE_IDLE_TIMEOUT` to 0: with sccache's 600 s the +server stopped during long builds and the stats table counted only what came after (67 of ~160 +requests in the Windows ROCm job, 2 in the Linux one). **Safety / transparency.** It is **inert** until `DEPOT_TOKEN` is configured and on **fork PRs** (secrets are hidden there) — those simply compile normally; the `Install sccache` step diff --git a/TODO.md b/TODO.md index 021aa0479..e729416d1 100644 --- a/TODO.md +++ b/TODO.md @@ -941,7 +941,15 @@ library jars, `JLLAMA_MODULE_ONLY` GPU module jars, no fat jars). What is still 5. **A Vulkan module for Android** (`vulkan-android-aarch64`, upstream ships none either): `JNI_OnLoad` already tries `libggml-vulkan.so` by soname, so it is one more module-only dockcross job plus the natives-jar wiring and an AAR flavor -- if anyone asks. -6. **The Windows ROCm job is the pipeline's critical path, and it builds uncached.** **Measured in run +6. **Addressed on 2026-10-22, the CI number still to read** (CLAUDE.md "sccache on every Windows + Ninja job" and "CI build cache"): the Windows ROCm job caches through + `llama/cmake/msvc-runtime-gnu-clang.cmake`, the Linux one through `CMAKE_HIP_COMPILER_LAUNCHER` in + `build.sh`; measured locally with one GPU target, 325 s -> 25 s and 310 s -> 10 s warm. **Open:** the + two jobs' build steps in the second run after the merge (the first fills the cache), with all 23 / 27 + targets, and the cache size that adds on Depot. A llama.cpp bump that touches the headers every HIP TU + includes (`common.cuh`, `ggml.h`, ...) is still a cold build -- 6 of the 17 bumps from b11558 to b11815 + did, 11 touched none of them. The history follows: + **The Windows ROCm job is the pipeline's critical path, and it builds uncached.** **Measured in run 38149667195: both ROCm jobs were still compiling after 2 h 23 min**, with every other build job finished (50 of 56 green) and `Package JARs` not started -- so everything behind `package`, which is the whole point of a round-0 run, waits on them. The dllimport flood above sits in the same job @@ -954,7 +962,7 @@ library jars, `JLLAMA_MODULE_ONLY` GPU module jars, no fat jars). What is still its 158 ninja steps ~140 are HIP TUs, each compiled for the 23 `GPU_TARGETS`; on Windows upstream compiles them as C++ with TheRock's clang, a plain clang, so they go through sccache -- and sccache 0.18.0 refuses every one of them for the `-Xclang` that CMake puts on any plain-clang Windows TU - (CLAUDE.md "sccache on every Windows Ninja job but one"; the job's stats table: 67 requests, all + (CLAUDE.md "sccache on every Windows Ninja job"; the job's stats table: 67 requests, all non-cacheable, 0 compilations). What would shorten it: a sccache that accepts `-Xclang` -- re-test on every `SCCACHE_DL_VERSION` bump, the stats table in the job summary answers at once, and do it on a branch with one dispatched run, because `build.bat`'s unconditional retry turns a sccache that @@ -977,49 +985,27 @@ interface (what already happens), and keep one guard: a configure-time check tha is on `jllama`'s include path, so a future upstream move of the vendor directory cannot silently flip the order back. Not done in the bump PR on purpose -- a bump changes the pin, nothing else. -### Waiting on sccache: four open issues, and they cost the pipeline hours - -**Nothing to do here but watch.** Checked 2026-10-12: `SCCACHE_DL_VERSION` is **0.18.0 and that is the -newest release** (2026-09-14), so there is no bump to take, and the cause is a family of **open** -issues upstream: - -| issue | what it is | -|---|---| -| [mozilla/sccache#1122](https://github.com/mozilla/sccache/issues/1122) | *"Generically assume that unknown flags are `PassThrough`"* -- **the structural fix**: it would make every one of the others moot | -| [#1603](https://github.com/mozilla/sccache/issues/1603) | handling `-Xclang` custom plugins and flags | -| [#2522](https://github.com/mozilla/sccache/issues/2522) | `-Xclang -fopenmp` errors with `UnknownFlag` | -| [#2717](https://github.com/mozilla/sccache/issues/2717) | add `-fno-cxx-modules` to the known `-Xclang` flags | - -**Why this is not cosmetic.** sccache refuses any translation unit carrying an `-Xclang` argument, and -CMake's `Platform/Windows-Clang.cmake` puts one on **every** plain-clang Windows TU, because that is -how it implements `CMAKE_MSVC_RUNTIME_LIBRARY` -- the hybrid CRT every Windows job here sets -(`-Xclang -flto-visibility-public-std -D_MT -Xclang --dependent-lib=libcmt`). So two jobs cache -nothing at all: - -* **`build-windows-x86_64`** (plain clang, modular CPU): measured at b11538, **719 of 720 compile - requests non-cacheable, 0.00% hit rate**. The job builds cold on every run, which is why it has no - sccache install step at all. -* **`build-windows-x86_64-rocm`**: TheRock's clang is a plain clang too, and upstream compiles the HIP - sources as C++ on Windows, so its ~140 HIP TUs -- each compiled for **23** GPU targets -- all go - through the launcher and are all refused (run 38069595101: 67 requests, every one non-cacheable, 0 - compilations). **This is the pipeline's critical path**: 2:40 h to 3:08 h for its build step in the - measured runs, against ~1 h for everything else in the run put together. A pipeline run is - three hours because of this one job, and a working cache is the only thing that would shorten it. - -**What is already in place, so nothing is blocked meanwhile:** `build.sh`/`build.bat` probe-compile -through sccache before trusting it as the launcher, retry once without it when a build fails with a -cache error, and pass `-DGGML_CCACHE=OFF` so ggml cannot re-enable a launcher behind the retry's back. -The refusal therefore costs time, never a red build. - -**The re-check, when it is worth doing.** On the next sccache **release** (not on a schedule): bump -`SCCACHE_DL_VERSION` in `build.sh`, add the install step back to `build-windows-x86_64`, and read the -`### sccache statistics` table the build script writes into the job summary. A hit rate above zero on -that job is the answer; anything else and the pin goes back. The ROCm job keeps its install step -already, so its summary table answers the same question for free on any run. - -**Worth one comment upstream** rather than a new issue: this project's case -- CMake putting `-Xclang` -on every Windows TU, so the flag is not exotic but unavoidable -- belongs in #1122, which is where the -general fix is being discussed. That is the only action available until something merges. +### sccache and `-Xclang`: worked around here, one flag upstream + +**No longer waiting.** Until 2026-10-22 this entry read "nothing to do but watch": sccache refuses any +TU carrying an `-Xclang` argument it does not know, CMake's `Platform/Windows-Clang.cmake` puts +`-Xclang -flto-visibility-public-std -D_MT -Xclang --dependent-lib=libcmt` on every plain-clang Windows +TU (`CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded`), and so `build-windows-x86_64` (719 of 720 requests +non-cacheable at b11538) and `build-windows-x86_64-rocm` (the pipeline's critical path, 2:40-4:15 h) +cached nothing. `llama/cmake/msvc-runtime-gnu-clang.cmake` now maps that runtime selection to the +driver flag `-fms-runtime-lib=static`, which hands cc1 the same flags without `-Xclang`; measured and +explained in CLAUDE.md "sccache on every Windows Ninja job". + +**What the four issues this entry listed actually are, read in sccache's source** (`main` and v0.18.0, +2026-10-22): sccache knows `--dependent-lib` (PR #1500, which was exactly "-Xclang on Windows CMake") +and `-fno-pch-timestamp` (#1721); the one flag of the sequence it does not know is +**`-flto-visibility-public-std`**, and no issue or PR names it. #1122 (generic pass-through) has been +quiet since 2022, #1603 and #2522 are other flags, and #2717 is a PR (for `-fno-cxx-modules`), not an +issue. sccache adds `-Xclang` flags one PR at a time. + +**Optional upstream follow-up:** a one-line PR adding `flag!("-flto-visibility-public-std", +PassThroughFlag)` to `src/compiler/clang.rs` with a test of CMake's full sequence. It would let a +plain-clang Windows build cache without this project's override; nothing here depends on it. ### CUDA job: nvcc through sccache (`SCCACHE_WRAP_NVCC`, off since the second failure) @@ -1314,13 +1300,11 @@ because each is a design decision in code this project does not own. And say pla downstream instead (the opt-in, the non-aborting registration, the `-w` on `ggml-hip`), so a maintainer can see what a merged change would let this project delete. -A fifth report is **not** for llama.cpp: sccache refuses every TU carrying an `-Xclang` argument, and -CMake's runtime-library selection puts one on every plain-clang Windows TU. Checked 2026-10-12: -**0.18.0 is the newest release** (2026-09-14), so there is no bump to take, and the flag handling is a -family of **open** sccache issues -- mozilla/sccache#1603, #2522, #2717 and #1122 (the structural one: -treat an unknown flag as pass-through). So the measurement stands as recorded and the only action is -to add this case to one of those issues, or wait. Re-check on the next sccache release before -spending a CI run on it. +A fifth report is **not** for llama.cpp: sccache refuses every TU carrying an `-Xclang` argument it +does not know, and CMake's runtime-library selection puts `-Xclang -flto-visibility-public-std` on +every plain-clang Windows TU. Worked around here since 2026-10-22 +(`llama/cmake/msvc-runtime-gnu-clang.cmake`); the upstream side is a one-flag PR to sccache, see +"sccache and `-Xclang`" above. There are **twelve** patches today (`0001`-`0003`, `0006`-`0008`, `0014`-`0019`). Each accepted upstream change deletes a patch from the bump checklist once the pin is bumped past it. diff --git a/llama/CMakeLists.txt b/llama/CMakeLists.txt index c4fefa586..28cea3b96 100644 --- a/llama/CMakeLists.txt +++ b/llama/CMakeLists.txt @@ -8,6 +8,15 @@ if(NOT DEFINED CMAKE_OSX_DEPLOYMENT_TARGET AND NOT DEFINED ENV{MACOSX_DEPLOYMENT set(CMAKE_OSX_DEPLOYMENT_TARGET "15.0" CACHE STRING "Minimum macOS version of libjllama.dylib") endif() +# Plain clang on Windows: CMake's runtime-library flags carry an -Xclang argument sccache refuses, +# so every TU of the plain-clang CPU and ROCm jobs went uncached. The override maps +# CMAKE_MSVC_RUNTIME_LIBRARY to -fms-runtime-lib for that toolchain only; the file says why and what +# was measured. It must be set before project(): CMake reads it while enabling each language. A +# -DCMAKE_USER_MAKE_RULES_OVERRIDE from outside wins (and then the -Xclang flags come back). +if(NOT CMAKE_USER_MAKE_RULES_OVERRIDE) + set(CMAKE_USER_MAKE_RULES_OVERRIDE "${CMAKE_CURRENT_LIST_DIR}/cmake/msvc-runtime-gnu-clang.cmake") +endif() + project(jllama CXX) # HYBRID CRT on Windows, for every toolchain but the two named below: the STL and vcruntime are diff --git a/llama/cmake/msvc-runtime-gnu-clang.cmake b/llama/cmake/msvc-runtime-gnu-clang.cmake new file mode 100644 index 000000000..ed0c81faa --- /dev/null +++ b/llama/cmake/msvc-runtime-gnu-clang.cmake @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT +# +# msvc-runtime-gnu-clang.cmake -- the MSVC runtime of plain clang on Windows, selected by the driver. +# +# Loaded through CMAKE_USER_MAKE_RULES_OVERRIDE (set before project() in llama/CMakeLists.txt), i.e. +# once per enabled language, right after CMake's own Platform/Windows-Clang.cmake. For clang with the +# GNU front end, that file implements CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded as +# +# -Xclang -flto-visibility-public-std -D_MT -Xclang --dependent-lib=libcmt +# +# and sccache (0.18.0, and main as of 2026-10-22) refuses every TU that carries an `-Xclang` argument it +# does not know -- `-flto-visibility-public-std` is one ("Can't handle UnknownFlag arguments with +# -Xclang"). So the Windows ROCm job (TheRock's clang compiles the ~150 HIP TUs as C++ for 23 GPU +# targets) and the plain-clang CPU job cached nothing at all. +# +# The driver flag -fms-runtime-lib=static hands cc1 the same runtime selection (-D_MT, +# -flto-visibility-public-std, --dependent-lib=libcmt, plus --dependent-lib=oldnames as cl.exe /MT +# does), and llama/CMakeLists.txt already adds it for this toolchain. Measured with TheRock's clang +# 23 and MSVC 14.44 (the windows-2022 runner's): every section of every object identical -- .text and +# the HIP device code included -- except .drectve, where CMake's sequence repeated +# /DEFAULTLIB:libcmt.lib. CLAUDE.md ("sccache on every Windows Ninja job") has the measurement. +# +# Why the option variables and not an empty CMAKE_MSVC_RUNTIME_LIBRARY: the variable is handed to +# try_compile, and an empty one leaves the compiler check linking -nostdlib without any CRT +# (`undefined symbol: mainCRTStartup`, configure fails). With this override the variable keeps its +# value and try_compile keeps working; only the flags it maps to change. +# +# clang-cl, cl.exe and icx (IntelLLVM) are not Clang with the GNU front end, so they are untouched. +if(WIN32) + foreach(_jllama_lang C CXX) + if(CMAKE_${_jllama_lang}_COMPILER_ID STREQUAL "Clang" + AND CMAKE_${_jllama_lang}_COMPILER_FRONTEND_VARIANT STREQUAL "GNU") + set(CMAKE_${_jllama_lang}_COMPILE_OPTIONS_MSVC_RUNTIME_LIBRARY_MultiThreaded -fms-runtime-lib=static) + set(CMAKE_${_jllama_lang}_COMPILE_OPTIONS_MSVC_RUNTIME_LIBRARY_MultiThreadedDebug -fms-runtime-lib=static_dbg) + endif() + endforeach() + unset(_jllama_lang) +endif()