diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2e2153651..324993458 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -369,6 +369,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + - name: Display CPU Info + shell: bash + run: bash .github/print-host-info.sh - name: Download Linux x86_64 native library (reused, not rebuilt) uses: actions/download-artifact@v8 with: @@ -485,6 +488,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + - name: Display CPU Info + shell: bash + run: bash .github/print-host-info.sh - name: Download Linux x86_64 native library (reused, not rebuilt) uses: actions/download-artifact@v8 with: @@ -755,6 +761,9 @@ jobs: SCCACHE_WEBDAV_TOKEN: ${{ secrets.DEPOT_TOKEN }} steps: - uses: actions/checkout@v7 + - name: Display CPU Info + shell: bash + run: bash .github/print-host-info.sh - name: Download shared WebUI assets uses: actions/download-artifact@v8 with: @@ -1127,6 +1136,9 @@ jobs: tool-cache: false swap-storage: false - uses: actions/checkout@v7 + - name: Display CPU Info + shell: bash + run: bash .github/print-host-info.sh - uses: actions/setup-java@v6 with: java-version-file: .java-version @@ -1569,6 +1581,9 @@ jobs: SCCACHE_WEBDAV_TOKEN: ${{ secrets.DEPOT_TOKEN }} steps: - uses: actions/checkout@v7 + - name: Display CPU Info + shell: bash + run: bash .github/print-host-info.sh - name: Download shared WebUI assets uses: actions/download-artifact@v8 with: @@ -2164,8 +2179,12 @@ jobs: - name: Build libraries shell: cmd # vcpkg toolchain file wires in the OpenCL (incl. cl2.hpp) that ggml-openvino needs. + # VCPKG_APPLOCAL_DEPS=OFF: vcpkg otherwise copies its OpenCL.dll next to the linked + # module (app-local deployment), and a module jar must hold its module and the + # jllama-*.txt lists only -- merge-native-artifacts.sh rejects the artifact otherwise, and + # bundling a vendor runtime is against the policy every other GPU jar follows. run: | - .github\build.bat -G "Ninja Multi-Config" -DJLLAMA_MODULE_ONLY=ON -DGGML_OPENVINO=ON -DOpenVINO_DIR="%OpenVINO_DIR%" -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -DOS_NAME=Windows -DOS_ARCH=x86_64 + .github\build.bat -G "Ninja Multi-Config" -DJLLAMA_MODULE_ONLY=ON -DGGML_OPENVINO=ON -DOpenVINO_DIR="%OpenVINO_DIR%" -DVCPKG_APPLOCAL_DEPS=OFF -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -DOS_NAME=Windows -DOS_ARCH=x86_64 - name: Upload artifacts uses: actions/upload-artifact@v7 with: @@ -2550,6 +2569,9 @@ jobs: # byte-level interleave -- two artifacts with the same path, which is how a corrupt macOS # dylib shipped; the test-only macOS builds (macos-26-metal, macos-15-no-metal) are named # outside the glob for that reason. + - name: Display CPU Info + shell: bash + run: bash .github/print-host-info.sh - uses: actions/download-artifact@v8 with: pattern: "natives-*" @@ -2682,6 +2704,9 @@ jobs: - uses: actions/checkout@v7 with: persist-credentials: false + - name: Display CPU Info + shell: bash + run: bash .github/print-host-info.sh - uses: actions/download-artifact@v8 with: name: llama-smoke-${{ matrix.target }} @@ -2739,6 +2764,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + - name: Display CPU Info + shell: bash + run: bash .github/print-host-info.sh - uses: actions/download-artifact@v8 with: name: llama-atmosphere-agent-jar @@ -2804,6 +2832,9 @@ jobs: runs-on: macos-15 steps: - uses: actions/checkout@v7 + - name: Display CPU Info + shell: bash + run: bash .github/print-host-info.sh - uses: actions/download-artifact@v8 with: name: llama-smoke-macos-aarch64 diff --git a/CHANGELOG.md b/CHANGELOG.md index 35ef32b05..1fd816e4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -453,7 +453,14 @@ from version 5.0.0 onward. Pre-fork releases (`1.x`–`4.2.0`) were authored by 5.1.0 Windows jar was a static Haswell-level build that carried no AMX code. Found by the second full CI run of the modular natives, whose Windows Java test job landed on an Intel Xeon 6973P-C runner; the first run's AMD EPYC 7763 never reached the code (the Windows runners are a CPU - lottery, CLAUDE.md "Modular natives"). + lottery, CLAUDE.md "Modular natives"). Of the six callers of `ggml_aligned_malloc` upstream this + was the only mismatched pair. Reproduced deliberately afterwards on a rented Granite Rapids + machine with one CPU module staged at a time: `sapphirerapids` crashed 6 of 6 runs and + `cooperlake`, `icelake` and `haswell` none, it reproduces at a **single thread** (so it is no data + race), and the same matrix with the patch applied is clean in all twelve runs. A configure-time + check (`llama/cmake/check-aligned-free-pairing.cmake`) now fails the build on **every** platform + when a ggml source allocates aligned and frees plain, because a machine without AMX cannot observe + the defect at run time -- which is exactly how it survived. - **`NativeServer` attach mode no longer leaves a sleep callback into freed memory behind.** `llama_server_attach` (`patches/0007`) built a `server_routes` on its own stack frame; its constructor registers a sleeping-state callback on the model's task queue, which has no unregister, diff --git a/CLAUDE.md b/CLAUDE.md index 113ef98f8..e8fe53145 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1308,7 +1308,7 @@ Current patches: | `0015-rpc-embeddable-client-and-stoppable-server.patch` | **Makes ggml-rpc usable inside a JVM** (see "RPC backend" below). Upstream treats every client-side RPC problem as `GGML_ABORT` — a malformed endpoint, a server that is not running, a failed handshake — which in a JVM kills the application over a typo in `--rpc`. **(1)** The *registration* path reports failure instead: `rpc_dispatcher::start()` returns `false`, `try_get_dispatcher()` `nullptr`, `ggml_backend_rpc_get_device_count()` `0`, `ggml_backend_rpc_add_server()` `nullptr`, and `add_rpc_devices()` (`common/arg.cpp`) throws `std::invalid_argument` naming the server where it used to register `nullptr` and silently run without it. Every path *after* registration keeps the original contract through `get_dispatcher()`, which still aborts — a server that vanishes mid-inference stays fatal (TODO). **(2)** `add_server()`'s cache hit is re-checked (a stopped server used to pass registration and abort on the first tensor upload), and `ggml_backend_rpc_get_device_memory()` of a gone server returns 0/0 instead of aborting — RPC devices stay in ggml's process-wide registry forever, and `common_init()` queries the memory of **every** registered device at `-lv 4`, so a stopped server would otherwise take down an unrelated later load. **(3)** The server becomes stoppable: `ggml_backend_rpc_stop_server()` (disconnects the client being served, wakes `accept()` with a loopback connection — the one portable way) and `ggml_backend_rpc_server_listening()`; the loop now reaches its cleanup and frees its backends on every early return, but deliberately **not** `rpc_transport_shutdown()`, whose `WSACleanup()` would invalidate every other RPC socket in the process. **(4)** transport: `MSG_NOSIGNAL`/`SO_NOSIGPIPE`, `socket_t::shutdown()`, and the fds `connect()`/`create_server()`/`accept()` leaked on their error paths are closed. Touches `ggml/include/ggml-rpc.h`, `ggml/src/ggml-rpc/{ggml-rpc.cpp,transport.cpp,transport.h}` and one function in `common/arg.cpp` (`add_rpc_devices`, away from `0001`'s `common_params_parse` hunks). **Runnable guard: `src/test/cpp/test_rpc.cpp`** — links both new functions (a bump that drops the patch fails `C++ Tests` at link time on every platform) and drives registration, stop and the memory query over loopback. Upstream-submittable; **not yet filed upstream**. On a bump, check whether upstream grew its own stop function or non-aborting registration (grep `stop_server` / `GGML_ABORT("Failed to connect` in `ggml-rpc.cpp`). **Refreshed at the b11450 bump** (#26610, `-sm tensor` over RPC, ~800 lines in `ggml-rpc.cpp`, protocol 7 → 8): three hunks lost their context -- new dispatcher members beside `start()`, and new `get_proc_address` names before `return NULL` (the two new names now sit after `ggml_backend_comm_allreduce_tensor`); the `+`/`-` lines are unchanged. The server-to-server comm #26610 adds (a listener on `0.0.0.0`, an `accept()` a stop does not wake) is on file in `TODO.md`. | | `0017-ggml-cpu-x86-prefetch-byte-offset-and-msvc-type.patch` | **Makes `GGML_CPU_ALL_VARIANTS=ON` buildable with plain clang on Windows, and fixes a prefetch address that is wrong everywhere.** The four `_mm_prefetch` calls in `ggml/src/ggml-cpu/arch/x86/quants.c` (the SSSE3-without-AVX branch of `ggml_vec_dot_q4_0_q8_0`, and the only `_mm_prefetch` calls in the whole ggml tree) carry two independent defects; both are fixed by routing the address through one new `static inline` helper in the new header `ggml/src/ggml-cpu/arch/x86/prefetch.h`, which is included relatively, so **no CMake change is needed**. **(1) Type -- fatal only on a current clang:** MSVC declares the intrinsic as `void _mm_prefetch(char const *, int)`; clang's own `xmmintrin.h` defines a casting macro but only `#ifndef _MSC_VER`, so on a `*-windows-msvc` target clang leaves the declaration to Microsoft and a typed pointer is `-Wincompatible-pointer-types`. **clang 22 promoted that to an error by default** ([llvm-project #157364](https://github.com/llvm/llvm-project/pull/157364); the [22.1.0 release notes](https://releases.llvm.org/22.1.0/tools/clang/docs/ReleaseNotes.html) add that `-Wno-error=incompatible-pointer-types` downgrades it) -- *not* clang 16, which promoted only `-Wincompatible-FUNCTION-pointer-types`; an earlier version of this row said otherwise. Measured on the real file at `-msse4.2`, unpatched → patched: clang **20.1.8** 4 warnings → 0/0, clang **22.1.3** 4 errors → 0/0, clang **23.1.3** 4 errors → 0/0. Plain clang is exactly what `cmake/x64-windows-llvm.cmake` selects -- and plain clang is the only way to get all **14** CPU variants, because `clang-cl` makes CMake's `MSVC` true and the `if (NOT MSVC)` block in `ggml/src/CMakeLists.txt` then drops `zen4`, `cooperlake`, `sapphirerapids`, `ivybridge` and `piledriver`. **(2) Arithmetic:** `&x[ib] + sizeof(block_q4_0)` adds `sizeof()` *elements*, not bytes -- +324 and +1156 instead of +18 and +34 (the sizes `ggml-common.h` pins by `static_assert`), and +648 instead of +36 for the `2 *` pair. That half is wrong on every platform, including where the call compiles. **Measured:** no throughput difference (see the test-table row), so this is a correctness and portability fix, not a performance one. **Upstream-submittable and meant to be submitted.** **Why upstream's own build is green is measured, not an open question:** their `windows-cpu` release job builds exactly this with plain clang, `GGML_CPU_ALL_VARIANTS=ON` and no suppressing flag (`CMAKE_ARGS` carries only build-target toggles), and ships `ggml-cpu-sse42.dll` -- because the `windows-2025-vs2026` image has **LLVM 20.1.8** on `PATH`, where these are warnings. That is narrower than it looks: the same image also ships **clang 22.1.3** inside Visual Studio 2026 (`VC/Tools/Llvm/x64`), which *fails* on the unpatched file -- upstream reaches the working compiler only because `C:\Program Files/LLVM/bin` comes first on `PATH`. It is **not** a header difference: all three MSVC toolsets on that image (14.29.30133, 14.44.35207, 14.51.36231) declare it identically as `extern void _mm_prefetch(char const*_A, int _Sel);` -- that was the first hypothesis, and a one-off probe workflow (deleted again once it had answered) disproved it by printing the declaration from each toolset on the image. So this patch is **ahead of** upstream rather than diverging: their build breaks the moment the image ships a clang that makes the diagnostic fatal, and until then they compile four warnings and a prefetch address 18x/34x too far. **Runnable guard: `src/test/cpp/test_prefetch.cpp`.** On a bump, run `git grep -n _mm_prefetch ggml/src/ggml-cpu/arch/x86/quants.c` at the new tag: if the call sites already cast **before** the addition, **drop this patch rather than refreshing it**. | | `0016-model-kolibri1.patch` | **Adds Aleph Alpha's Kolibri-1 (architecture `kolibri1`) ahead of upstream** -- a 78B German/English reasoning MoE (3.46B active) that upstream does not support yet ([ggml-org/llama.cpp#29922](https://github.com/ggml-org/llama.cpp/issues/29922), no PR at the time). A **temporary carry, not a fix**: drop it the moment upstream registers the architecture (`git grep -n kolibri src/llama-arch.cpp` at the new tag), and keep `test_kolibri1.cpp` -- its numerical comparisons must then pass against upstream's implementation, while its GGUF-format rows follow whatever format upstream's converter fixes (see the test-table row). **Architecture** (Aleph Alpha's vLLM reference, `aleph_alpha_inference/kolibri1.py` at `049a6a7`): GQA with per-head q/k RMSNorm; `layer_types` interleaves sliding-window layers (NEOX RoPE) with full-attention layers that have **no** positional encoding; sandwich norms around attention and MoE; every layer routed MoE + one ungated shared expert; the router selects top-k on **`logits + expert_bias`** and weights by the **unbiased `sigmoid(logits)`** -- not DeepSeek-V3's sigmoid router (selection on `sigmoid(logits) + bias`), which picks other experts as soon as the bias is non-zero. The graph therefore builds the selection itself and passes it to `build_moe_ffn` (`selected_experts_in`), leaving upstream's shared MoE code untouched. **Derived from two community ports, which are incompatible with each other's GGUFs** -- Eliasfpv28's AFMoE-based one (base b11378, `expert_gating_func` 2, pre-tokenizer `qwen2`) and the Qwen3-MoE-based one behind Hob-forge's GGUFs (base b11381, as carried in [mjwsolo/localcode#101](https://github.com/mjwsolo/localcode/pull/101): gating 5, pre-tokenizer `kolibri1`, rejects gating 2). This patch **loads both dialects**: gating 2, 5 or absent (all mean the one router), `kolibri1` mapped to the Qwen2 pre-tokenizer in `llama-vocab.cpp`, `expert_weights_norm` / `expert_shared_count` / `output.weight` optional. Every source, with license, is listed in the patch header; `REUSE.toml` annotates the file `MIT AND Apache-2.0` (Eliasfpv28's additions are Apache-2.0). Touches `src/llama-arch.{h,cpp}`, `src/llama-model.cpp` (mapping + NEOX rope list), `src/llama-vocab.cpp` (one name), `src/models/models.h` and the new `src/models/kolibri1.cpp`; no converter. **Runnable guard: `src/test/cpp/test_kolibri1.cpp`** (writes tiny random GGUFs in both dialects and compares every logit with an independent double-precision reference). **Not verified here:** the real 78B model (HuggingFace is unreachable from the sessions this was written in, and the smallest GGUF is 28.6 GB) and GPU backends -- the community ports report real-model checks (top-1 67/67 against a float64 reference for Q8_0, tool calls, German text). | -| `0018-ggml-amx-free-the-aligned-buffer.patch` | **The AMX weight buffer is freed with `free()` but allocated with `_aligned_malloc()` on Windows -- `STATUS_HEAP_CORRUPTION` (0xC0000374) the moment a model is unloaded.** `ggml_backend_amx_buffer_type_alloc_buffer()` (`ggml/src/ggml-cpu/amx/amx.cpp`) allocates with `ggml_aligned_malloc()`, which is `_aligned_malloc()` under `_MSC_VER` (plain clang on a `*-windows-msvc` target defines it too); `ggml_backend_amx_buffer_free_buffer()` called `free()` on it. Fine on Linux (`posix_memalign` + `free`), fatal on Windows, and only there. The patch frees with `ggml_aligned_free(buffer->context, buffer->size)`, exactly what `ggml_backend_cpu_buffer_free_buffer()` does; on Linux that is `free()`, so nothing changes there. **Reached by** the `sapphirerapids` CPU module (the only variant built with `AMX_TILE AMX_INT8`, which `ggml_backend_load_best` picks on Sapphire/Emerald/Granite Rapids; `ggml_amx_init()` is `return true` on `_WIN32`) and a model whose weights have AMX kernels (Q4_0, Q4_1, Q8_0, Q4_K, Q5_K, Q6_K, IQ4_XS -- i.e. every CI model). **Found by the second full run of the modular natives** (38069595101): `Java Tests Windows 2025 x86_64 (clang)` landed on an `Intel(R) Xeon(R) 6973P-C` runner (Granite Rapids), loaded `ggml-cpu-sapphirerapids.dll`, ran `GrammarExample`'s two generations fine and died with exit code -1073740940 at `LlamaModel.close()`; the first run (38065017595) had the same job green on an AMD EPYC 7763 (no AVX-512, `haswell`). **The Windows runners are a CPU lottery**, so which CPU module a Windows job exercises differs per run; before the modular natives the Windows jar was a static Haswell-level build and this code never ran. **No runnable guard in `jllama_test`**: the AMX code exists only in that one variant of a `GGML_CPU_ALL_VARIANTS` build and executes only on AMX hardware, which the static test build, the local agent's Zen 3 and the cloud sandboxes lack. What exercises it: the Windows x86-64 Java test job and the `windows-x86-64` smoke row whenever GitHub hands them an Intel AMX runner; the patch itself is held by `check-patches.py`, applies at b11538, and the patched file compiles with the variant's flags (GCC 13, `-mamx-tile -mamx-int8 -mavx512vnni ...`). Upstream master still carried the `free()` on 2026-10-10; **not yet filed upstream** (nothing before the release). **Drop-check on every bump:** `grep -n "free(buffer->context)" ggml/src/ggml-cpu/amx/amx.cpp` at the new tag -- gone means upstream fixed it, drop the patch. | +| `0018-ggml-amx-free-the-aligned-buffer.patch` | **The AMX weight buffer is freed with `free()` but allocated with `_aligned_malloc()` on Windows -- `STATUS_HEAP_CORRUPTION` (0xC0000374) the moment a model is unloaded.** `ggml_backend_amx_buffer_type_alloc_buffer()` (`ggml/src/ggml-cpu/amx/amx.cpp`) allocates with `ggml_aligned_malloc()`, which is `_aligned_malloc()` under `_MSC_VER` (plain clang on a `*-windows-msvc` target defines it too); `ggml_backend_amx_buffer_free_buffer()` called `free()` on it. Fine on Linux (`posix_memalign` + `free`), fatal on Windows, and only there. The patch frees with `ggml_aligned_free(buffer->context, buffer->size)`, exactly what `ggml_backend_cpu_buffer_free_buffer()` does; on Linux that is `free()`, so nothing changes there. **Reached by** the `sapphirerapids` CPU module (the only variant built with `AMX_TILE AMX_INT8`, which `ggml_backend_load_best` picks on Sapphire/Emerald/Granite Rapids; `ggml_amx_init()` is `return true` on `_WIN32`) and a model whose weights have AMX kernels (Q4_0, Q4_1, Q8_0, Q4_K, Q5_K, Q6_K, IQ4_XS -- i.e. every CI model). **Found by the second full run of the modular natives** (38069595101): `Java Tests Windows 2025 x86_64 (clang)` landed on an `Intel(R) Xeon(R) 6973P-C` runner (Granite Rapids), loaded `ggml-cpu-sapphirerapids.dll`, ran `GrammarExample`'s two generations fine and died with exit code -1073740940 at `LlamaModel.close()`; the first run (38065017595) had the same job green on an AMD EPYC 7763 (no AVX-512, `haswell`). **The Windows runners are a CPU lottery**, so which CPU module a Windows job exercises differs per run; before the modular natives the Windows jar was a static Haswell-level build and this code never ran. **Of the six callers of `ggml_aligned_malloc` in the tree this was the only mismatched pair** (`ggml-backend.cpp`, `ggml-cpu.c` twice, `ggml-openvino.cpp`, `ggml-zdnn.cpp` and `ggml.c` all pair correctly), and `ggml-backend.cpp:2464` is the line the fix copies. **Reproduced deliberately rather than waiting for the lottery again**: on a rented Granite Rapids machine, one CPU module staged at a time with the same model and the DLLs CI itself built, `sapphirerapids` crashed **6 of 6** runs -- including at **one** thread, which rules out a data race -- while `cooperlake`, `icelake` and `haswell` were clean 6 of 6, and on Linux with the same CPU all six exit 0 with byte-identical output. The crash lands *after* the inference prints its timings, in cleanup: the signature of a heap corrupted earlier and detected at `free`. With the patch applied the same matrix on the same machine is **clean in all twelve runs**. **The guard is a configure-time check, not a `jllama_test` case**: the AMX code exists only in that one variant of a `GGML_CPU_ALL_VARIANTS` build and executes only on AMX hardware, which the static test build, the local agent's Zen 3 and the cloud sandboxes lack -- so **`llama/cmake/check-aligned-free-pairing.cmake`** fails the configure on **every** platform when a ggml source calls `ggml_aligned_malloc` and never `ggml_aligned_free` (verified red against the unpatched source, naming `amx.cpp`, and green with this patch). A run-time test cannot replace it: without an AMX CPU there is nothing to observe, which is exactly how this shipped. What exercises it: the Windows x86-64 Java test job and the `windows-x86-64` smoke row whenever GitHub hands them an Intel AMX runner; the patch itself is held by `check-patches.py`, applies at b11538, and the patched file compiles with the variant's flags (GCC 13, `-mamx-tile -mamx-int8 -mavx512vnni ...`). Upstream master still carried the `free()` on 2026-10-10; **not yet filed upstream** (nothing before the release). **Drop-check on every bump:** `grep -n "free(buffer->context)" ggml/src/ggml-cpu/amx/amx.cpp` at the new tag -- gone means upstream fixed it, drop the patch. | **Dropped patches** (`0004`, `0005`, `0009`, `0010`, `0011`, `0013` -- upstream fixed each defect itself) are recorded in [`docs/history/dropped-llama-patches.md`](docs/history/dropped-llama-patches.md): diff --git a/TODO.md b/TODO.md index 2cbbabcc2..ced4ea6c4 100644 --- a/TODO.md +++ b/TODO.md @@ -17,6 +17,25 @@ so everything below is genuinely still open. ## Open — jllama-specific +### `package` is red: the Windows OpenVINO module jar fails the merge check + +**This blocks every release and was untracked until 2026-10-11.** `Package JARs` fails with + + artifact 'natives-openvino-windows-x86-64' is a module jar: + Windows/x86_64/openvino/ must hold ggml-openvino.dll (named in jllama-files.txt) + +so `merge-native-artifacts.sh` rejects the artifact before anything is packaged. Because the smoke +jobs `need: package`, all of them **skip** rather than fail -- which is why the run reads as "3 +failures" and the Java test jobs look green: nothing downstream of `package` ran at all. Seen on run +38112603436; the check itself came with `4f811a23` on `main`, so this is **pre-existing**, not a +consequence of the AMX work on the branch. + +What to find out first: whether the Windows OpenVINO build produces a differently named module (the +other GPU jobs write `ggml-.dll`, and `JLLAMA_MODULE_ONLY` writes `$` into +`jllama-files.txt`, so a target name mismatch would show exactly like this) or produces nothing and +the upload's `if-no-files-found: error` did not catch it. The two GPG jobs red in the same run are +**unrelated and expected** on a branch: they need the `maven-central` environment secret. + ### Modular natives -- follow-ups Every platform a GPU can join ships modular since 5.2.0 (CLAUDE.md "Modular natives": `JLLAMA_BACKEND_DL` @@ -173,6 +192,46 @@ round-trips — see CLAUDE.md "Two server modes"). **Owner priority: the native- Responses client) at a running server, since round-trips confirm wire shapes but not each client's parser. +### Code-sign the Windows DLLs (Azure Trusted Signing, owner's decision -- it costs money) + +**What.** Microsoft's **Azure Trusted Signing** (formerly Azure Code Signing) signs Windows PE files +from CI without a hardware token, on a monthly subscription -- the **Basic** tier is in the ~10 +USD/month range and includes a signature quota; **verify the current price and quota before +committing**, and whether the quota covers signing *18 files per Windows natives jar* on every run +(`jllama.dll`, `ggml.dll`, `ggml-base.dll`, `ggml-rpc.dll` and the 14 `ggml-cpu-.dll`, plus +one module per Windows GPU jar). The traditional alternative -- an OV/EV certificate -- is several +hundred euros a year **and** has required a hardware token since 2023, which a GitHub runner cannot +use, so it is not a real option for this pipeline. + +**Why it is worth considering here specifically.** This project's DLLs are **extracted from a jar +into a temp directory at run time and loaded with `System.load`**, which is exactly the shape +antivirus and EDR products treat as suspicious: an unsigned native library appearing in `%TEMP%` and +being loaded by `java.exe`. A signature does not silence a SmartScreen installer prompt (there is no +installer), but it is what lets an enterprise allowlist the files by publisher instead of by hash, +and it removes the "unknown publisher" class of false positive that a consumer cannot debug. There +is no evidence of a user hitting this yet -- so this is prevention, not a fix, which is why it is a +decision rather than a task. + +**The gatekeeper, and the reason to check it first:** Trusted Signing requires **identity +validation**. An organisation needs a verifiable legal entity; an **individual** must have a +verifiable public history (Microsoft has required ~3 years), and the account must be in a supported +region. **Find out whether the account qualifies before paying for anything** -- the subscription is +useless without a validated identity, and that is the step that fails, not the plumbing. + +**How it would hook into GitHub.** `azure/trusted-signing-action` with federated OIDC credentials +(no long-lived secret), run in the Windows build jobs *after* the build and *before* the artifact +upload, so what gets packaged is already signed. Note what that touches on our side: signing +**modifies the files**, so it must happen before `merge-native-artifacts.sh` and +`package-smoke-sets.sh` -- both compare files byte for byte -- and the signature must survive the +jar round trip (it does; a PE signature is part of the file). Signing *after* packaging would mean +re-writing jars and re-checking them, which is strictly worse. + +**What it does not cover.** Linux has no equivalent and needs none. **macOS is a separate purchase +and a separate problem**: a Developer ID certificate (Apple Developer Program, ~99 USD/year) plus +notarisation, and the dylib is already ad-hoc signed by the linker -- which is what +`smoke-natives-macos` verifies with `codesign --verify --strict` after a corrupt dylib shipped in +three releases. Do not bundle the two decisions; the Windows one is cheap and self-contained. + ### License compliance gate (owner's decision) The FSFE REUSE check is green on every run. A FOSSA-style *dependency-license* gate (the scanner @@ -185,7 +244,7 @@ nothing here blocks on it. **Nothing is filed upstream before the release is out**, neither issues nor PRs; the drafts are not written before that either. Everything below is the inventory for that day, so it is not rediscovered. -There are **eleven** patches today (`0001`-`0003`, `0006`-`0008`, `0012`, `0014`-`0017`). Each +There are **twelve** patches today (`0001`-`0003`, `0006`-`0008`, `0012`, `0014`-`0018`). Each accepted upstream change deletes a patch from the bump checklist once the pin is bumped past it. - **Submittable as they are** (self-contained, no jllama specifics): `0001` (Windows arg-parse embed @@ -194,7 +253,11 @@ accepted upstream change deletes a patch from the bump checklist once the pin is direction), `0002` (preserve the caller's load-progress callback), `0006` (embeddable `llama_server`), `0007` (`llama_server_attach`), `0008` (`LLAMA_SERVER_WORKER_CMD`), `0012` (zero split-sum guard, ships `tests/test-model-split.cpp`), `0014` (`common_log_set_callback`), `0015` - (non-aborting RPC registration, stoppable server, transport fixes), `0017` (x86 prefetch helper). + (non-aborting RPC registration, stoppable server, transport fixes), `0017` (x86 prefetch helper), + **`0018`** (the AMX buffer released with `free()` instead of `ggml_aligned_free` -- one line, with + the 6/6-against-0/6 measurement in its header and a configure-time guard that is red without it; + the clearest submission of the set, since the defect reproduces on any AMX machine and the fix + copies the line `ggml-backend.cpp` already has). - **Permanent:** `0003` carries upstream PR #22393, which upstream closed without merging. - **Temporary carry, not a submission:** `0016` (Kolibri-1). Upstream will add the architecture itself ([ggml-org/llama.cpp#29922](https://github.com/ggml-org/llama.cpp/issues/29922)). Drop it on the @@ -212,6 +275,14 @@ accepted upstream change deletes a patch from the bump checklist once the pin is carries an `-Xclang` argument, which on Windows every plain-clang TU does -- CMake's runtime-library selection puts it there -- so no plain-clang Windows build caches at all (CLAUDE.md "Windows natives"); that one is an sccache issue, if none is open. +- **A third, measured 2026-10-11: `ggml_backend_load_best` picks a slower CPU module than one it + rejects.** On a Graviton 4 it selects `armv8.6_2` (198.0 t/s pp512) while `armv8.2_2` reaches + **271.4** on the same machine and model -- the SVE/SVE2/i8mm paths losing to plain DOTPROD+FP16 -- + with generation inside the noise. The scoring function ranks by feature *level*, not by measured + throughput, so it is by construction rather than a bug, but it costs 37 % of prompt processing on + the hardware AWS rents most cheaply, and it is invisible to anyone who only runs the module their + own CPU picks. Worth an issue with the table from `docs/local-test-plan.md`; **no patch here** -- + overriding upstream's scoring downstream would be a maintenance burden with no end. (`0009`, `0010`, `0011` and `0013` are gone -- upstream fixed each defect itself -- and recorded in `docs/history/dropped-llama-patches.md`.) @@ -325,10 +396,33 @@ The consolidated investigation lives in high-value from it has shipped, the jbang one-file example (`examples/jbang/Chat.java`) included. Further per-repo findings in the doc can be pulled on demand; none is prioritized. +### The ROCm Windows job runs with a CRT strategy that was never measured + +**Recovered from the session transcript on 2026-10-11; it had been stated repeatedly and recorded +nowhere.** The hybrid-CRT change reaches the ROCm Windows build through its plain-clang branch, and +that branch applies because the job compiles with HIP's own clang (`lib/llvm/bin/clang.exe` under +driver, so CMake's `MSVC` is false there. Before the change that job had **no** branch and therefore +the dynamic CRT; it now gets the hybrid one (`-fms-runtime-lib=static` plus +`/nodefaultlib:libucrt.lib`). That may be an improvement (fewer dependencies) or it may break the +link, and it is exactly the `/MT`-against-`/MD` mixture across library boundaries that the comment in +`CMakeLists.txt` warns about for SYCL and OpenVINO: **ROCm's device libraries may be built against +the dynamic CRT.** + +The job has been **green** since, so the link holds. What is unmeasured is the **runtime** behaviour +of the resulting module on an actual Radeon GPU -- and that cannot be rented (see the rented-hardware +plan), because the only AMD-GPU instance class has been withdrawn and never officially supported +ROCm anyway. So this needs a real Radeon machine, or a deliberate decision to ship it on the strength +of a green link plus `verify-native-deps.py`'s dependency allowlist. Do not quietly treat "the job is +green" as the measurement. + ### Needs a machine the sessions do not have Everything here is implemented or decided; what is missing is hardware or a model the cloud sessions -cannot reach. The local agent (Windows, Ryzen 7 5800H + RTX 3070 + AMD iGPU) covers the first three. +cannot reach. **Most of it is rentable** -- see +[`docs/rented-hardware-test-plan.md`](docs/rented-hardware-test-plan.md) for the queue, the +invariants and what rented hardware explicitly cannot answer. The items that stay with a developer +machine are the **interactive agent** (console behaviour needs a human at a real terminal) and a real +**arm64 Android device**. - **The GPU module jars on a GPU**: item 1 of "Modular natives -- follow-ups" above (CUDA + Vulkan modules next to the Windows CPU jar, devices, offload, the CUDA graphs re-measurement). @@ -340,13 +434,109 @@ cannot reach. The local agent (Windows, Ryzen 7 5800H + RTX 3070 + AMD iGPU) cov and UI-tested there too; what no session has is a real arm64 device, and the Adreno/OpenCL flavor needs one. Nothing to write, only to run. - **AVX-512/AMX hardware** (Sapphire Rapids, Zen 4/5, Core Ultra): item 2 of "Modular natives" above. - And on **Windows** with AMX specifically: `patches/0018` (the AMX weight buffer freed with - `free()` after `_aligned_malloc()`, CLAUDE.md patch table) is proven by reading, by a GCC compile - of the patched file with the variant's flags, and by the CI lottery -- the Windows test jobs - reach the AMX path only when GitHub hands them an Intel AMX runner (seen: Xeon 6973P-C). A - Windows machine with AMX would pin it directly: load a Q4_K model with the modular set at `-lv 4` - (`AMX model buffer size` in the load log), generate, close -- no exit code -1073740940. -- **Kolibri-1 on the real 78B model** (after a full build; the shards need more than 64 GB of RAM). + Partly answered since: a Cascade Lake machine (`avx512f` + `avx512_vnni`, no AMX) picks + `cascadelake` and loads cleanly, which lifted the modules ever *chosen* anywhere from six to eight. + AMX itself was that **suspected defect** and is now a found and fixed one (`patches/0018`, proven on + a rented Granite Rapids machine and verified there with the patch -- see the rented-hardware section + below; it is additionally backed by a GCC compile of the patched file with the variant's own + flags, `-mamx-tile -mamx-int8 -mavx512vnni`). What stays open is the original question, the *throughput* gain of `sapphirerapids` and + `zen4` over `haswell`, which needs the same hardware but no longer blocks anything. +- **Kolibri-1 on the real 78B model.** `patches/0016` is verified only against tiny synthesized GGUFs + (`test_kolibri1.cpp`); CLAUDE.md's own row says "Not verified here: the real 78B model ... and GPU + backends". **The RAM objection is measured and wrong for the small quants:** the smallest published + GGUF of each dialect is **Q2_K at 26.7 GB** (gating 5, pre-tokenizer `kolibri1`) and **Q3_K_S at + 31.5 GB** (gating 2, `qwen2`), both of which fit in 63 GB without paging -- only Q6_K (59.8 GB) does + not. What is open is the run, on the published set: both dialects must load, a German prompt must + come back coherent (the router selects on `logits + expert_bias` but weights by the *unbiased* + `sigmoid(logits)`, and a DeepSeek-V3-style router produces fluent-looking nonsense instead), a + thinking block, a tool call through the chat endpoint with `--jinja`, tokens/s and peak RAM. A load + error or gibberish is a finding **against** `0016`, not something to work around. + **In flight since 2026-10-11:** a single rented run on an `r7i.4xlarge` (16 vCPU, 128 GB RAM, + ~1.06 USD/h, dead-man switch at 150 min) loads **both** dialects in one boot, downloading each GGUF + straight from HuggingFace on the instance -- `Hob-forge/Kolibri-1-GGUF` **Q4_K_M (44.2 GB)** rather + than its Q2_K, because the question is whether the German answer is *coherent* and a 78B MoE at + Q2_K would make that ambiguous, and `Eliasfpv28/Kolibri-1-Q3_K_S-GGUF` **Q3_K_S (31.5 GB)** for the + gating-2 dialect. Hob-forge publishes Q2_K through Q8_0, so quant choice is free; 128 GB fits even + Q6_K (59.8 GB), which no local machine does. The probe prints the architecture, expert and gating + lines from the load at verbosity 4, then the generated **text** of two raw completions and one chat + turn through the model's own template, each guarded separately so one failure still leaves the rest + measured. + +### Compatibility matrix on rented hardware (one measurement per generation) + +The project ships **14 x86-64 CPU modules and 8 aarch64 ones**, of which `ggml_backend_load_best` +keeps exactly one -- the best for the running CPU. So a module is only ever exercised *as the chosen +backend* on a machine whose CPU prefers it, and no amount of running the suite on one machine or one +CI runner changes that. Measured coverage today is **8 of 14 x86 levels** (six on a Zen 3 +workstation, `skylakex` and `cascadelake` added by a Cascade Lake machine) and **0 of 8 aarch64 +levels**. + +Rented cloud instances can close most of that, one short run each, and it is a *per-generation +measurement* -- not something CI should repeat. Each family maps to exactly one module: + +| instance family | CPU | module it exercises | +|---|---|---| +| `m8i` | Granite Rapids | **`sapphirerapids`** (AMX) | +| `m7i` / `c7i` | Sapphire Rapids | `sapphirerapids` (AMX) | +| `m6i` / `c6i` | Ice Lake-SP | `icelake` | +| `m7a` / `c7a` | EPYC Genoa (Zen 4) | **`zen4`** | +| `m5n` / `c5n` | Cascade Lake | `cascadelake` | +| `m5` / `c5` | Skylake-SP | `skylakex` | +| `m6a` / `c6a` | EPYC Milan (Zen 3) | `haswell` | +| `m4` and older | Broadwell / Ivy Bridge | `haswell`, `ivybridge` | +| **`m7g` / `c8g` (Graviton)** | **aarch64** | **done: 6 of the 8 run on a Graviton 4** (`armv9.2_*` need SME, which no instance has) | + +**Four levels stay unreachable there** and are only ever *load*-tested: `piledriver` (no +Bulldozer-era instance exists), `cannonlake` (barely shipped), `alderlake` (consumer Intel) and +`cooperlake`. + +**Graviton was the biggest gain per euro, and it has been taken**: 6 of the 8 aarch64 modules are now +measured with real inference on a Graviton 4 (`docs/local-test-plan.md` has the table). +`armv9.2_1`/`armv9.2_2` need **SME**, which no AWS instance has, so they stay unmeasured. Two things +came out of it: the two SME modules decline *cleanly* (no CPU backend, `LlamaException`, not a +crash), and **ggml picks a module that is 37 % slower at prompt processing than a lower one** -- +`armv8.6_2` at 198 t/s against `armv8.2_2` at 271 t/s on the same machine and model. That is a +scoring question for upstream and invisible to anyone who only runs what the CPU picks. + +GPU backends, same idea, with two that do not work out: + +| backend | rentable | +|---|---| +| CUDA, Vulkan, OpenCL | yes -- T4 / A10G / L4 class instances are enough | +| OpenVINO | yes (CPU) | +| **ROCm** | **no** -- the one AMD-GPU instance family is not an officially supported ROCm target | +| **SYCL** | **no** -- needs an Intel GPU, which is not offered | + +**How to run these, and the invariants that came out of doing it, are in +[`docs/rented-hardware-test-plan.md`](docs/rented-hardware-test-plan.md)** -- including the one that +costs days when missed: a GPU instance family starts at a **zero vCPU quota** on a fresh account, and +raising it is a support request measured in hours to days, so ask before the run is due. The large +accelerator families (8-GPU training instances) prove nothing here that a single small GPU does not, +and cost two orders of magnitude more. + +**The AMX row is closed.** It was a suspected defect, turned out to be one line of upstream ggml +(the AMX buffer released with `free()` instead of `ggml_aligned_free`), and is fixed as +`patches/0018` -- proven on a rented Granite Rapids machine (`sapphirerapids` 6/6 crash against 0/6 +for the other modules, Linux unaffected) and verified there again with the patch (12/12 clean). The +full record is in the patch header, `CLAUDE.md`'s patch table, `CHANGELOG.md` and the module table in +`docs/local-test-plan.md`; `llama/cmake/check-aligned-free-pairing.cmake` is the guard. **What stays +open is only the original question** -- the *throughput* gain of `sapphirerapids` and `zen4` over +`haswell` -- plus one rule worth keeping: a green Windows job proves nothing unless its log names an +AMX CPU, which is why `print-host-info.sh` now runs in every job that touches native code. + +### AIX on POWER -- an open porting question, not a test + +Not supported and never built. Worth a line because the access exists (an AIX 7.3 POWER8 machine on +the GCC Compile Farm) and because it would be the **second big-endian platform** after s390x -- +which is exactly the coverage s390x was added for, so the endian-sensitive surface of *our* layer is +already guarded by `ctest` under qemu there. + +What makes it a port rather than a run: AIX is big-endian POWER with **XCOFF**, not ELF, so the +loader's assumptions (`$ORIGIN`-style run paths, `dlopen` semantics, `verify-native-deps.py`'s ELF +reader) do not carry over; upstream llama.cpp does not target AIX; and a JVM there means IBM Semeru, +not Temurin. **Nothing is scheduled**: the useful first step is a plain `cmake` build of the static +library on that machine to see how far it gets and what breaks, with the result written down -- +before anything is promised about a natives jar. ### GraalVM Native Image -- evaluated, parked until a consumer asks @@ -428,3 +618,133 @@ flags, the `acquire`/`release` context-guard unit tests, the `OSInfo` alias test - **Additional ArchUnit rules to consider** — the full **`layeredArchitecture()`** rule and a **per-module banned-import** rule (`jacksonBannedFromContractsAndLoader` — Jackson kept out of `args`/`callback`/`exception`/`loader`) are now DONE. Still open: more per-module banned-imports if useful, public-API-surface constraints (no public mutable static state, etc.). Partial progress: `7b6667d` covers the "no public field that is not final" sub-rule. - **Cross-repo code-quality TODOs** — see [`../workspace/policies/code-quality-todos.md`](../workspace/policies/code-quality-todos.md) for the canonical `@VisibleForTesting` design-fit review, package hierarchy review, and class/method naming review. This repo has no `@VisibleForTesting` usages today; package and naming reviews remain open. + +## GPU backends on rented hardware + +**AWS blocks this outright on a new account: the G-instance vCPU quota is 0.** A `g4dn.xlarge` +launch (Tesla T4, ~0.21 USD/h, so cents for a probe) is refused with +`VcpuLimitExceeded ... your current vCPU limit of 0`, and raising it is a support request the +account owner has to file -- the probe package (CPU + `cuda13` + `vulkan` module jars, 169 MB) and +the driver-equipped AMI (`Deep Learning Base OSS Nvidia Driver GPU AMI (Amazon Linux 2023)`) are +otherwise ready, so this is a one-line unblock once the quota exists. Cost was never the obstacle. + +**Which GPU backends are reachable on rented hardware at all, measured 2026-10-11:** + +| backend | rentable | how | +|---|---|---| +| `cuda13` | yes | G instances (T4 / A10G / L4 / L40S). A T4 proves as much as an H100 for "the module loads and computes", so the **P** buckets (V100/A100/H100) are deliberately not requested | +| `vulkan` | yes | the same NVIDIA cards -- the driver supplies `vulkan-1` / `libvulkan.so.1` | +| `opencl` | yes | the same cards -- NVIDIA ships an ICD | +| `openvino` | **yes, with no GPU at all** | OpenVINO has a CPU plugin, so an `m7i`/`m8i` can load `libggml-openvino.so`. Currently build-only; this is the cheapest open coverage gap | +| `rocm` | **no** | `g4ad` (Radeon Pro V520) is **gone from every region checked** (us-east-1, us-east-2, us-west-2, eu-central-1) and was AWS's only AMD GPU; ROCm never officially supported Navi 12 either. Stays build-only | +| `sycl` (GPU) | **no** | AWS has no discrete Intel GPU. Intel's CPU OpenCL runtime covers part of it | +| `metal` | **no** | EC2 Mac exists but bills a 24-hour minimum, and macOS ships the backend inside the library anyway | + +**Spot cannot be stopped**, only terminated, so a persistent switch-off-in-between machine needs the +**On-Demand** G quota (`L-DB2E81BA`), not the Spot one. Measured spot price for `g4dn.xlarge` on +2026-10-11: **0.2925 USD/h**; On-Demand is roughly 0.53. A stopped instance costs only its EBS +volume (40 GB gp3 ~ 3 EUR/month), and starts in ~40 s with the toolchain and models still on it -- +against 10 to 15 minutes for the boot-from-scratch probes used so far. + +**`SME` is not rentable anywhere, and an Apple instance does not help.** AWS does offer +`mac-m4.metal` / `mac-m4pro.metal` / `mac-m4max.metal`, and Apple M4 does have SME -- but macOS +ships **one static library** (`metal-macos-aarch64`, Metal compiled in), not the aarch64 CPU module +set, so `armv9.2_1`/`armv9.2_2` cannot even be loaded there. They would need **Linux aarch64 with +SME**, which Graviton 4 lacks and no current instance provides. EC2 Mac also bills a **24-hour +minimum** (dedicated host), so it is the expensive way to answer nothing. + + +## Handed over from the modular-natives session (2026-10-11) -- read the first item before merging #507 + +Appended on `local/cpu-module-isolation-test` by the session that shipped the modular natives +(#504 to #506) and watched their two full runs (38065017595, 38069595101). Nothing above was moved. +Items `main` already carries through #506 are named, not repeated. + +### Before #507 merges: two `0018` patches for the same hunk + +`main` (cfc10fd2, #506) carries `llama/patches/0018-ggml-amx-free-the-aligned-buffer.patch`; this +branch carries `llama/patches/0018-ggml-amx-aligned-free.patch`. Both replace the same line of +`ggml/src/ggml-cpu/amx/amx.cpp` (main's adds a four-line comment above the call, this one is bare). +After a merge both files exist, the applier takes them in filename order, the first applies, the +second no longer finds its `-` line and aborts the configure with "does not apply cleanly" -- every +native build job red, on every platform. `git merge-tree main HEAD` (2026-10-11) reports a textual +conflict in `TODO.md` **only**; the two patch files, CLAUDE.md (two `0018` rows in the patch table) +and CHANGELOG.md (two "Fixed" bullets for the same defect) merge *silently*. Resolve in the merge +commit: keep exactly one patch file (the comment is the only difference), one patch-table row and +one CHANGELOG bullet, and keep this branch's `llama/cmake/check-aligned-free-pairing.cmake` guard +either way (main has none). For the `TODO.md` conflict: take main's wording for the three places +#506 touched -- a fifth "Modular natives -- follow-ups" item (the Windows ROCm job is the pipeline's +critical path at 4:15 h and builds uncached), the corrected sccache sentence under "Upstream +submissions" (the `-Xclang` that sccache 0.18.0 refuses comes from CMake's +`Platform/Windows-Clang.cmake` runtime-library selection on every plain-clang Windows TU, not only +from llama.cpp's `-fno-pch-timestamp`; CLAUDE.md "sccache on every Windows Ninja job but one" on +main has the measurement) and the AMX bullet under "Needs a machine" -- and this branch's structure +for everything else. + +### Open from that session, recorded nowhere else + +- **WER minidumps are switched on but unproven.** #506 makes the Windows test step set + `Disabled=0` / `DontShowUI=1` under both WER keys next to the `LocalDumps\java.exe` entry + (`DumpType` 2), because the AMX heap corruption left no dump behind. No Windows crash has happened + since, so nothing shows that a `.dmp` now lands in the failure artifact. Prove it once rather + than wait: a throwaway step that crashes a `java.exe` on purpose (an access violation through + `sun.misc.Unsafe`, or `jshell` writing to address 0) on a branch run, then look for the dump in + the artifact -- and if none appears, replace the WER route with `procdump -ma -e` around the + surefire run. +- **The emulator package pre-install is unproven.** Both emulator jobs died in one minute inside the + action's own `sdkmanager --install emulator --channel=0` ("Error on ZipFile unknown archive", run + 38069595101) an hour after being green; #506 installs the package in a step before the action, + four attempts, so the action's install is a no-op. Neither job has run since. If the download + keeps flaking, the sturdier form is `actions/cache` over `$ANDROID_HOME/emulator` keyed on the + emulator package version `sdkmanager --list` reports, so dl.google.com is reached once per + version instead of once per job. +- **Everything behind `package` has never run on the modular natives.** Run 38069595101 skipped + `package` (it `needs` the Windows clang test job, red with the AMX crash); this branch's run + 38112603436 reaches it and fails on the Windows OpenVINO module jar (the blocker at the top of + this file). Still unexercised after that: the smoke-set assembly (`package-smoke-sets.sh`), + `smoke-natives-jars.sh` (classes + all 23 natives jars on the classpath **and** the module path), + the bytecode gate and `verify-native-deps.py` over all 23 jars, the four `smoke-natives` rows + (loader line naming every module jar of the platform, the HTTPS round, the RPC smoke, the + bytecode gate over the set), `smoke-natives-macos`, `smoke-agent-linux`, and the publish jobs + attaching the Maven artifacts. Expect further rounds there; the `windows-x86-64` smoke row runs + on the same CPU lottery as the Windows test job, so it exercises AMX only when GitHub hands it an + Intel runner. +- **The two server defaults disagree, by design or not -- owner's call.** Upstream #30159 (b11529) + moved llama-server's default port from 8080 to 9931, which `NativeServer.DEFAULT_PORT` follows + because it inherits the forwarded argv; `OpenAiServerConfig.DEFAULT_PORT` stayed 8080 as the + project's own constant. Raised 2026-10-09, never answered: align `OpenAiCompatServer` to 9931 + (one constant, its tests, README) or keep the split and say so in the README. +- **Maven plugins without a pinned version, and no `requirePluginVersions` enforcer rule** (all + four repos; raised 2026-10-01, never answered). `clean`, `resources`, `install`, `deploy`, `site` + and in places `compiler`/`jar` run at whatever version the Maven on the runner defaults to, which + differs between CI and a local build (the parent's `` here covers the plugins + more than one module uses, not these). `check-versions.py` would then also compare them across + the siblings. One pull request per repo, warnings-only until all four have landed. +- **`SECURITY.md` and `CONTRIBUTING.md` are the two per-repo files never compared.** Every other + file that was "probably the same" across the four repos is either in `shared-files.sha256` + byte-for-byte or as a `?repo` entry; these two were left out on 2026-10-01 as "differs per repo, + probably wanted, not checked". Diff them once and either add `?repo` entries or write down why + they differ. +- **SonarCloud reported "1 new issue" on #486 and nobody looked.** The sandbox's proxy answers 403 + for sonarcloud.io, so the finding was never read; the quality gate has passed since, so it is + low, but it is unread. +- **The local-agent assignment of 2026-10-10 exists only in the session, not in `docs/handover/`.** + Its deliverables map onto "Modular natives -- follow-ups" item 1 (the module jars on the RTX 3070 + + iGPU, throughput against 265.6 / 88.8 / 243.7, `--device` names across module sets, the + filter's negative paths), the first two "Needs a machine" bullets (extraction reuse, stale + cleanup against a running JVM) and the Kolibri bullet; the agent was asked to write + `docs/handover/local-agent-report-5.2.0-modular-windows.md` on a branch + `local/modular-natives-windows-report`. One question in it is not in item 1 yet: NVIDIA's and + AMD's drivers ship OpenCL ICDs, so CUDA, Vulkan **and** OpenCL may each see the same RTX 3070 -- + whether llama.cpp folds the three into one device by `device_id` and which backend wins is + unmeasured. The input it waits for is the first green `llama-smoke-windows-x86-64` artifact. +- **REUSE attribution is probably wrong in two places, and changing it needs the owner's word.** + `REUSE.toml` credits the generated dockcross wrappers (`.github/dockcross/dockcross-*`) to the + original fork author and the owner, while they are output of the dockcross project; and about 35 + Java files plus the documentation block still name the fork author, which a `git blame` pass + would have to confirm or refute file by file. Raised 2026-10-01; no attribution is removed + without an explicit decision. +- **srcmorph's smoke tests a fat jar it never ships** (its thin Central jars are launched by no job, + its 15 classifier fat jars are checked structurally only). Whether srcmorph follows jllama -- thin + jar plus `jbang --deps`, smoke on the consumer's classpath -- is the owner's decision and is + recorded in srcmorph's `TODO.md` and the workspace fat-jar policy, not here. diff --git a/docs/local-test-plan.md b/docs/local-test-plan.md index 397bd6ef3..dd7811e93 100644 --- a/docs/local-test-plan.md +++ b/docs/local-test-plan.md @@ -59,10 +59,54 @@ on a Zen 3 the plain `x64` baseline module is **10.4x slower at prompt processin | Machine | Expected module | Why | Measured? | |---|---|---|---| -| Ryzen 7 5800H (Zen 3) | `haswell` | AVX2, no AVX-512 | **yes** -- confirmed from the JVM's mapped modules, see the b11538 report | +| Ryzen 7 5800H (Zen 3) | `haswell` | AVX2, no AVX-512 | **yes** -- from the JVM's mapped modules, b11538 report | +| Xeon Cascade Lake | `cascadelake` | AVX-512 + VNNI, no VBMI/BF16 | **yes** | +| Xeon Platinum 8370C (Ice Lake-SP) | `icelake` | AVX-512 + VBMI + VNNI | **yes**, on a CI runner | +| **Xeon 697xP-C (Granite Rapids)** | **`sapphirerapids`** | the only level asking for `AMX_TILE`/`AMX_INT8` | **yes** -- and it found the bug `patches/0018` fixes | | **Ryzen AI Max+ 395** (Zen 5) | **`zen4`** | AVX-512 + VNNI + BF16; `sapphirerapids` would need AMX, which Zen 5 lacks | **open -- needs the device** | | **Core Ultra 7 165U** (Meteor Lake) | **`alderlake`** | AVX2 + AVX-VNNI, no AVX-512 | **open -- needs the device** | +**x86-64 module coverage is complete for *selection*.** On a Granite Rapids machine every one of the +14 modules registers its backend when staged alone, because that CPU carries every feature set below +AMX as well -- so "can this module be selected and used at all" is answered for all 14 in one run, +and the remaining per-family runs would only add *throughput* numbers. Four levels were additionally +measured with real inference there (`sapphirerapids`, `cooperlake`, `icelake`, `haswell`), on Windows +and on Linux. + +### aarch64: 6 of the 8 modules measured, and ggml does not pick the fastest + +Measured on an AWS Graviton 4 (Neoverse V2, `sve sve2 i8mm bf16`, no SME), the shipped +`cpu-linux-aarch64` library, Qwen3-0.6B Q4_K_M, 4 threads, each module staged alone: + +| module | needs (ggml's list) | exit | pp512 | tg128 | +|---|---|---:|---:|---:| +| `armv9.2_2` | DOTPROD FP16 SVE MATMUL_INT8 SVE2 **SME** | 1 | -- | -- | +| `armv9.2_1` | DOTPROD FP16 SVE MATMUL_INT8 **SME** | 1 | -- | -- | +| **`armv8.6_2`** *(ggml's choice)* | DOTPROD FP16 SVE MATMUL_INT8 SVE2 | 0 | 198.0 | 63.6 | +| `armv8.6_1` | DOTPROD FP16 SVE MATMUL_INT8 | 0 | 197.7 | 60.8 | +| `armv8.2_3` | DOTPROD FP16 SVE | 0 | 176.7 | 58.2 | +| **`armv8.2_2`** | DOTPROD FP16 | 0 | **271.4** | 61.6 | +| `armv8.2_1` | DOTPROD | 0 | 270.7 | 54.0 | +| `armv8.0_1` | -- | 0 | 116.6 | 43.3 | + +The two `exit 1` rows are **correct behaviour, not a crash**: the CPU has no SME, the module +registers no CPU backend, and the load fails cleanly with +`make_cpu_buft_list: no CPU backend found` and a `LlamaException`. Verify that wording before +reading a 1 as a defect. + +**The finding worth following up: ggml's choice is 37 % slower at prompt processing than a lower +level.** It selects `armv8.6_2` (198 t/s) while `armv8.2_2` reaches 271 t/s on the same machine and +model -- i.e. the SVE/SVE2/i8mm paths lose to plain DOTPROD+FP16 here. Generation is within noise +(63.6 against 61.6). That is a scoring question for upstream, not a correctness one, and it is +invisible to anyone who only runs the module the CPU picks. + +**`SME` has no AWS instance**, so `armv9.2_1` and `armv9.2_2` stay unmeasured; Apple M4 and some +newer phone SoCs have it. + +**What the Granite Rapids run is worth remembering for**, beyond the coverage: staging one module at +a time on a *single* instance, same model each run, is what turned an intermittent CI crash into a +one-line defect. The module was the only variable left. + Steps: 1. Build the variant library (Windows needs plain clang; `clang-cl` and `cl.exe` are refused with a diff --git a/docs/rented-hardware-test-plan.md b/docs/rented-hardware-test-plan.md new file mode 100644 index 000000000..d8226ac3a --- /dev/null +++ b/docs/rented-hardware-test-plan.md @@ -0,0 +1,163 @@ + + +# Test plan for rented hardware + +A work queue for the questions this project cannot answer on CI or on a developer machine, to be +worked through one rented instance at a time. The companion document for a local machine is +[`local-test-plan.md`](local-test-plan.md); the open items themselves live in +[`../TODO.md`](../TODO.md), and this file is only the *how and in which order*. + +**The account-specific mechanics are deliberately not here** — how an instance is launched, how a +result comes back, which bucket and which credentials. Those are private notes, because they are +account data rather than project knowledge. + +## Why rented hardware answers anything at all + +CI already runs the whole suite on every platform, so renting a machine is only worth it where CI is +structurally blind: + +| CI cannot | because | +|---|---| +| choose a CPU | GitHub mixes runner hardware; which feature set a run draws is chance. **This is not theoretical:** the AMX heap corruption (`patches/0018`) showed up as `Tests run: 0` on the runs that happened to land on Sapphire Rapids or newer, and a green Windows job proves nothing unless its log names an AMX CPU | +| use a GPU | every hosted runner is GPU-less, so the GPU module jars are only ever *placed* and skipped, never loaded | +| hold a large model | a 78B GGUF needs more RAM than a runner has, and more disk than the cache allows | +| run for an hour | the throughput questions need sustained load, not a build step | + +A developer machine answers the first and last but has exactly one CPU and one GPU, which is the +whole problem: `ggml` has 14 x86 and 8 aarch64 CPU modules, and a machine proves one of them. + +## The invariants, learned the expensive way + +These are not style preferences; each one comes from a run that went wrong. + +1. **Always terminate, never stop.** A terminated instance costs nothing afterwards; a stopped one + keeps paying for its disk. Nothing here needs to survive between runs. +2. **A dead-man switch is the first line of every script** (`shutdown -h +`, with the + instance set to terminate on shutdown). Without it, a script that hangs runs until something + external kills it — and nothing does when the watching process on the developer machine dies. + With it, the worst case is bounded by one number you choose in advance. +3. **Many checks per boot.** Booting, installing a JDK and fetching a model is 10–15 minutes; the + measurement is often seconds. Stage every variant in one script and vary only the thing under + test. +4. **One variable at a time, on the same machine.** What made the AMX finding conclusive was staging + one CPU module at a time with the same model on the same instance: the module was the only + difference left. A comparison across two instances proves much less. +5. **Stream the result out after every step**, to a destination with a name unique to that run. A + run-specific name is not pedantry — reusing a result file twice made a watcher read the *previous* + run's finished log, declare success and terminate a running instance mid-measurement. +6. **Read the error lines, not the exit code.** A probe whose every run exits non-zero looks like + "the thing under test is broken" and is usually the harness: a mistyped path, an unexpanded + placeholder, a missing file. Print the output. +7. **Fetch public models on the instance, not through your own connection.** A rented machine + downloads from Hugging Face at datacentre bandwidth. Only stage something yourself when it is not + publicly available. + +## The queue + +Ordered by what each one unblocks, not by cost. Every row ends with *what counts as an answer*, so a +run cannot end in "looked fine". + +### 1. The GPU module jars on a real GPU + +**The question.** Modular natives (5.2.0) made each GPU backend its own module jar, loaded next to +the platform's CPU library. CI places them on a GPU-less runner, where `ggml` skips them — so the +*successful* path has never run. Does a `cuda13` or `vulkan` module jar actually register its device, +offload layers and compute, next to the CPU library it was built beside? + +**How.** The cheapest NVIDIA instance class, a small CPU-only-capable model, a driver-equipped image +so no toolkit install is needed. Five runs in one boot: automatic backend selection, the GPU backend +forced with full offload, the same forced with zero offload, a second GPU backend forced, and the +CPU library alone as the baseline. Log verbosity high enough that the device enumeration and the +`offloaded N/N layers` line appear. + +**An answer looks like:** the loader line names the module, `ggml` reports the device, the offload +line shows all layers, and generation is materially faster than the CPU baseline. A module that +silently falls back to the CPU is a *finding*, not a pass. + +**Also re-measures:** whether the CUDA module still carries `GGML_CUDA_GRAPHS` — a build that lost +that default once cost two thirds of its token generation, and only a GPU can see it. + +### 2. Kolibri-1 against the real 78B model + +**The question.** `patches/0016` adds an architecture upstream does not support, and it is verified +only against tiny synthesized GGUFs. Two community conversions exist whose GGUFs are mutually +incompatible, and the patch claims to load **both**. + +**How.** One high-RAM instance, both dialects in a single boot, each GGUF fetched from its repository +on the instance. Take a middle quant rather than the smallest: the question is whether the German +answer is *coherent*, and a heavily quantized 78B MoE would make that ambiguous. Print the +architecture, expert-count and gating lines from the load, then the generated **text** of a raw +completion and of a chat turn through the model's own template. + +**An answer looks like:** both dialects load, and the German answer is coherent. Fluent-looking +nonsense is the specific failure to watch for — the router selects on `logits + expert_bias` but +weights by the *unbiased* sigmoid, and getting that wrong produces text that reads well and is wrong. + +### 3. The throughput gain of the top CPU modules + +**The question.** The CPU-variant work proved the *lower* bound (the baseline `x64` module is ~10× +slower at prompt processing than `haswell`). The gain upwards — `zen4` with BF16, `sapphirerapids` +with AMX — is unmeasured, which is why the default build ships every level. + +**How.** One instance per microarchitecture (Granite Rapids, EPYC Genoa, and an older generation as +the `haswell` reference), identical model, identical thread count, each staging its own module plus +`haswell` for a same-machine comparison. + +**An answer looks like:** a table of pp512 and tg128 per module per machine. **Beware the result +already in hand:** on Graviton 4, `ggml`'s own choice was 37 % *slower* at prompt processing than a +module it rejected, so measure every module the CPU accepts, not only the selected one. + +### 4. The OpenVINO module, with no GPU at all + +**The question.** `openvino-linux-x86-64` is build-only; nothing has ever loaded it. OpenVINO has a +CPU plugin, so this needs no GPU and no quota — the cheapest open item in the matrix. + +**An answer looks like:** the module registers a device and computes, or it fails with a reason worth +recording. + +### 5. Extraction reuse on Windows + +**The question.** The extraction directory is keyed by build and module set and reused across JVM +starts. A second start of the same build must copy nothing, a running JVM's locked libraries must +survive another build's cleanup, and a start must not block on them. This is Windows-specific +(file locking) and needs two JVMs. + +**An answer looks like:** no extraction lines on the second start, and both JVMs healthy. + +### 6. The full Java suite on a rented platform + +Worth doing once the staging is in place anyway: the whole model-backed suite against a shipped +natives artifact, on a platform or CPU feature set CI cannot pin. Lower priority than the above +because CI already runs this suite on every platform — what rented hardware adds is *which* CPU. + +## What rented hardware cannot answer + +Recording the limits so they are not re-derived per attempt: + +| open item | why not | +|---|---| +| `armv9.2_1` / `armv9.2_2` (the two **SME** aarch64 modules) | no cloud instance has SME. Apple M4 does, but macOS ships one static library rather than the aarch64 module set, so those modules cannot even be loaded there | +| **ROCm** at runtime | the only AMD-GPU instance class has been withdrawn, and ROCm never officially supported that card anyway. Needs a real Radeon machine | +| **SYCL** on a GPU | no discrete Intel GPU is offered. Intel's CPU OpenCL runtime covers part of the path | +| **Metal** | macOS instances bill a 24-hour minimum, and Metal is compiled into the library there, so CI's macOS jobs already cover it | +| a real **arm64 Android device** | the AAR is exercised on an x86-64 emulator in CI; the Adreno/OpenCL flavor needs physical hardware | +| the **interactive agent** | the terminal agent's console behaviour — line editing, the status line, interrupting a stream — needs a human at a real terminal. This is the one item that stays with a developer machine | + +## Cost discipline + +The honest framing: these runs cost **cents to low single-digit euros each**, and the risk is never +the hourly rate but a forgotten resource. Three habits cover it: + +* the dead-man switch above, so an instance cannot outlive its purpose; +* a **sweep over every region** after a run — not just the obvious one, and not only for instances: a + disk can **survive a terminate** and keeps costing money with no instance attached, which an + instance query does not show. Snapshots and reserved addresses are the same class; +* a **budget alert** on the account, which is the only guard that reaches a human whether or not + anything is watching. + +A refused launch — a quota not yet raised, no capacity for that instance type — costs nothing, so a +retry loop may poll for hours without consequence. diff --git a/llama/CMakeLists.txt b/llama/CMakeLists.txt index 92c36e717..5585d8778 100644 --- a/llama/CMakeLists.txt +++ b/llama/CMakeLists.txt @@ -298,6 +298,13 @@ FetchContent_Declare( ${JLLAMA_LLAMA_SOURCE_SUBDIR} ) FetchContent_MakeAvailable(llama.cpp) + +# Every ggml source allocating with ggml_aligned_malloc must also release with ggml_aligned_free. +# The AMX buffer type did not, which corrupted the Windows heap on AMX-capable CPUs and killed the +# process at cleanup; patches/0018 fixes it. Checked here on every platform in every build, +# because no machine without AMX can notice the defect at run time. +include(cmake/check-aligned-free-pairing.cmake) +jllama_check_aligned_free_pairing(${llama.cpp_SOURCE_DIR}) if(JLLAMA_BACKEND_DL) # Where ggml can be loaded as modules here (CLAUDE.md "Modular natives"): Linux x86-64 / aarch64 # (the dockcross Android toolchain reports Linux too, see the Android block above) and Windows diff --git a/llama/cmake/check-aligned-free-pairing.cmake b/llama/cmake/check-aligned-free-pairing.cmake new file mode 100644 index 000000000..96ce141f0 --- /dev/null +++ b/llama/cmake/check-aligned-free-pairing.cmake @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Every ggml source that allocates with ggml_aligned_malloc must also release with +# ggml_aligned_free -- checked at configure time, on every platform, in every build. +# +# Why this is worth a check of its own. On Windows ggml_aligned_malloc is `_aligned_malloc` and +# ggml_aligned_free is `_aligned_free` (ggml.c), and passing an `_aligned_malloc` pointer to plain +# `free()` is undefined behaviour that corrupts the process heap. The AMX buffer type did exactly +# that -- allocating with ggml_aligned_malloc at amx.cpp and releasing with `free(buffer->context)` +# -- so every model load that used an AMX-capable CPU killed the process at cleanup with +# 0xC0000374 STATUS_HEAP_CORRUPTION, after the inference had already succeeded. Measured on a rented +# Granite Rapids machine: six runs with the `sapphirerapids` module crashed (including at one +# thread, which rules out a data race), while `cooperlake`, `icelake` and `haswell` on the same +# machine and model were clean. In CI it showed up only as `Tests run: 0` on whichever runs happened +# to land on AMX hardware. +# +# The pairing is what makes it checkable without that hardware: five callers in the tree get it +# right and one got it wrong, and the wrong one is recognisable from the source alone -- a file that +# calls ggml_aligned_malloc and never calls ggml_aligned_free. That is a coarse rule (it cannot see +# *which* pointer goes where), but it is exact for the shape that occurred, costs milliseconds, and +# fails the configure on any platform rather than only on the machines where the bug is fatal. +# +# A failure here means one of three things: the local patch that fixes it was dropped, upstream +# reintroduced the shape somewhere else, or upstream fixed it in a way that no longer calls +# ggml_aligned_free in that file. Read the file before changing this check. + +function(jllama_check_aligned_free_pairing root) + file(GLOB_RECURSE sources + "${root}/ggml/src/*.c" + "${root}/ggml/src/*.cpp" + "${root}/ggml/src/*.cu" + "${root}/ggml/src/*.m" + "${root}/ggml/src/*.mm") + + set(offenders "") + set(checked 0) + foreach(source IN LISTS sources) + file(READ "${source}" text) + # ggml.c defines both, so it names them for reasons other than calling them. + string(FIND "${text}" "ggml_aligned_malloc" has_malloc) + if(has_malloc EQUAL -1) + continue() + endif() + math(EXPR checked "${checked} + 1") + string(FIND "${text}" "ggml_aligned_free" has_free) + if(has_free EQUAL -1) + file(RELATIVE_PATH shown "${root}" "${source}") + list(APPEND offenders "${shown}") + endif() + endforeach() + + if(offenders) + string(REPLACE ";" "\n - " shown "${offenders}") + message(FATAL_ERROR + "ggml_aligned_malloc without ggml_aligned_free in:\n - ${shown}\n" + "On Windows ggml_aligned_malloc is _aligned_malloc, whose memory MUST be released with " + "_aligned_free (ggml_aligned_free); plain free() corrupts the process heap and kills it " + "at cleanup with STATUS_HEAP_CORRUPTION, after the work already succeeded. This is the " + "defect llama/patches/0018 fixes -- check whether that patch still applies, or whether " + "upstream reintroduced the shape elsewhere. See llama/cmake/check-aligned-free-pairing.cmake.") + endif() + + message(STATUS "aligned-free pairing: ${checked} ggml source(s) use ggml_aligned_malloc, all paired") +endfunction() diff --git a/llama/patches/0018-ggml-amx-free-the-aligned-buffer.patch b/llama/patches/0018-ggml-amx-free-the-aligned-buffer.patch index 52474a75d..1aa963c3e 100644 --- a/llama/patches/0018-ggml-amx-free-the-aligned-buffer.patch +++ b/llama/patches/0018-ggml-amx-free-the-aligned-buffer.patch @@ -20,6 +20,29 @@ model's close(); the same job was green an hour earlier (run 38065017595) on an runner, which has no AVX-512 and loads the haswell module. Upstream master still had the free() on 2026-10-10. Not yet reported upstream (nothing is filed before this project's release). +Every other caller of ggml_aligned_malloc() in the tree pairs it correctly -- ggml-backend.cpp, +ggml-cpu.c (twice), ggml-openvino.cpp, ggml-zdnn.cpp and ggml.c itself -- so this was the one +mismatched pair, and ggml-backend.cpp:2464 is the line this fix copies. + +Reproduced deliberately afterwards, rather than waiting for CI to draw AMX hardware again: on a +rented Granite Rapids machine, one CPU backend module staged at a time, same model, same binaries: + + module runs exit + sapphirerapids 6 -1073740940 (0xC0000374) <- the only module with the AMX path + cooperlake 2 0 + icelake 2 0 + haswell 2 0 + +The same six runs on Linux with the same CPU all exit 0 with byte-identical output. It reproduces at +one thread, so it is not a data race, and the crash lands after the inference has printed its +timings -- the signature of a heap corrupted earlier and detected at free. With this patch applied, +the identical matrix on the same machine gives exit 0 in all twelve runs. + +Guard on the downstream side: llama/cmake/check-aligned-free-pairing.cmake fails the configure on +any platform when a ggml source calls ggml_aligned_malloc() and never ggml_aligned_free(). A +run-time test cannot replace it -- without an AMX CPU there is nothing to observe, which is exactly +how this shipped. + Drop this patch when upstream frees the AMX buffer with ggml_aligned_free() or _aligned_free(): grep "free(buffer->context)" ggml/src/ggml-cpu/amx/amx.cpp at the new tag. diff --git a/llama/pom.xml b/llama/pom.xml index 6e111b48b..a8a1291fd 100644 --- a/llama/pom.xml +++ b/llama/pom.xml @@ -62,7 +62,7 @@ SPDX-License-Identifier: MIT Both run on the build JDK and neither ships (checker-qual is provided scope), so this tracks the newest release. --> 4.3.0 - 2.22.3 + 3.8.7 2.0.20 1.6.5 diff --git a/llama/src/test/java/net/ladenthin/llama/loader/CpuModuleIsolationTest.java b/llama/src/test/java/net/ladenthin/llama/loader/CpuModuleIsolationTest.java new file mode 100644 index 000000000..3c3617ea5 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/loader/CpuModuleIsolationTest.java @@ -0,0 +1,306 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: Apache-2.0 + +package net.ladenthin.llama.loader; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.TimeUnit; +import java.util.jar.Attributes.Name; +import java.util.jar.JarFile; +import java.util.jar.Manifest; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import net.ladenthin.llama.ClaudeGenerated; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Loads every shipped {@code ggml-cpu-} module on its own, each in a forked JVM, and + * requires every one of them to exit cleanly. + * + *

Why this cannot be a normal in-process test, and why it is worth its runtime: a modular build + * (see CLAUDE.md "Modular natives") ships up to 14 CPU modules, of which {@code + * ggml_backend_load_best} keeps exactly one -- the best for the running CPU. The other 13 + * are loaded, scored and unloaded again, so on any single machine only one module is ever the + * chosen backend. A module that corrupts the process therefore stays invisible until a + * machine turns up whose CPU prefers it, and it then takes down the JVM before the first test + * reports: a Granite Rapids runner (AVX-512 + AMX) died with Windows + * {@code 0xC0000374 STATUS_HEAP_CORRUPTION} and {@code Tests run: 0}, while the identical build and + * jars passed 1915 tests on a Zen 3 machine, which declines the AMX module. No amount of running the + * suite on one CPU can find that. + * + *

What this test does not cover, measured afterwards on a rented Granite Rapids machine: + * loading is not where that crash is. There the full module set loads cleanly, ggml picks the AMX + * module, and all 14 modules load one by one as well -- so the crash is in the first test's real + * work (a model load and inference), after {@code System.load} has returned. This test still earns + * its runtime, because a module that cannot even be loaded is a different and cheaper failure to + * find, and because its per-module table records which levels a given CPU accepts at all. + * + *

So each module gets a directory of its own holding the library, {@code ggml}, {@code ggml-base}, + * {@code ggml-rpc} and that single module, and a JVM of its own -- which also exercises the + * one-module layout a platform without a variant list ships (Windows arm64). A module whose + * instruction set the CPU lacks must decline, i.e. register no CPU backend, and must not + * crash: loading a DLL executes none of its kernels, ggml's score function only reads CPUID. + * + *

The crash is the signal, not the exit message: a corrupted heap kills the process before the + * JVM can write {@code hs_err}, so this test asserts the exit code and quotes whatever the child + * printed. + */ +@ClaudeGenerated( + purpose = "Load each ggml-cpu module in isolation in its own JVM, so a module that corrupts the " + + "process is found on every CPU rather than only on the one whose CPU selects it -- the " + + "shape of the Granite Rapids STATUS_HEAP_CORRUPTION that reported Tests run: 0.") +public class CpuModuleIsolationTest { + + /** How long a child JVM may take to load the library and print its line. */ + private static final long CHILD_TIMEOUT_SECONDS = 180; + + private static final String MODULE_PREFIX = "ggml-cpu-"; + + /** Separator of the manifest {@code Class-Path} entries. */ + private static final java.util.regex.Pattern WHITESPACE = java.util.regex.Pattern.compile("[ \\t\\r\\n]+"); + + /** Files a module directory needs besides the module itself; absent ones are skipped. */ + private static final List SUPPORT = List.of( + System.mapLibraryName("jllama"), + libraryName("ggml"), + libraryName("ggml-base"), + libraryName("ggml-rpc"), + "jllama-build.txt"); + + private static String libraryName(String stem) { + return System.mapLibraryName(stem); + } + + @Test + void everyCpuModuleLoadsOnItsOwn(@TempDir Path tempDir) throws Exception { + assumeTrue(NativeLibraryPresence.onClasspath(), "no jllama library on the classpath"); + Path backendDir = backendDirectoryOnDisk(); + assumeTrue(backendDir != null, "the library is not an extracted directory on disk"); + List modules = modulesIn(backendDir); + assumeTrue(!modules.isEmpty(), "a static build ships no ggml-cpu modules: " + backendDir); + + List crashed = new ArrayList<>(); + List report = new ArrayList<>(); + for (Path module : modules) { + Path root = stageSingleModule(tempDir, backendDir, module); + Result result = runChild(root); + String line = module.getFileName() + ": exit " + result.exitCode + + (result.chosenModule == null ? ", no CPU backend registered" : ", chose " + result.chosenModule); + report.add(line); + if (result.exitCode != 0) { + crashed.add(line + "\n" + result.output); + } + } + + // Printed on success too: which modules this CPU accepts is the diagnosis a green run on an + // unfamiliar runner should still leave behind. GitHub mixes runner hardware -- an Ice Lake + // runner (Xeon Platinum 8370C) picks icelake and passes, a Granite Rapids one (Xeon 6973P-C, + // the only AMX machine seen so far) picks sapphirerapids, so without this table a green log + // does not say which path was actually exercised. + System.out.println("CpuModuleIsolationTest: " + modules.size() + " module(s) in " + backendDir); + report.forEach(line -> System.out.println(" " + line)); + + assertTrue( + crashed.isEmpty(), + () -> "these CPU modules did not load cleanly on their own:\n" + String.join("\n", crashed) + + "\n\nall modules:\n" + String.join("\n", report)); + } + + /** + * This JVM's real classpath entries. + * + *

Surefire starts the fork with a manifest-only booter jar, so {@code java.class.path} + * is that single jar and the real entries sit in its manifest {@code Class-Path} as {@code file:} + * URLs. Splitting the property alone therefore hands the child a classpath that still carries the + * real natives directory: the staged per-module directory is merely prepended to the full module + * set, every row reports the module the CPU would have picked anyway, and the test proves nothing. + * It did exactly that until this was found -- 14 rows, all "chose haswell". + * + * @param separator the platform path separator + * @return the entries, expanded out of the booter jar when there is one + */ + private static List effectiveClasspath(String separator) { + String property = System.getProperty("java.class.path", ""); + List entries = new ArrayList<>(List.of(property.split(java.util.regex.Pattern.quote(separator)))); + if (entries.size() != 1 || !entries.get(0).endsWith(".jar")) { + return entries; + } + Path booter = Paths.get(entries.get(0)); + try (JarFile jar = new JarFile(booter.toFile())) { + Manifest manifest = jar.getManifest(); + String classPath = + manifest == null ? null : manifest.getMainAttributes().getValue(Name.CLASS_PATH); + if (classPath == null || classPath.isBlank()) { + return entries; + } + List expanded = new ArrayList<>(); + for (String token : WHITESPACE.split(classPath.trim())) { + try { + expanded.add(Paths.get(new URI(token)).toString()); + } catch (Exception e) { + expanded.add( + booter.getParent() == null + ? token + : booter.getParent().resolve(token).toString()); + } + } + return expanded; + } catch (IOException e) { + return entries; + } + } + + /** The extracted/available backend directory holding the library for this platform, or null. */ + private static Path backendDirectoryOnDisk() { + ClassLoader loader = CpuModuleIsolationTest.class.getClassLoader(); + String libraryFile = System.mapLibraryName("jllama"); + for (String backend : LlamaLoader.LIBRARY_BACKENDS) { + String resource = "net/ladenthin/llama/" + OSInfo.getNativeLibFolderPathForCurrentOS() + "/" + backend + "/" + + libraryFile; + URL url = loader.getResource(resource); + if (url != null && "file".equals(url.getProtocol())) { + try { + return Paths.get(url.toURI()).getParent(); + } catch (Exception e) { + return null; + } + } + } + return null; + } + + private static List modulesIn(Path backendDir) throws IOException { + try (Stream files = Files.list(backendDir)) { + return files.filter(p -> p.getFileName().toString().startsWith(MODULE_PREFIX)) + .sorted() + .collect(Collectors.toList()); + } + } + + /** A natives root holding only {@code module} beside the library, with matching file lists. */ + private static Path stageSingleModule(Path tempDir, Path backendDir, Path module) throws IOException { + String moduleName = module.getFileName().toString(); + Path root = tempDir.resolve(moduleName.replace('.', '_')); + Path target = root.resolve( + "net/ladenthin/llama/" + OSInfo.getNativeLibFolderPathForCurrentOS() + "/" + backendDir.getFileName()); + Files.createDirectories(target); + for (String name : SUPPORT) { + Path source = backendDir.resolve(name); + if (Files.exists(source)) { + Files.copy(source, target.resolve(name)); + } + } + Files.copy(module, target.resolve(moduleName)); + + List extracted = new ArrayList<>(); + extracted.add("# CpuModuleIsolationTest: this module only"); + extracted.add(moduleName); + if (Files.exists(target.resolve(libraryName("ggml-rpc")))) { + extracted.add(libraryName("ggml-rpc")); + } + Files.write(target.resolve("jllama-files.txt"), extracted, StandardCharsets.UTF_8); + + List preloaded = new ArrayList<>(); + for (String stem : List.of("ggml-base", "ggml")) { + if (Files.exists(target.resolve(libraryName(stem)))) { + preloaded.add(libraryName(stem)); + } + } + if (!preloaded.isEmpty()) { + Files.write(target.resolve("jllama-extras.txt"), preloaded, StandardCharsets.UTF_8); + } + return root; + } + + /** Forks a JVM whose classpath carries {@code root} instead of the real natives directory. */ + private static Result runChild(Path root) throws IOException, InterruptedException { + String separator = System.getProperty("path.separator"); + Path backendDir = backendDirectoryOnDisk(); + String real = backendDir == null + ? null + : backendDir.toAbsolutePath().toString().toLowerCase(Locale.ROOT); + List entries = new ArrayList<>(); + entries.add(root.toAbsolutePath().toString()); + for (String entry : effectiveClasspath(separator)) { + String candidate = Paths.get(entry).toAbsolutePath().toString().toLowerCase(Locale.ROOT); + if (real == null || !real.startsWith(candidate)) { + entries.add(entry); + } + } + // Each child needs a temp directory of its own. The extraction directory is keyed by the + // LIBRARY (plus any GPU module jars), never by the CPU modules lying beside it, so every + // staged set shares one name -- the first child extracts its single module, and every later + // one finds the directory "in place" and loads whatever is already there. That silently + // defeated the isolation this test exists for: all 14 rows reported the module the CPU would + // have picked anyway (measured: 14x "chose haswell" from one directory holding 14 modules). + Path childTemp = root.resolveSibling(root.getFileName() + "-tmp"); + Files.createDirectories(childTemp); + List command = List.of( + Paths.get(System.getProperty("java.home"), "bin", "java").toString(), + "-D" + LlamaSystemProperties.PREFIX + ".tmpdir=" + childTemp.toAbsolutePath(), + "-cp", + String.join(separator, entries), + Probe.class.getName()); + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output; + try (var in = process.getInputStream()) { + output = new String(in.readAllBytes(), StandardCharsets.UTF_8); + } + if (!process.waitFor(CHILD_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + process.destroyForcibly(); + return new Result(-1, output + "\n(timed out after " + CHILD_TIMEOUT_SECONDS + " s)", null); + } + String chosen = null; + for (String line : output.split("\\R")) { + // The line names the module by its full path, and the per-child temp directory itself + // starts with the module prefix -- take the LAST occurrence, i.e. the file name. + int index = line.lastIndexOf(MODULE_PREFIX); + if (line.contains("loaded CPU backend") && index >= 0) { + chosen = line.substring(index).trim(); + } + } + return new Result(process.exitValue(), output, chosen); + } + + private static final class Result { + private final int exitCode; + private final String output; + private final String chosenModule; + + private Result(int exitCode, String output, String chosenModule) { + this.exitCode = exitCode; + this.output = output; + this.chosenModule = chosenModule; + } + } + + /** Child entry point: forces the library load, i.e. {@code JNI_OnLoad} and ggml's module scan. */ + public static final class Probe { + private Probe() {} + + /** + * Loads the native library and exits. + * + * @param args ignored + * @throws Exception when the library cannot be loaded at all + */ + public static void main(String[] args) throws Exception { + Class.forName("net.ladenthin.llama.LlamaModel"); + System.out.println("PROBE_OK"); + } + } +} diff --git a/pom.xml b/pom.xml index 871337867..04f0e24f0 100644 --- a/pom.xml +++ b/pom.xml @@ -54,6 +54,29 @@ SPDX-License-Identifier: MIT + + + 2.22.3 + + + + + + com.fasterxml.jackson + jackson-bom + ${jackson.version} + pom + import + + + + llama llama-langchain4j