From 7762780272382fb70b6e4669e1502c5b0bed5575 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:14:00 -0300 Subject: [PATCH 1/3] Fix ROCm CPU performance, update the ROCm toolchain, and bundle Windows ARM64 CUDA libraries --- .../unsloth-prebuilt-cuda-windows-arm64.yml | 40 ++++----- .github/workflows/unsloth-prebuilt-rocm.yml | 85 ++++++++++++++++--- scripts/unsloth/package_bundle.py | 12 ++- 3 files changed, 100 insertions(+), 37 deletions(-) diff --git a/.github/workflows/unsloth-prebuilt-cuda-windows-arm64.yml b/.github/workflows/unsloth-prebuilt-cuda-windows-arm64.yml index 9ea06ccce0bb..af2e2af2158c 100644 --- a/.github/workflows/unsloth-prebuilt-cuda-windows-arm64.yml +++ b/.github/workflows/unsloth-prebuilt-cuda-windows-arm64.yml @@ -14,9 +14,8 @@ name: "Unsloth prebuilt: CUDA Windows ARM64" # compiles on an x64 runner with the amd64_arm64 MSVC toolset, the same way # unsloth-prebuilt-cpu.yml already cross compiles the Windows arm64 CPU # bundle, and the same way ggml-org's release.yml builds its arm64 CUDA zip. -# * Only CUDA 13.4 ships Windows ARM64 target libraries (cudart + cublas), and -# only from packages.nvidia.com rather than the redist CDN the x64 legs use, -# so it needs its own component list, not another row in the x64 table. +# * CUDA 13.4 needs a separate component list: x86_64 host tools and ARM64 +# target libraries. # * It builds one target (ggml-cuda), not the whole tool set: everything else # in the bundle comes from the arm64 CPU bundle this run already builds. # That is exactly upstream's arrangement -- their arm64 CUDA job packs @@ -148,23 +147,19 @@ jobs: with: arch: amd64_arm64 - # The x64 legs install from the redist CDN, which has no ARM64 target - # libraries at any version. 13.4 is the first toolkit that ships them and - # it lives in the bin-archive pool instead; the host-side pieces (nvcc, - # crt, nvvm, cccl) stay x86_64 because the compiler runs on this runner. - # Same component set and same pool ggml-org's release.yml installs. + # CUDA 13.4.1 GA: x86_64 compiler tools and ARM64 runtime libraries, + # matching ggml-org/llama.cpp#28687. - name: Install CUDA toolkit 13.4 (x86_64 host, ARM64 target) run: | $ErrorActionPreference = 'Stop' - $pool = 'https://packages.nvidia.com/bin-archive/pool' - $guid = '5B515474-7E78-11F1-8656-C51E4F4B317F' + $pool = 'https://developer.download.nvidia.com/compute/cuda/redist' $components = @( - "windows-x86_64/$guid/cccl-windows-x86_64-13.3.4.1.2-archive", - "windows-x86_64/$guid/cuda_crt-windows-x86_64-13.4.46-archive", - "windows-x86_64/$guid/cuda_nvcc-windows-x86_64-13.4.46-archive", - "windows-x86_64/$guid/libnvvm-windows-x86_64-13.4.46-archive", - "windows-arm64/$guid/cuda_cudart-windows-arm64-13.4.46-archive", - "windows-arm64/$guid/libcublas-windows-arm64-13.7.0.10-archive" + "cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive", + "cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive", + "cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive", + "libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive", + "cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.4.49-archive", + "libcublas/windows-arm64/libcublas-windows-arm64-13.7.0.27-archive" ) $prefix = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ env.CUDA }}" $stage = Join-Path $env:RUNNER_TEMP 'cuda-redist' @@ -353,15 +348,19 @@ jobs: name: app-${{ inputs.tag }}-windows-arm64-cpu path: cpubundle - # The CPU zip is the whole build/bin/Release tree plus the OpenMP runtime. - # Dropping the backend in beside it gives package_bundle.py the same shape - # of input directory a native build tree would have. + # Add the CUDA backend to the CPU bundle, which includes OpenMP. + # Bundle the CUDA runtime because PyTorch has no Windows ARM64 CUDA wheels. - name: Compose the bundle input run: | $zip = "cpubundle/app-${{ inputs.tag }}-windows-arm64-cpu.zip" if (-not (Test-Path $zip)) { throw "the CPU leg's artifact does not contain $zip" } Expand-Archive -Path $zip -DestinationPath bin -Force Copy-Item cudabin/ggml-cuda.dll bin/ -Force + foreach ($dll in @('cudart64_13.dll', 'cublas64_13.dll', 'cublasLt64_13.dll')) { + $src = Join-Path "$env:CUDA_PATH\bin\arm64" $dll + if (-not (Test-Path $src)) { throw "the toolkit has no $src" } + Copy-Item $src bin/ -Force + } foreach ($required in @('llama-server.exe', 'ggml-base.dll', 'ggml-cpu.dll')) { if (-not (Test-Path (Join-Path 'bin' $required))) { throw "the CPU bundle is missing $required" } } @@ -383,7 +382,8 @@ jobs: KLASS: ${{ env.KLASS }} RANK: ${{ env.RANK }} TOOLKIT_LINE: ${{ env.CUDA }} - DOCKER_IMAGE: github-hosted windows-2022 cross-compiled to arm64, CUDA ${{ env.CUDA }} (NVIDIA bin-archive) + DOCKER_IMAGE: github-hosted windows-2022 cross-compiled to arm64, CUDA ${{ env.CUDA }} (NVIDIA redist) + BUNDLE_CUDA_RUNTIME: '1' ARCHS: ${{ env.ARCHS }} run: python tooling/scripts/unsloth/package_bundle.py diff --git a/.github/workflows/unsloth-prebuilt-rocm.yml b/.github/workflows/unsloth-prebuilt-rocm.yml index ae8870ad54e0..25f0f4ec07c0 100644 --- a/.github/workflows/unsloth-prebuilt-rocm.yml +++ b/.github/workflows/unsloth-prebuilt-rocm.yml @@ -232,9 +232,8 @@ jobs: Write-Host "Using target with -all suffix: $archiveTarget" } - # TheRock publishes nightlies to the multi-arch tarball index. The - # static HTML page embeds a JSON `files` array with names and mtimes. - $baseUrl = "https://rocm.nightlies.amd.com/tarball-multi-arch" + # The index embeds a JSON `files` array with names and mtimes. + $baseUrl = "https://nightly.repo.amd.com/rocm/core/tarball" if ($rocmVersion -eq "latest" -or $rocmVersion -eq "weekly") { # weekly: see the Linux job. $cutoff = "99999999" @@ -409,11 +408,11 @@ jobs: REM Configure the project REM Only the C/CXX launchers here: ggml-hip forces CXX_IS_HIPCC on REM WIN32, so the device sources compile as C++, not the HIP language. + REM Keep native CPU detection; select CPU variants at runtime. cmake .. -G Ninja ^ -DCMAKE_C_COMPILER="C:\opt\rocm\lib\llvm\bin\clang.exe" ^ -DCMAKE_CXX_COMPILER="C:\opt\rocm\lib\llvm\bin\clang++.exe" ^ -DCMAKE_CXX_FLAGS="-IC:\opt\rocm\include" ^ - -DCMAKE_CROSSCOMPILING=ON ^ -DCMAKE_BUILD_TYPE=Release ^ -DGPU_TARGETS="%mapped_target%" ^ -DBUILD_SHARED_LIBS=ON ^ @@ -426,9 +425,10 @@ jobs: -DLLAMA_BUILD_BORINGSSL=ON ^ -DGGML_NATIVE=OFF ^ -DGGML_STATIC=OFF ^ + -DGGML_BACKEND_DL=ON ^ + -DGGML_CPU_ALL_VARIANTS=ON ^ -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }} ^ - -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }} ^ - -DCMAKE_SYSTEM_NAME=Windows + -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }} REM Build the project cmake --build . -j %NUMBER_OF_PROCESSORS% @@ -455,7 +455,8 @@ jobs: "hipblaslt.dll", "libhipblaslt.dll", "hipblas.dll", - "origami.dll" + "origami.dll", + "libtensilelite-host.dll" ) foreach ($pattern in $filesToCopy) { @@ -508,6 +509,33 @@ jobs: exit 1 } + # Catch runtime dependencies missing from the manual copy list. + - name: Check the bundle carries every ROCm DLL it imports + run: | + python -m pip install --quiet --disable-pip-version-check pefile + @' + import os, sys, pefile + bundle, rocm = r"llama.cpp\build\bin", r"C:\opt\rocm\bin" + have = {f.lower() for f in os.listdir(bundle)} + rocm_dlls = {f.lower() for f in os.listdir(rocm) if f.lower().endswith(".dll")} + dirs = [pefile.DIRECTORY_ENTRY[d] for d in ("IMAGE_DIRECTORY_ENTRY_IMPORT", "IMAGE_DIRECTORY_ENTRY_DELAY_IMPORT")] + missing = [] + for name in sorted(have): + if not name.endswith((".dll", ".exe")): + continue + pe = pefile.PE(os.path.join(bundle, name), fast_load=True) + pe.parse_data_directories(directories=dirs) + for entry in getattr(pe, "DIRECTORY_ENTRY_IMPORT", []) + getattr(pe, "DIRECTORY_ENTRY_DELAY_IMPORT", []): + dep = entry.dll.decode().lower() + if dep in rocm_dlls and dep not in have: + missing.append(f"{name} -> {dep}") + if missing: + print("\n".join(missing)) + print("::error::the bundle imports ROCm DLLs it does not carry") + sys.exit(1) + print("every imported ROCm DLL is in the bundle") + '@ | python - + - name: List build artifacts (including ROCm files) run: | cd llama.cpp\build\bin @@ -659,9 +687,8 @@ jobs: echo "Using target with -all suffix: $archive_target" fi - # TheRock publishes nightlies to the multi-arch tarball index. The - # static HTML page embeds a JSON `files` array with names and mtimes. - base_url="https://rocm.nightlies.amd.com/tarball-multi-arch" + # The index embeds a JSON `files` array with names and mtimes. + base_url="https://nightly.repo.amd.com/rocm/core/tarball" if [ "$rocm_version" = "latest" ] || [ "$rocm_version" = "weekly" ]; then # weekly: take the newest alpha up to the Saturday before the last SF Sunday, so a whole week uses one toolchain and the ccache hits. # The cutoff day must be settled before the first run reads it: TheRock usually publishes the evening before, but has landed as late as 18:33 PT on the named day, which splits the week. @@ -826,11 +853,12 @@ jobs: # CXX compiler is ROCm's clang++ (not hipcc), so ggml-hip leaves # CXX_IS_HIPCC false and calls enable_language(HIP), putting the device # sources on the HIP language rather than CXX. + # Setting CMAKE_SYSTEM_NAME without a processor forces generic CPU code. + # Keep native detection and select CPU variants at runtime. cmake .. -G Ninja \ -DCMAKE_C_COMPILER=/opt/rocm/llvm/bin/clang \ -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \ -DCMAKE_CXX_FLAGS="-I/opt/rocm/include" \ - -DCMAKE_CROSSCOMPILING=ON \ -DCMAKE_BUILD_TYPE=Release \ -DGPU_TARGETS="$mapped_target" \ -DBUILD_SHARED_LIBS=ON \ @@ -843,10 +871,11 @@ jobs: -DLLAMA_BUILD_BORINGSSL=ON \ -DGGML_NATIVE=OFF \ -DGGML_STATIC=OFF \ + -DGGML_BACKEND_DL=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }} \ -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }} \ - -DCMAKE_HIP_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }} \ - -DCMAKE_SYSTEM_NAME=Linux + -DCMAKE_HIP_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }} # Build the project cmake --build . -j $(nproc) @@ -907,11 +936,14 @@ jobs: cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_numa.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_numa.so* not found" cp -v /opt/rocm/lib/librocroller.so* "$build_bin_path/" 2>/dev/null || echo "librocroller.so* not found" cp -v /opt/rocm/lib/liborigami.so* "$build_bin_path/" 2>/dev/null || echo "liborigami.so* not found" + cp -v /opt/rocm/lib/libtensilelite-host.so* "$build_bin_path/" 2>/dev/null || echo "libtensilelite-host.so* not found" cp -v /opt/rocm/lib/librocm_kpack.so* "$build_bin_path/" 2>/dev/null || echo "librocm_kpack.so* not found" cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_z.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_z.so* not found" cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_zstd.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_zstd.so* not found" cp -v /opt/rocm/lib/llvm/lib/libLLVM.so* "$build_bin_path/" 2>/dev/null || echo "libLLVM.so* not found" cp -v /opt/rocm/lib/llvm/lib/libclang-cpp.so* "$build_bin_path/" 2>/dev/null || echo "libclang-cpp.so* not found" + # libtensilelite-host needs the unversioned libomp.so; cp follows the symlink. + cp -v /opt/rocm/lib/llvm/lib/libomp.so "$build_bin_path/" 2>/dev/null || echo "libomp.so not found" cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_elf.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_elf.so* not found" cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm.so* not found" @@ -938,6 +970,33 @@ jobs: [ -f "$file" ] && [ ! -L "$file" ] && patchelf --set-rpath '$ORIGIN' "$file" 2>/dev/null || true done + # Reject missing dependencies and runner libraries that mask an incomplete + # bundle. Only glibc, libstdc++ and libgcc_s may resolve outside the bundle. + - name: Check the bundle resolves its own dependencies + run: | + cd llama.cpp/build/bin + bundle="$(pwd -P)" + system='^((libc|libm|libdl|libpthread|librt|libstdc\+\+|libgcc_s)\.so\.[0-9]+|ld-linux-x86-64\.so\.2)$' + bad="" + for file in *.so* llama-*; do + [ -f "$file" ] && [ ! -L "$file" ] || continue + while read -r name arrow path _; do + # vdso (and on some hosts the dynamic loader) is listed without "=>". + [ "$arrow" = "=>" ] || continue + if [ "$path" = "not" ]; then + bad+="$file: $name not found"$'\n' + elif [[ ! "$name" =~ $system ]] && [ "$(dirname "$(readlink -f "$path")")" != "$bundle" ]; then + bad+="$file: $name resolved outside the bundle ($path)"$'\n' + fi + done < <(env -u LD_LIBRARY_PATH ldd "$file" 2>/dev/null || true) + done + if [ -n "$bad" ]; then + printf '%s' "$bad" + echo "::error::the bundle has dependencies it does not carry" + exit 1 + fi + echo "every packaged library resolves inside the bundle or to the base C/C++ runtime" + - name: List build artifacts (including ROCm files) run: | cd llama.cpp/build/bin diff --git a/scripts/unsloth/package_bundle.py b/scripts/unsloth/package_bundle.py index 4bd2dc9fba66..68d35ab53d13 100644 --- a/scripts/unsloth/package_bundle.py +++ b/scripts/unsloth/package_bundle.py @@ -12,9 +12,9 @@ convention, backend glob, and archive format), not writing a new packaging script. -The CUDA runtime (libcudart/libcublas, cudart DLLs) is intentionally NOT -bundled: the installer pairs it with the user's PyTorch runtime, selected by -runtime_line. +The installer normally uses PyTorch's CUDA runtime, selected by runtime_line. +BUNDLE_CUDA_RUNTIME=1 includes cudart/cublas DLLs for Windows ARM64, where +PyTorch has no CUDA build. Linux is the CI-validated path. macOS/Windows strategies follow the correct platform conventions (otool/@loader_path/tar.gz; dir-local DLLs/zip) but have @@ -146,11 +146,15 @@ class WindowsStrategy(PlatformStrategy): # ggml-cpu.dll imports it by name, so a curated bundle built from that tree # (the arm64 CUDA leg) has to keep it or nothing in the bundle loads. LOCAL_DLL_PREFIXES = ("ggml", "llama", "mtmd", "libomp") + CUDA_RUNTIME_DLL_PREFIXES = ("cudart64_", "cublas64_", "cublaslt64_") def local_needed(self, path: Path, bin_dir: Path) -> list[str]: + prefixes = self.LOCAL_DLL_PREFIXES + if os.environ.get("BUNDLE_CUDA_RUNTIME") == "1": + prefixes += self.CUDA_RUNTIME_DLL_PREFIXES return [ p.name for p in bin_dir.glob("*.dll") - if p.name.lower().startswith(self.LOCAL_DLL_PREFIXES) + if p.name.lower().startswith(prefixes) ] def backend_patterns(self) -> list[str]: From 9c124eab54e4e7096e821c2c6ba7bc8efe662682 Mon Sep 17 00:00:00 2001 From: Daniel Han <23090290+danielhanchen@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:53:29 +0000 Subject: [PATCH 2/3] Drop two comments that restate the step --- .github/workflows/unsloth-prebuilt-cuda-windows-arm64.yml | 1 - .github/workflows/unsloth-prebuilt-rocm.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/unsloth-prebuilt-cuda-windows-arm64.yml b/.github/workflows/unsloth-prebuilt-cuda-windows-arm64.yml index af2e2af2158c..3d96e3112e1b 100644 --- a/.github/workflows/unsloth-prebuilt-cuda-windows-arm64.yml +++ b/.github/workflows/unsloth-prebuilt-cuda-windows-arm64.yml @@ -348,7 +348,6 @@ jobs: name: app-${{ inputs.tag }}-windows-arm64-cpu path: cpubundle - # Add the CUDA backend to the CPU bundle, which includes OpenMP. # Bundle the CUDA runtime because PyTorch has no Windows ARM64 CUDA wheels. - name: Compose the bundle input run: | diff --git a/.github/workflows/unsloth-prebuilt-rocm.yml b/.github/workflows/unsloth-prebuilt-rocm.yml index 25f0f4ec07c0..7b1d1377a6f6 100644 --- a/.github/workflows/unsloth-prebuilt-rocm.yml +++ b/.github/workflows/unsloth-prebuilt-rocm.yml @@ -509,7 +509,6 @@ jobs: exit 1 } - # Catch runtime dependencies missing from the manual copy list. - name: Check the bundle carries every ROCm DLL it imports run: | python -m pip install --quiet --disable-pip-version-check pefile From 62c131bb17a1a64560dee7669e405099b54751f2 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:33:14 -0300 Subject: [PATCH 3/3] Let ROCm's clang build the Windows x86 CPU variants --- .github/workflows/unsloth-prebuilt-rocm.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/unsloth-prebuilt-rocm.yml b/.github/workflows/unsloth-prebuilt-rocm.yml index 7b1d1377a6f6..2edac5384d50 100644 --- a/.github/workflows/unsloth-prebuilt-rocm.yml +++ b/.github/workflows/unsloth-prebuilt-rocm.yml @@ -409,9 +409,12 @@ jobs: REM Only the C/CXX launchers here: ggml-hip forces CXX_IS_HIPCC on REM WIN32, so the device sources compile as C++, not the HIP language. REM Keep native CPU detection; select CPU variants at runtime. + REM ROCm's clang rejects ggml's _mm_prefetch calls against MSVC's declaration + REM (arch/x86/quants.c); upstream avoids it by building only ggml-hip here. cmake .. -G Ninja ^ -DCMAKE_C_COMPILER="C:\opt\rocm\lib\llvm\bin\clang.exe" ^ -DCMAKE_CXX_COMPILER="C:\opt\rocm\lib\llvm\bin\clang++.exe" ^ + -DCMAKE_C_FLAGS="-Wno-error=incompatible-pointer-types" ^ -DCMAKE_CXX_FLAGS="-IC:\opt\rocm\include" ^ -DCMAKE_BUILD_TYPE=Release ^ -DGPU_TARGETS="%mapped_target%" ^