Fix ROCm CPU performance, update the ROCm toolchain, and bundle Windows ARM64 CUDA libraries

.github/workflows/unsloth-prebuilt-cuda-windows-arm64.yml

@@ -14,9 +14,8 @@ name: "Unsloth prebuilt: CUDA Windows ARM64"
 #     compiles on an x64 runner with the amd64_arm64 MSVC toolset, the same way
 #     unsloth-prebuilt-cpu.yml already cross compiles the Windows arm64 CPU
 #     bundle, and the same way ggml-org's release.yml builds its arm64 CUDA zip.
-#   * Only CUDA 13.4 ships Windows ARM64 target libraries (cudart + cublas), and
-#     only from packages.nvidia.com rather than the redist CDN the x64 legs use,
-#     so it needs its own component list, not another row in the x64 table.
+#   * CUDA 13.4 needs a separate component list: x86_64 host tools and ARM64
+#     target libraries.
 #   * It builds one target (ggml-cuda), not the whole tool set: everything else
 #     in the bundle comes from the arm64 CPU bundle this run already builds.
 #     That is exactly upstream's arrangement -- their arm64 CUDA job packs
@@ -148,23 +147,19 @@ jobs:
         with:
           arch: amd64_arm64
 
-      # The x64 legs install from the redist CDN, which has no ARM64 target
-      # libraries at any version. 13.4 is the first toolkit that ships them and
-      # it lives in the bin-archive pool instead; the host-side pieces (nvcc,
-      # crt, nvvm, cccl) stay x86_64 because the compiler runs on this runner.
-      # Same component set and same pool ggml-org's release.yml installs.
+      # CUDA 13.4.1 GA: x86_64 compiler tools and ARM64 runtime libraries,
+      # matching ggml-org/llama.cpp#28687.
       - name: Install CUDA toolkit 13.4 (x86_64 host, ARM64 target)
         run: |
           $ErrorActionPreference = 'Stop'
-          $pool = 'https://packages.nvidia.com/bin-archive/pool'
-          $guid = '5B515474-7E78-11F1-8656-C51E4F4B317F'
+          $pool = 'https://developer.download.nvidia.com/compute/cuda/redist'
           $components = @(
-            "windows-x86_64/$guid/cccl-windows-x86_64-13.3.4.1.2-archive",
-            "windows-x86_64/$guid/cuda_crt-windows-x86_64-13.4.46-archive",
-            "windows-x86_64/$guid/cuda_nvcc-windows-x86_64-13.4.46-archive",
-            "windows-x86_64/$guid/libnvvm-windows-x86_64-13.4.46-archive",
-            "windows-arm64/$guid/cuda_cudart-windows-arm64-13.4.46-archive",
-            "windows-arm64/$guid/libcublas-windows-arm64-13.7.0.10-archive"
+            "cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive",
+            "cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive",
+            "cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive",
+            "libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive",
+            "cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.4.49-archive",
+            "libcublas/windows-arm64/libcublas-windows-arm64-13.7.0.27-archive"
           )
           $prefix = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ env.CUDA }}"
           $stage = Join-Path $env:RUNNER_TEMP 'cuda-redist'
@@ -353,15 +348,18 @@ jobs:
           name: app-${{ inputs.tag }}-windows-arm64-cpu
           path: cpubundle
 
-      # The CPU zip is the whole build/bin/Release tree plus the OpenMP runtime.
-      # Dropping the backend in beside it gives package_bundle.py the same shape
-      # of input directory a native build tree would have.
+      # Bundle the CUDA runtime because PyTorch has no Windows ARM64 CUDA wheels.
       - name: Compose the bundle input
         run: |
           $zip = "cpubundle/app-${{ inputs.tag }}-windows-arm64-cpu.zip"
           if (-not (Test-Path $zip)) { throw "the CPU leg's artifact does not contain $zip" }
           Expand-Archive -Path $zip -DestinationPath bin -Force
           Copy-Item cudabin/ggml-cuda.dll bin/ -Force
+          foreach ($dll in @('cudart64_13.dll', 'cublas64_13.dll', 'cublasLt64_13.dll')) {
+            $src = Join-Path "$env:CUDA_PATH\bin\arm64" $dll
+            if (-not (Test-Path $src)) { throw "the toolkit has no $src" }
+            Copy-Item $src bin/ -Force
+          }
           foreach ($required in @('llama-server.exe', 'ggml-base.dll', 'ggml-cpu.dll')) {
             if (-not (Test-Path (Join-Path 'bin' $required))) { throw "the CPU bundle is missing $required" }
           }
@@ -383,7 +381,8 @@ jobs:
           KLASS: ${{ env.KLASS }}
           RANK: ${{ env.RANK }}
           TOOLKIT_LINE: ${{ env.CUDA }}
-          DOCKER_IMAGE: github-hosted windows-2022 cross-compiled to arm64, CUDA ${{ env.CUDA }} (NVIDIA bin-archive)
+          DOCKER_IMAGE: github-hosted windows-2022 cross-compiled to arm64, CUDA ${{ env.CUDA }} (NVIDIA redist)
+          BUNDLE_CUDA_RUNTIME: '1'
           ARCHS: ${{ env.ARCHS }}
         run: python tooling/scripts/unsloth/package_bundle.py
 

.github/workflows/unsloth-prebuilt-rocm.yml

@@ -232,9 +232,8 @@ jobs:
           Write-Host "Using target with -all suffix: $archiveTarget"
         }
 
-        # TheRock publishes nightlies to the multi-arch tarball index. The
-        # static HTML page embeds a JSON `files` array with names and mtimes.
-        $baseUrl = "https://rocm.nightlies.amd.com/tarball-multi-arch"
+        # The index embeds a JSON `files` array with names and mtimes.
+        $baseUrl = "https://nightly.repo.amd.com/rocm/core/tarball"
         if ($rocmVersion -eq "latest" -or $rocmVersion -eq "weekly") {
           # weekly: see the Linux job.
           $cutoff = "99999999"
@@ -409,11 +408,14 @@ jobs:
         REM Configure the project
         REM Only the C/CXX launchers here: ggml-hip forces CXX_IS_HIPCC on
         REM WIN32, so the device sources compile as C++, not the HIP language.
+        REM Keep native CPU detection; select CPU variants at runtime.
+        REM ROCm's clang rejects ggml's _mm_prefetch calls against MSVC's declaration
+        REM (arch/x86/quants.c); upstream avoids it by building only ggml-hip here.
         cmake .. -G Ninja ^
           -DCMAKE_C_COMPILER="C:\opt\rocm\lib\llvm\bin\clang.exe" ^
           -DCMAKE_CXX_COMPILER="C:\opt\rocm\lib\llvm\bin\clang++.exe" ^
+          -DCMAKE_C_FLAGS="-Wno-error=incompatible-pointer-types" ^
           -DCMAKE_CXX_FLAGS="-IC:\opt\rocm\include" ^
-          -DCMAKE_CROSSCOMPILING=ON ^
           -DCMAKE_BUILD_TYPE=Release ^
           -DGPU_TARGETS="%mapped_target%" ^
           -DBUILD_SHARED_LIBS=ON ^
@@ -426,9 +428,10 @@ jobs:
           -DLLAMA_BUILD_BORINGSSL=ON ^
           -DGGML_NATIVE=OFF ^
           -DGGML_STATIC=OFF ^
+          -DGGML_BACKEND_DL=ON ^
+          -DGGML_CPU_ALL_VARIANTS=ON ^
           -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }} ^
-          -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }} ^
-          -DCMAKE_SYSTEM_NAME=Windows
+          -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }}
 
         REM Build the project
         cmake --build . -j %NUMBER_OF_PROCESSORS%
@@ -455,7 +458,8 @@ jobs:
             "hipblaslt.dll",
             "libhipblaslt.dll",
             "hipblas.dll",
-            "origami.dll"
+            "origami.dll",
+            "libtensilelite-host.dll"
           )
 
           foreach ($pattern in $filesToCopy) {
@@ -508,6 +512,32 @@ jobs:
           exit 1
         }
 
+    - name: Check the bundle carries every ROCm DLL it imports
+      run: |
+        python -m pip install --quiet --disable-pip-version-check pefile
+        @'
+        import os, sys, pefile
+        bundle, rocm = r"llama.cpp\build\bin", r"C:\opt\rocm\bin"
+        have = {f.lower() for f in os.listdir(bundle)}
+        rocm_dlls = {f.lower() for f in os.listdir(rocm) if f.lower().endswith(".dll")}
+        dirs = [pefile.DIRECTORY_ENTRY[d] for d in ("IMAGE_DIRECTORY_ENTRY_IMPORT", "IMAGE_DIRECTORY_ENTRY_DELAY_IMPORT")]
+        missing = []
+        for name in sorted(have):
+            if not name.endswith((".dll", ".exe")):
+                continue
+            pe = pefile.PE(os.path.join(bundle, name), fast_load=True)
+            pe.parse_data_directories(directories=dirs)
+            for entry in getattr(pe, "DIRECTORY_ENTRY_IMPORT", []) + getattr(pe, "DIRECTORY_ENTRY_DELAY_IMPORT", []):
+                dep = entry.dll.decode().lower()
+                if dep in rocm_dlls and dep not in have:
+                    missing.append(f"{name} -> {dep}")
+        if missing:
+            print("\n".join(missing))
+            print("::error::the bundle imports ROCm DLLs it does not carry")
+            sys.exit(1)
+        print("every imported ROCm DLL is in the bundle")
+        '@ | python -
+
     - name: List build artifacts (including ROCm files)
       run: |
         cd llama.cpp\build\bin
@@ -659,9 +689,8 @@ jobs:
           echo "Using target with -all suffix: $archive_target"
         fi
 
-        # TheRock publishes nightlies to the multi-arch tarball index. The
-        # static HTML page embeds a JSON `files` array with names and mtimes.
-        base_url="https://rocm.nightlies.amd.com/tarball-multi-arch"
+        # The index embeds a JSON `files` array with names and mtimes.
+        base_url="https://nightly.repo.amd.com/rocm/core/tarball"
         if [ "$rocm_version" = "latest" ] || [ "$rocm_version" = "weekly" ]; then
           # weekly: take the newest alpha up to the Saturday before the last SF Sunday, so a whole week uses one toolchain and the ccache hits.
           # The cutoff day must be settled before the first run reads it: TheRock usually publishes the evening before, but has landed as late as 18:33 PT on the named day, which splits the week.
@@ -826,11 +855,12 @@ jobs:
         # CXX compiler is ROCm's clang++ (not hipcc), so ggml-hip leaves
         # CXX_IS_HIPCC false and calls enable_language(HIP), putting the device
         # sources on the HIP language rather than CXX.
+        # Setting CMAKE_SYSTEM_NAME without a processor forces generic CPU code.
+        # Keep native detection and select CPU variants at runtime.
         cmake .. -G Ninja \
           -DCMAKE_C_COMPILER=/opt/rocm/llvm/bin/clang \
           -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \
           -DCMAKE_CXX_FLAGS="-I/opt/rocm/include" \
-          -DCMAKE_CROSSCOMPILING=ON \
           -DCMAKE_BUILD_TYPE=Release \
           -DGPU_TARGETS="$mapped_target" \
           -DBUILD_SHARED_LIBS=ON \
@@ -843,10 +873,11 @@ jobs:
           -DLLAMA_BUILD_BORINGSSL=ON \
           -DGGML_NATIVE=OFF \
           -DGGML_STATIC=OFF \
+          -DGGML_BACKEND_DL=ON \
+          -DGGML_CPU_ALL_VARIANTS=ON \
           -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }} \
           -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }} \
-          -DCMAKE_HIP_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }} \
-          -DCMAKE_SYSTEM_NAME=Linux
+          -DCMAKE_HIP_COMPILER_LAUNCHER=${{ env.CCACHE_LAUNCHER }}
 
         # Build the project
         cmake --build . -j $(nproc)
@@ -907,11 +938,14 @@ jobs:
         cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_numa.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_numa.so* not found"
         cp -v /opt/rocm/lib/librocroller.so* "$build_bin_path/" 2>/dev/null || echo "librocroller.so* not found"
         cp -v /opt/rocm/lib/liborigami.so* "$build_bin_path/" 2>/dev/null || echo "liborigami.so* not found"
+        cp -v /opt/rocm/lib/libtensilelite-host.so* "$build_bin_path/" 2>/dev/null || echo "libtensilelite-host.so* not found"
         cp -v /opt/rocm/lib/librocm_kpack.so* "$build_bin_path/" 2>/dev/null || echo "librocm_kpack.so* not found"
         cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_z.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_z.so* not found"
         cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_zstd.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_zstd.so* not found"
         cp -v /opt/rocm/lib/llvm/lib/libLLVM.so* "$build_bin_path/" 2>/dev/null || echo "libLLVM.so* not found"
         cp -v /opt/rocm/lib/llvm/lib/libclang-cpp.so* "$build_bin_path/" 2>/dev/null || echo "libclang-cpp.so* not found"
+        # libtensilelite-host needs the unversioned libomp.so; cp follows the symlink.
+        cp -v /opt/rocm/lib/llvm/lib/libomp.so "$build_bin_path/" 2>/dev/null || echo "libomp.so not found"
 
         cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_elf.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_elf.so* not found"
         cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm.so* not found"
@@ -938,6 +972,33 @@ jobs:
           [ -f "$file" ] && [ ! -L "$file" ] && patchelf --set-rpath '$ORIGIN' "$file" 2>/dev/null || true
         done
 
+    # Reject missing dependencies and runner libraries that mask an incomplete
+    # bundle. Only glibc, libstdc++ and libgcc_s may resolve outside the bundle.
+    - name: Check the bundle resolves its own dependencies
+      run: |
+        cd llama.cpp/build/bin
+        bundle="$(pwd -P)"
+        system='^((libc|libm|libdl|libpthread|librt|libstdc\+\+|libgcc_s)\.so\.[0-9]+|ld-linux-x86-64\.so\.2)$'
+        bad=""
+        for file in *.so* llama-*; do
+          [ -f "$file" ] && [ ! -L "$file" ] || continue
+          while read -r name arrow path _; do
+            # vdso (and on some hosts the dynamic loader) is listed without "=>".
+            [ "$arrow" = "=>" ] || continue
+            if [ "$path" = "not" ]; then
+              bad+="$file: $name not found"$'\n'
+            elif [[ ! "$name" =~ $system ]] && [ "$(dirname "$(readlink -f "$path")")" != "$bundle" ]; then
+              bad+="$file: $name resolved outside the bundle ($path)"$'\n'
+            fi
+          done < <(env -u LD_LIBRARY_PATH ldd "$file" 2>/dev/null || true)
+        done
+        if [ -n "$bad" ]; then
+          printf '%s' "$bad"
+          echo "::error::the bundle has dependencies it does not carry"
+          exit 1
+        fi
+        echo "every packaged library resolves inside the bundle or to the base C/C++ runtime"
+
     - name: List build artifacts (including ROCm files)
       run: |
         cd llama.cpp/build/bin

scripts/unsloth/package_bundle.py

@@ -12,9 +12,9 @@
 convention, backend glob, and archive format), not writing a new packaging
 script.
 
-The CUDA runtime (libcudart/libcublas, cudart DLLs) is intentionally NOT
-bundled: the installer pairs it with the user's PyTorch runtime, selected by
-runtime_line.
+The installer normally uses PyTorch's CUDA runtime, selected by runtime_line.
+BUNDLE_CUDA_RUNTIME=1 includes cudart/cublas DLLs for Windows ARM64, where
+PyTorch has no CUDA build.
 
 Linux is the CI-validated path. macOS/Windows strategies follow the correct
 platform conventions (otool/@loader_path/tar.gz; dir-local DLLs/zip) but have
@@ -146,11 +146,15 @@ class WindowsStrategy(PlatformStrategy):
     # ggml-cpu.dll imports it by name, so a curated bundle built from that tree
     # (the arm64 CUDA leg) has to keep it or nothing in the bundle loads.
     LOCAL_DLL_PREFIXES = ("ggml", "llama", "mtmd", "libomp")
+    CUDA_RUNTIME_DLL_PREFIXES = ("cudart64_", "cublas64_", "cublaslt64_")
 
     def local_needed(self, path: Path, bin_dir: Path) -> list[str]:
+        prefixes = self.LOCAL_DLL_PREFIXES
+        if os.environ.get("BUNDLE_CUDA_RUNTIME") == "1":
+            prefixes += self.CUDA_RUNTIME_DLL_PREFIXES
         return [
             p.name for p in bin_dir.glob("*.dll")
-            if p.name.lower().startswith(self.LOCAL_DLL_PREFIXES)
+            if p.name.lower().startswith(prefixes)
         ]
 
     def backend_patterns(self) -> list[str]: